- Devel - libvirt List Archives

[PATCH] os-posix: Allow 'chroot' via '-run-with' and deprecate the old '-chroot' option
by Thomas Huth 30 Jun '23

30 Jun '23

We recently introduced "-run-with" for options that influence the runtime behavior of QEMU. This option has the big advantage that it can group related options (so that it is easier for the users to spot them) and that the options become introspectable via QMP this way. So let's start moving more switches into this option group, starting with "-chroot" now. Signed-off-by: Thomas Huth <thuth(a)redhat.com> --- docs/about/deprecated.rst | 5 +++++ os-posix.c | 35 ++++++++++++++++++++++++++++++++++- util/async-teardown.c | 21 --------------------- qemu-options.hx | 18 +++++++++++++----- 4 files changed, 52 insertions(+), 27 deletions(-) diff --git a/docs/about/deprecated.rst b/docs/about/deprecated.rst index 0743459862..1cf53b86ce 100644 --- a/docs/about/deprecated.rst +++ b/docs/about/deprecated.rst @@ -116,6 +116,11 @@ Use "whpx" (on Windows) or "hvf" (on macOS) instead. Use ``-run-with async-teardown=on`` instead. +``-chroot`` (since 8.1) +''''''''''''''''''''''' + +Use ``-run-with chroot=dir`` instead. + ``-singlestep`` (since 8.1) ''''''''''''''''''''''''''' diff --git a/os-posix.c b/os-posix.c index 90ea71725f..0ae1fb2347 100644 --- a/os-posix.c +++ b/os-posix.c @@ -38,6 +38,7 @@ #include "qemu/cutils.h" #include "qemu/config-file.h" #include "qemu/option.h" +#include "qemu/module.h" #ifdef CONFIG_LINUX #include <sys/prctl.h> @@ -148,6 +149,7 @@ int os_parse_cmd_args(int index, const char *optarg) } break; case QEMU_OPTION_chroot: + warn_report("option is deprecated, use '-run-with chroot=...' instead"); chroot_dir = optarg; break; case QEMU_OPTION_daemonize: @@ -158,18 +160,25 @@ int os_parse_cmd_args(int index, const char *optarg) case QEMU_OPTION_asyncteardown: init_async_teardown(); break; +#endif case QEMU_OPTION_run_with: { + const char *str; QemuOpts *opts = qemu_opts_parse_noisily(qemu_find_opts("run-with"), optarg, false); if (!opts) { exit(1); } +#if defined(CONFIG_LINUX) if (qemu_opt_get_bool(opts, "async-teardown", false)) { init_async_teardown(); } +#endif + str = qemu_opt_get(opts, "chroot"); + if (str) { + chroot_dir = str; + } break; } -#endif default: return -1; } @@ -348,3 +357,27 @@ int os_mlock(void) return -ENOSYS; #endif } + +static QemuOptsList qemu_run_with_opts = { + .name = "run-with", + .head = QTAILQ_HEAD_INITIALIZER(qemu_run_with_opts.head), + .desc = { +#if defined(CONFIG_LINUX) + { + .name = "async-teardown", + .type = QEMU_OPT_BOOL, + }, +#endif + { + .name = "chroot", + .type = QEMU_OPT_STRING, + }, + { /* end of list */ } + }, +}; + +static void register_teardown(void) +{ + qemu_add_opts(&qemu_run_with_opts); +} +opts_init(register_teardown); diff --git a/util/async-teardown.c b/util/async-teardown.c index 3ab19c8740..62cdeb0f20 100644 --- a/util/async-teardown.c +++ b/util/async-teardown.c @@ -12,9 +12,6 @@ */ #include "qemu/osdep.h" -#include "qemu/config-file.h" -#include "qemu/option.h" -#include "qemu/module.h" #include <dirent.h> #include <sys/prctl.h> #include <sched.h> @@ -147,21 +144,3 @@ void init_async_teardown(void) clone(async_teardown_fn, new_stack_for_clone(), CLONE_VM, NULL); sigprocmask(SIG_SETMASK, &old_signals, NULL); } - -static QemuOptsList qemu_run_with_opts = { - .name = "run-with", - .head = QTAILQ_HEAD_INITIALIZER(qemu_run_with_opts.head), - .desc = { - { - .name = "async-teardown", - .type = QEMU_OPT_BOOL, - }, - { /* end of list */ } - }, -}; - -static void register_teardown(void) -{ - qemu_add_opts(&qemu_run_with_opts); -} -opts_init(register_teardown); diff --git a/qemu-options.hx b/qemu-options.hx index b57489d7ca..f49d4c0e3c 100644 --- a/qemu-options.hx +++ b/qemu-options.hx @@ -4670,11 +4670,12 @@ ERST #ifndef _WIN32 DEF("chroot", HAS_ARG, QEMU_OPTION_chroot, \ - "-chroot dir chroot to dir just before starting the VM\n", + "-chroot dir chroot to dir just before starting the VM (deprecated)\n", QEMU_ARCH_ALL) #endif SRST ``-chroot dir`` + Deprecated, use '-run-with chroot=...' instead. Immediately before starting guest execution, chroot to the specified directory. Especially useful in combination with -runas. ERST @@ -4861,13 +4862,16 @@ SRST This option is deprecated and should no longer be used. The new option ``-run-with async-teardown=on`` is a replacement. ERST +#endif +#ifdef CONFIG_POSIX DEF("run-with", HAS_ARG, QEMU_OPTION_run_with, - "-run-with async-teardown[=on|off]\n" - " misc QEMU process lifecycle options\n" - " async-teardown=on enables asynchronous teardown\n", + "-run-with [async-teardown=on|off][,chroot=dir]\n" + " Set miscellaneous QEMU process lifecycle options:\n" + " async-teardown=on enables asynchronous teardown (Linux only)\n" + " chroot=dir chroot to dir just before starting the VM\n", QEMU_ARCH_ALL) SRST -``-run-with`` +``-run-with [async-teardown=on|off][,chroot=dir]`` Set QEMU process lifecycle options. ``async-teardown=on`` enables asynchronous teardown. A new process called @@ -4880,6 +4884,10 @@ SRST performed correctly. This only works if the cleanup process is not forcefully killed with SIGKILL before the main QEMU process has terminated completely. + + ``chroot=dir`` can be used for doing a chroot to the specified directory + immediately before starting the guest execution. This is especially useful + in combination with -runas. ERST #endif -- 2.39.3

4 4

[PATCH v6] vfio/pci: Propagate ACPI notifications to user-space via eventfd
by Grzegorz Jaszczyk 30 Jun '23

30 Jun '23

To allow pass-through devices receiving ACPI notifications, permit to register ACPI notify handler (via VFIO_DEVICE_SET_IRQS) for a given device. The handler role is to receive and propagate such ACPI notifications to the user-space through the user provided eventfd. This allows VMM to receive and propagate them further to the VM, where the actual driver for pass-through device resides and can react to device specific notifications accordingly. The eventfd usage ensures VMM and device isolation: it allows to use a dedicated channel associated with the device for such events, such that the VMM has direct access. Since the eventfd counter is used as ACPI notification value placeholder, the eventfd signaling needs to be serialized in order to not end up with notification values being coalesced. Therefore ACPI notification values are buffered and signalized one by one, when the previous notification value has been consumed. Signed-off-by: Grzegorz Jaszczyk <jaz(a)semihalf.com> --- Changelog v5..v6 - Get rid of #if IS_ENABLED(CONFIG_ACPI) from vfio_pci_intrs.c - There is no need to keep vfio_acpi_notification struct definition in header, make it private and move to cpi_notify.c - Add lockdep_asserts in acpi_notification_dequeue - notification_list_lock was not initialized which triggers some warning with CONFIG_DEBUG_MUTEXES enabled (it was not logical error since mentioned mutex was zeroed thanks to kzalloc it is just DEBUG_MUTEXES performs extra checks and sets lock->magic). - Change notification_lock from mutex to semaphore (notification_sem). The logic was working fine with the mutex but enabling lockdep support made me realize that the mutex has to be released by the same task which acquires it and it is checked with lockdep enabled. Semaphore does not have such a restriction. - In order to correctly handle some sequences such as: * user configures a notification eventfd * a notification fires * the user closes the eventfd without reading it, and then tries to swap in a different eventfd ctx separate the entire eventfd context together with notification_sem from the main vfio_acpi_notification struct. Thanks to that entire, separate eventfd ctx could be swapped leaving the previous eventfd ctx untouched. The new eventfd has its separate notification_sem initialized and will not depend on previous eventfd ctx. Additionally when the last handle to old eventfd will be released and EPOLLHUP triggered, old eventfd ctx will be cleaned-up. - Together with the above add support for required EPOLLHUP handling. - Add missing fput invocations. - Move some common code into vfio_acpi_eventfd_init and use it during registration and eventfd swapping. - v5: https://patchwork.kernel.org/project/kvm/patch/20230609133950.2197552-1-jaz… Changelog v4..v5 Address Alex Williamson's feedback: - s/vfio_acpi_notify.{c,o}/acpi_notify.{c,o} - Do not put acpi_notify to its own module but fold it into main vfio.ko. Additionally select it from VFIO_PCI_CORE instead of VFIO_PCI. - Cleanup acpi notify under igate mutex (in vfio_pci_core_close_device). - Add extra check for ACPI companion in vfio_pci_get_irq_count and extend vfio_pci_ioctl_get_irq_info. - Drop acpi.h include - linux/vfio_acpi_notify.h includes it already. - Send device check notification value for DATA_NONE and non-zero count and for DATA_BOOL and non-zero count (as for loopback testing). - Drop some redundant !acpi_notify->acpi_notify_trigger checks. - Move some common code to new helper functions: 1) acpi_notification_dequeue 2) vfio_acpi_notify_cleanup and rename previous vfio_acpi_notify_cleanup into vfio_remove_acpi_notify which uses it - Add rate limited logging for dropped notifications. - Move vdev->acpi_notification pointer cleanup to the vfio_acpi_notify_cleanup function this also fixes two bigger issues caught by Alex. - Allow the eventfd to be swapped. - s/GFP_KERNEL/GFP_KERNEL_ACCOUNT. - s/VFIO_PCI_ACPI_NTFY_IRQ_INDEX/VFIO_PCI_ACPI_IRQ_INDEX. - Add header protection for multiple includes. - v4: https://patchwork.kernel.org/project/kvm/patch/20230522165811.123417-1-jaz@… Changelog v3..v4 Address Alex Williamson feedback: - Instead of introducing new ioctl used for eventfd registration, take advantage of VFIO_DEVICE_SET_IRQS which already supports virtual IRQs for things like error notification and device release requests. - Introduced mechanism preventing creation of large queues. Other: - Move the implementation into the newly introduced VFIO_ACPI_NOTIFY helper module. It is actually not bound to VFIO_PCI but VFIO_PCI enables it whenever ACPI support is enabled. This change is introduced since ACPI notifications are not limited to PCI devices, making it PCI independent will allow to re-use it also for other VFIO_* like supports: e.g. VFIO_PLATFORM in the future if needed. Moving it out of drivers/vfio/pci/ was also suggested offline. - s/notify_val_next/node - v3: https://patchwork.kernel.org/project/kvm/patch/20230502132700.654528-1-jasz… Changelog v2..v3: - Fix compilation warnings when building with "W=1" Changelog v1..v2: - The v2 implementation is actually completely different then v1: instead of using acpi netlink events for propagating ACPI notifications to the user space take advantage of eventfd, which can provide better VMM and device isolation: it allows to use a dedicated channel associated with the device for such events, such that the VMM has direct access. - Using eventfd counter as notification value placeholder was suggested in v1 and requires additional serialization logic introduced in v2. - Since the vfio-pci supports non-ACPI platforms address !CONFIG_ACPI case. - v1 discussion: https://patchwork.kernel.org/project/kvm/patch/20230307220553.631069-1-jaz@… --- drivers/vfio/Kconfig | 5 + drivers/vfio/Makefile | 1 + drivers/vfio/acpi_notify.c | 324 ++++++++++++++++++++++++++++++ drivers/vfio/pci/Kconfig | 1 + drivers/vfio/pci/vfio_pci_core.c | 13 ++ drivers/vfio/pci/vfio_pci_intrs.c | 80 ++++++++ include/linux/vfio_acpi_notify.h | 36 ++++ include/linux/vfio_pci_core.h | 1 + include/uapi/linux/vfio.h | 1 + 9 files changed, 462 insertions(+) create mode 100644 drivers/vfio/acpi_notify.c create mode 100644 include/linux/vfio_acpi_notify.h diff --git a/drivers/vfio/Kconfig b/drivers/vfio/Kconfig index 89e06c981e43..cd9df43a4eb4 100644 --- a/drivers/vfio/Kconfig +++ b/drivers/vfio/Kconfig @@ -12,6 +12,11 @@ menuconfig VFIO If you don't know what to do here, say N. if VFIO +config VFIO_ACPI_NOTIFY + bool + depends on ACPI + default n + config VFIO_CONTAINER bool "Support for the VFIO container /dev/vfio/vfio" select VFIO_IOMMU_TYPE1 if MMU && (X86 || S390 || ARM || ARM64) diff --git a/drivers/vfio/Makefile b/drivers/vfio/Makefile index 70e7dcb302ef..003c2b041785 100644 --- a/drivers/vfio/Makefile +++ b/drivers/vfio/Makefile @@ -7,6 +7,7 @@ vfio-y += vfio_main.o \ vfio-$(CONFIG_IOMMUFD) += iommufd.o vfio-$(CONFIG_VFIO_CONTAINER) += container.o vfio-$(CONFIG_VFIO_VIRQFD) += virqfd.o +vfio-$(CONFIG_VFIO_ACPI_NOTIFY) += acpi_notify.o obj-$(CONFIG_VFIO_IOMMU_TYPE1) += vfio_iommu_type1.o obj-$(CONFIG_VFIO_IOMMU_SPAPR_TCE) += vfio_iommu_spapr_tce.o diff --git a/drivers/vfio/acpi_notify.c b/drivers/vfio/acpi_notify.c new file mode 100644 index 000000000000..b78c33077dfa --- /dev/null +++ b/drivers/vfio/acpi_notify.c @@ -0,0 +1,324 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * VFIO ACPI notification propagation + * + * Author: Grzegorz Jaszczyk <jaz(a)semihalf.com> + */ +#include <linux/file.h> +#include <linux/semaphore.h> +#include <linux/vfio_acpi_notify.h> + +#define NOTIFICATION_QUEUE_SIZE 20 + +struct acpi_eventfd_ctx { + struct eventfd_ctx *acpi_notify_trigger; + struct file *acpi_notify_trigger_file; + struct semaphore notification_sem; + struct work_struct acpi_notification_work; + wait_queue_entry_t wait; + poll_table pt; + struct vfio_acpi_notification *acpi_notify; +}; + +struct vfio_acpi_notification { + struct acpi_eventfd_ctx *acpi_eventfd; + struct list_head notification_list; + struct mutex notification_list_lock; + int notification_queue_count; +}; + +struct notification_queue { + int notification_val; + struct list_head node; +}; + +static int vfio_eventfd_wakeup(wait_queue_entry_t *wait, unsigned int mode, + int sync, void *key) +{ + struct acpi_eventfd_ctx *acpi_eventfdctx = + container_of(wait, struct acpi_eventfd_ctx, wait); + __poll_t flags = key_to_poll(key); + + /* + * eventfd_read signalize EPOLLOUT at the end of its function - this + * means previous eventfd with its notification value was consumed so + * the next notification can be signalized now if pending - schedule + * proper work. + */ + if (flags & EPOLLOUT) { + up(&acpi_eventfdctx->notification_sem); + + schedule_work(&acpi_eventfdctx->acpi_notification_work); + } + + /* + * Even if the eventfd is closed lets still queue notifications so they + * can be replicated when new eventfd is registered (see "Allow eventfd + * to be swapped"). + * + * Below will be reached only in case user closes eventfd and then + * trigger eventfd swap (or vice-versa). + */ + if (flags & EPOLLHUP) { + /* + * eventfd_release after signalling EPOLLHUP calls eventfd_ctx_put + * so no need to do it here. + */ + + kfree(acpi_eventfdctx); + } + + return 0; +} + +static void vfio_ptable_queue_proc(struct file *file, + wait_queue_head_t *wqh, poll_table *pt) +{ + struct acpi_eventfd_ctx *acpi_eventfdctx = + container_of(pt, struct acpi_eventfd_ctx, pt); + + add_wait_queue(wqh, &acpi_eventfdctx->wait); +} + +static struct notification_queue * +acpi_notification_dequeue(struct vfio_acpi_notification *acpi_notify) +{ + struct notification_queue *oldest_entry; + + lockdep_assert_held(&acpi_notify->notification_list_lock); + + oldest_entry = list_first_entry(&acpi_notify->notification_list, + struct notification_queue, + node); + list_del(&oldest_entry->node); + acpi_notify->notification_queue_count--; + + return oldest_entry; +} + +static void acpi_notification_work_fn(struct work_struct *work) +{ + struct acpi_eventfd_ctx *acpi_eventfdctx; + struct vfio_acpi_notification *acpi_notify; + struct notification_queue *entry; + + acpi_eventfdctx = container_of(work, struct acpi_eventfd_ctx, + acpi_notification_work); + + acpi_notify = acpi_eventfdctx->acpi_notify; + + mutex_lock(&acpi_notify->notification_list_lock); + if (list_empty(&acpi_notify->notification_list)) + goto out; + + /* + * If the previous eventfd was not yet consumed by user-space lets hold + * on and exit. The notification function will be rescheduled when + * signaling eventfd will be possible (when the EPOLLOUT will be + * signalized and unlocks notify_events or when eventfd will be swapped). + */ + if (down_trylock(&acpi_eventfdctx->notification_sem)) + goto out; + + entry = acpi_notification_dequeue(acpi_notify); + + mutex_unlock(&acpi_notify->notification_list_lock); + + eventfd_signal(acpi_eventfdctx->acpi_notify_trigger, entry->notification_val); + + kfree(entry); + + return; +out: + mutex_unlock(&acpi_notify->notification_list_lock); +} + +static void +vfio_acpi_notify_cleanup(struct vfio_acpi_notification **acpi_notify_ptr, + struct acpi_device *adev) +{ + struct vfio_acpi_notification *acpi_notify = *acpi_notify_ptr; + struct acpi_eventfd_ctx *acpi_eventfd = acpi_notify->acpi_eventfd; + struct notification_queue *entry, *entry_tmp; + u64 cnt; + + eventfd_ctx_remove_wait_queue(acpi_eventfd->acpi_notify_trigger, + &acpi_eventfd->wait, &cnt); + + flush_work(&acpi_eventfd->acpi_notification_work); + + mutex_lock(&acpi_notify->notification_list_lock); + list_for_each_entry_safe(entry, entry_tmp, + &acpi_notify->notification_list, + node) { + list_del(&entry->node); + kfree(entry); + } + mutex_unlock(&acpi_notify->notification_list_lock); + + eventfd_ctx_put(acpi_eventfd->acpi_notify_trigger); + + /* + * fput releases references to eventfd file but it will not trigger the + * vfio_eventfd_wakeup with EPOLLHUP since eventfd_ctx_remove_wait_queue + * was already called and removed wq entry (wait) from the eventfd + * wq head. + */ + fput(acpi_eventfd->acpi_notify_trigger_file); + + kfree(acpi_notify->acpi_eventfd); + kfree(acpi_notify); + + *acpi_notify_ptr = NULL; +} + +static void vfio_acpi_notify_handler(acpi_handle handle, u32 event, void *data) +{ + struct vfio_acpi_notification *acpi_notify = (struct vfio_acpi_notification *)data; + struct notification_queue *entry; + + entry = kmalloc(sizeof(*entry), GFP_KERNEL); + if (!entry) + return; + + entry->notification_val = event; + INIT_LIST_HEAD(&entry->node); + + mutex_lock(&acpi_notify->notification_list_lock); + if (acpi_notify->notification_queue_count > NOTIFICATION_QUEUE_SIZE) { + struct notification_queue *oldest_entry = + acpi_notification_dequeue(acpi_notify); + + if (printk_ratelimit()) + acpi_handle_warn(handle, + "dropping notification value %d\n", + oldest_entry->notification_val); + + kfree(oldest_entry); + } + list_add_tail(&entry->node, &acpi_notify->notification_list); + acpi_notify->notification_queue_count++; + mutex_unlock(&acpi_notify->notification_list_lock); + + schedule_work(&acpi_notify->acpi_eventfd->acpi_notification_work); +} + +void vfio_acpi_notify(struct acpi_device *adev, u32 event, void *data) +{ + acpi_handle handle = adev->handle; + + vfio_acpi_notify_handler(handle, event, data); +} +EXPORT_SYMBOL_GPL(vfio_acpi_notify); + +void vfio_remove_acpi_notify(struct vfio_acpi_notification **acpi_notify_ptr, + struct acpi_device *adev) +{ + struct vfio_acpi_notification *acpi_notify = *acpi_notify_ptr; + + if (!acpi_notify) + return; + + acpi_remove_notify_handler(adev->handle, ACPI_DEVICE_NOTIFY, + vfio_acpi_notify_handler); + + vfio_acpi_notify_cleanup(acpi_notify_ptr, adev); +} +EXPORT_SYMBOL_GPL(vfio_remove_acpi_notify); + +static void vfio_acpi_eventfd_init(struct vfio_acpi_notification *acpi_notify, + struct eventfd_ctx *efdctx, int32_t fd) +{ + struct acpi_eventfd_ctx *acpi_eventfd; + + acpi_eventfd = kzalloc(sizeof(struct acpi_eventfd_ctx), GFP_KERNEL_ACCOUNT); + + INIT_WORK(&acpi_eventfd->acpi_notification_work, acpi_notification_work_fn); + acpi_eventfd->acpi_notify_trigger = efdctx; + + sema_init(&acpi_eventfd->notification_sem, 1); + + /* + * Install custom wake-up handler to be notified whenever underlying + * eventfd is consumed by the user-space. + */ + init_waitqueue_func_entry(&acpi_eventfd->wait, vfio_eventfd_wakeup); + init_poll_funcptr(&acpi_eventfd->pt, vfio_ptable_queue_proc); + + acpi_eventfd->acpi_notify_trigger_file = eventfd_fget(fd); + vfs_poll(acpi_eventfd->acpi_notify_trigger_file, &acpi_eventfd->pt); + + acpi_eventfd->acpi_notify = acpi_notify; + acpi_notify->acpi_eventfd = acpi_eventfd; +} + +int vfio_register_acpi_notify_handler(struct vfio_acpi_notification **acpi_notify_ptr, + struct acpi_device *adev, int32_t fd) +{ + struct vfio_acpi_notification *acpi_notify = *acpi_notify_ptr; + struct eventfd_ctx *efdctx; + acpi_status status; + + if (fd < -1) + return -EINVAL; + else if (fd == -1) { + vfio_remove_acpi_notify(acpi_notify_ptr, adev); + return 0; + } + + efdctx = eventfd_ctx_fdget(fd); + if (IS_ERR(efdctx)) + return PTR_ERR(efdctx); + + /* Allow eventfd to be swapped */ + if (acpi_notify) { + struct file *trigger_file_before_swap = + acpi_notify->acpi_eventfd->acpi_notify_trigger_file; + + /* + * Lets allocate new acpi_eventfd_ctx, the previous is + * alive until eventfd is closed. + */ + vfio_acpi_eventfd_init(acpi_notify, efdctx, fd); + + /* + * The ACPI notifications could arrive and be queued during + * eventfd swap, retrigger the worker after notification + * replication unlocking. + */ + schedule_work(&acpi_notify->acpi_eventfd->acpi_notification_work); + + /* + * If the last reference to acpi_notify_trigger_file was + * released, the EPOLLHUP will be handled (but not immediately + * since fput is async). + */ + fput(trigger_file_before_swap); + + return 0; + } + + acpi_notify = kzalloc(sizeof(*acpi_notify), GFP_KERNEL_ACCOUNT); + if (!acpi_notify) + return -ENOMEM; + + *acpi_notify_ptr = acpi_notify; + mutex_init(&acpi_notify->notification_list_lock); + INIT_LIST_HEAD(&acpi_notify->notification_list); + + vfio_acpi_eventfd_init(acpi_notify, efdctx, fd); + + status = acpi_install_notify_handler(adev->handle, ACPI_DEVICE_NOTIFY, + vfio_acpi_notify_handler, (void *)acpi_notify); + if (ACPI_FAILURE(status)) { + dev_err(&adev->dev, "Failed to install notify handler: %s", + acpi_format_exception(status)); + + vfio_acpi_notify_cleanup(acpi_notify_ptr, adev); + + return -ENODEV; + } + + return 0; +} +EXPORT_SYMBOL_GPL(vfio_register_acpi_notify_handler); diff --git a/drivers/vfio/pci/Kconfig b/drivers/vfio/pci/Kconfig index f9d0c908e738..f03ca773dfd9 100644 --- a/drivers/vfio/pci/Kconfig +++ b/drivers/vfio/pci/Kconfig @@ -4,6 +4,7 @@ config VFIO_PCI_CORE tristate select VFIO_VIRQFD select IRQ_BYPASS_MANAGER + select VFIO_ACPI_NOTIFY if ACPI config VFIO_PCI_MMAP def_bool y if !S390 diff --git a/drivers/vfio/pci/vfio_pci_core.c b/drivers/vfio/pci/vfio_pci_core.c index a5ab416cf476..1cc4a9c05403 100644 --- a/drivers/vfio/pci/vfio_pci_core.c +++ b/drivers/vfio/pci/vfio_pci_core.c @@ -27,6 +27,7 @@ #include <linux/vgaarb.h> #include <linux/nospec.h> #include <linux/sched/mm.h> +#include <linux/vfio_acpi_notify.h> #if IS_ENABLED(CONFIG_EEH) #include <asm/eeh.h> #endif @@ -683,6 +684,7 @@ void vfio_pci_core_close_device(struct vfio_device *core_vdev) { struct vfio_pci_core_device *vdev = container_of(core_vdev, struct vfio_pci_core_device, vdev); + struct acpi_device *adev = ACPI_COMPANION(&vdev->pdev->dev); if (vdev->sriov_pf_core_dev) { mutex_lock(&vdev->sriov_pf_core_dev->vf_token->lock); @@ -704,6 +706,8 @@ void vfio_pci_core_close_device(struct vfio_device *core_vdev) eventfd_ctx_put(vdev->req_trigger); vdev->req_trigger = NULL; } + if (adev) + vfio_remove_acpi_notify(&vdev->acpi_notification, adev); mutex_unlock(&vdev->igate); } EXPORT_SYMBOL_GPL(vfio_pci_core_close_device); @@ -725,6 +729,8 @@ EXPORT_SYMBOL_GPL(vfio_pci_core_finish_enable); static int vfio_pci_get_irq_count(struct vfio_pci_core_device *vdev, int irq_type) { + struct acpi_device *adev = ACPI_COMPANION(&vdev->pdev->dev); + if (irq_type == VFIO_PCI_INTX_IRQ_INDEX) { u8 pin; @@ -761,6 +767,8 @@ static int vfio_pci_get_irq_count(struct vfio_pci_core_device *vdev, int irq_typ return 1; } else if (irq_type == VFIO_PCI_REQ_IRQ_INDEX) { return 1; + } else if (adev && irq_type == VFIO_PCI_ACPI_IRQ_INDEX) { + return 1; } return 0; @@ -1084,6 +1092,7 @@ static int vfio_pci_ioctl_get_irq_info(struct vfio_pci_core_device *vdev, struct vfio_irq_info __user *arg) { unsigned long minsz = offsetofend(struct vfio_irq_info, count); + struct acpi_device *adev = ACPI_COMPANION(&vdev->pdev->dev); struct vfio_irq_info info; if (copy_from_user(&info, arg, minsz)) @@ -1096,6 +1105,10 @@ static int vfio_pci_ioctl_get_irq_info(struct vfio_pci_core_device *vdev, case VFIO_PCI_INTX_IRQ_INDEX ... VFIO_PCI_MSIX_IRQ_INDEX: case VFIO_PCI_REQ_IRQ_INDEX: break; + case VFIO_PCI_ACPI_IRQ_INDEX: + if (adev) + break; + return -EINVAL; case VFIO_PCI_ERR_IRQ_INDEX: if (pci_is_pcie(vdev->pdev)) break; diff --git a/drivers/vfio/pci/vfio_pci_intrs.c b/drivers/vfio/pci/vfio_pci_intrs.c index bffb0741518b..27d0c4cdc3c9 100644 --- a/drivers/vfio/pci/vfio_pci_intrs.c +++ b/drivers/vfio/pci/vfio_pci_intrs.c @@ -19,6 +19,7 @@ #include <linux/vfio.h> #include <linux/wait.h> #include <linux/slab.h> +#include <linux/vfio_acpi_notify.h> #include "vfio_pci_priv.h" @@ -667,6 +668,71 @@ static int vfio_pci_set_req_trigger(struct vfio_pci_core_device *vdev, count, flags, data); } +static int +vfio_pci_set_acpi_ntfy_trigger(struct vfio_pci_core_device *vdev, + unsigned int index, unsigned int start, + unsigned int count, uint32_t flags, void *data) +{ + struct acpi_device *adev = ACPI_COMPANION(&vdev->pdev->dev); + + if (index != VFIO_PCI_ACPI_IRQ_INDEX || start != 0 || count > 1) + return -EINVAL; + + if (!adev) + return -ENODEV; + + if (!vdev->acpi_notification) + return -EINVAL; + + /* + * Disable notifications: flags = (DATA_NONE|ACTION_TRIGGER), count = 0 + * Enable loopback testing: (DATA_BOOL|ACTION_TRIGGER) or + * (DATA_NONE|ACTION_TRIGGER), count != 0 + */ + if (flags & VFIO_IRQ_SET_DATA_NONE) { + if (count) + vfio_acpi_notify(adev, ACPI_NOTIFY_DEVICE_CHECK, + vdev->acpi_notification); + else + vfio_remove_acpi_notify(&vdev->acpi_notification, adev); + + return 0; + } else if (flags & VFIO_IRQ_SET_DATA_BOOL) { + uint8_t trigger; + + if (!count) + return -EINVAL; + + trigger = *(uint8_t *)data; + if (trigger) + vfio_acpi_notify(adev, ACPI_NOTIFY_DEVICE_CHECK, + vdev->acpi_notification); + + return 0; + } + + return -EINVAL; +} + +static int +vfio_pci_set_acpi_ntfy_eventfd_trigger(struct vfio_pci_core_device *vdev, + unsigned int index, unsigned int start, + unsigned int count, uint32_t flags, void *data) +{ + struct acpi_device *adev = ACPI_COMPANION(&vdev->pdev->dev); + int32_t fd; + + if (index != VFIO_PCI_ACPI_IRQ_INDEX || start != 0 || count != 1) + return -EINVAL; + + if (!adev) + return -ENODEV; + + fd = *(int32_t *)data; + + return vfio_register_acpi_notify_handler(&vdev->acpi_notification, adev, fd); +} + int vfio_pci_set_irqs_ioctl(struct vfio_pci_core_device *vdev, uint32_t flags, unsigned index, unsigned start, unsigned count, void *data) @@ -716,6 +782,20 @@ int vfio_pci_set_irqs_ioctl(struct vfio_pci_core_device *vdev, uint32_t flags, break; } break; + case VFIO_PCI_ACPI_IRQ_INDEX: + switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) { + case VFIO_IRQ_SET_ACTION_TRIGGER: + switch (flags & VFIO_IRQ_SET_DATA_TYPE_MASK) { + case VFIO_IRQ_SET_DATA_BOOL: + case VFIO_IRQ_SET_DATA_NONE: + func = vfio_pci_set_acpi_ntfy_trigger; + break; + case VFIO_IRQ_SET_DATA_EVENTFD: + func = vfio_pci_set_acpi_ntfy_eventfd_trigger; + break; + } + } + break; } if (!func) diff --git a/include/linux/vfio_acpi_notify.h b/include/linux/vfio_acpi_notify.h new file mode 100644 index 000000000000..e29e4274d192 --- /dev/null +++ b/include/linux/vfio_acpi_notify.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * VFIO ACPI notification replication + * + * Author: Grzegorz Jaszczyk <jaz(a)semihalf.com> + */ +#include <linux/acpi.h> +#include <linux/eventfd.h> +#include <linux/poll.h> + +#ifndef VFIO_ACPI_NOTIFY_H +#define VFIO_ACPI_NOTIFY_H + +struct vfio_acpi_notification; + +#if IS_ENABLED(CONFIG_ACPI) +void vfio_acpi_notify(struct acpi_device *adev, u32 event, void *data); +int vfio_register_acpi_notify_handler(struct vfio_acpi_notification **acpi_notify, + struct acpi_device *adev, int32_t fd); +void vfio_remove_acpi_notify(struct vfio_acpi_notification **acpi_notify, + struct acpi_device *adev); +#else +static inline void vfio_acpi_notify(struct acpi_device *adev, u32 event, void *data) {} +static inline int +vfio_register_acpi_notify_handler(struct vfio_acpi_notification **acpi_notify, + struct acpi_device *adev, int32_t fd) +{ + return -ENODEV; +} + +static inline void +vfio_remove_acpi_notify(struct vfio_acpi_notification **acpi_notify, + struct acpi_device *adev) {} +#endif /* CONFIG_ACPI */ + +#endif /* VFIO_ACPI_NOTIFY_H */ diff --git a/include/linux/vfio_pci_core.h b/include/linux/vfio_pci_core.h index 367fd79226a3..a4491b3d8064 100644 --- a/include/linux/vfio_pci_core.h +++ b/include/linux/vfio_pci_core.h @@ -96,6 +96,7 @@ struct vfio_pci_core_device { struct mutex vma_lock; struct list_head vma_list; struct rw_semaphore memory_lock; + struct vfio_acpi_notification *acpi_notification; }; /* Will be exported for vfio pci drivers usage */ diff --git a/include/uapi/linux/vfio.h b/include/uapi/linux/vfio.h index b71276bd7f91..ee8ba6354662 100644 --- a/include/uapi/linux/vfio.h +++ b/include/uapi/linux/vfio.h @@ -625,6 +625,7 @@ enum { VFIO_PCI_MSIX_IRQ_INDEX, VFIO_PCI_ERR_IRQ_INDEX, VFIO_PCI_REQ_IRQ_INDEX, + VFIO_PCI_ACPI_IRQ_INDEX, VFIO_PCI_NUM_IRQS }; -- 2.41.0.255.g8b1d071c50-goog

1 0

[PATCH 0/2] qemu: VM saving related fixes
by Peter Krempa 30 Jun '23

30 Jun '23

Peter Krempa (2): qemu: driver: Reformat helpers for saving VM state Properly mark auto-added 'terminator' virStorageSource src/qemu/qemu_domain.c | 4 +++- src/qemu/qemu_driver.c | 11 +++++++---- src/storage_file/storage_source.c | 1 + 3 files changed, 11 insertions(+), 5 deletions(-) -- 2.41.0

2 4

[PATCH 0/2] NEWS update for upcoming release
by Peter Krempa 30 Jun '23

30 Jun '23

Peter Krempa (2): NEWS: Move section about 'discard_no_unref' to features NEWS: Document changes for the release NEWS.rst | 49 ++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 44 insertions(+), 5 deletions(-) -- 2.41.0

2 4

Entering freeze for libvirt-9.5.0
by Jiri Denemark 30 Jun '23

30 Jun '23

I have just tagged v9.5.0-rc1 in the repository and pushed signed tarballs and source RPMs to https://download.libvirt.org/ Please give the release candidate some testing and in case you find a serious issue which should have a fix in the upcoming release, feel free to reply to this thread to make sure the issue is more visible. If you have not done so yet, please update NEWS.rst to document any significant change you made since the last release. Thanks, Jirka

1 1

[PATCH] apparmor: Add support for local profile customizations
by Jim Fehlig 29 Jun '23

29 Jun '23

Apparmor profiles in /etc/apparmor.d/ are config files that can and should be replaced on package upgrade, which introduces the potential to overwrite any local changes. Apparmor supports local profile customizations via /etc/apparmor.d/local/<service> [1]. This change makes the support explicit by adding libvirtd, virtqemud, and virtxend profile customization stubs to /etc/apparmor.d/local/. The stubs are conditionally included by the corresponding main profiles. [1] https://ubuntu.com/server/docs/security-apparmor See "Profile customization" section Signed-off-by: Jim Fehlig <jfehlig(a)suse.com> --- This patch was inspired by an internal bug report. The SUSE libvirt package has marked /etc/apparmor.d/<some-libvirt-service> profiles as 'config(noreplace)' for as long as I can remember. On rare occasions a profile receives a change that is required to avoid regression. And on rarer occasions a user might have made local customizations to the profile. With 'noreplace', the trap is set for the user to experience the regression. Unless other apparmor users convince me otherwise, I'm planning to make this change in the SUSE package, along with changing the main /etc/apparmor.d/ profiles to 'config' and using 'config(noreplace)' for the local customizations only. Note: I'm fine keeping this as a downstream-only patch if upstream isn't interested in the clutter. src/security/apparmor/meson.build | 12 +++++++----- src/security/apparmor/usr.sbin.libvirtd.in | 3 +++ src/security/apparmor/usr.sbin.libvirtd.local | 1 + src/security/apparmor/usr.sbin.virtqemud.in | 3 +++ src/security/apparmor/usr.sbin.virtqemud.local | 1 + src/security/apparmor/usr.sbin.virtxend.in | 3 +++ src/security/apparmor/usr.sbin.virtxend.local | 1 + 7 files changed, 19 insertions(+), 5 deletions(-) diff --git a/src/security/apparmor/meson.build b/src/security/apparmor/meson.build index 58b4024b85..02a6d098ad 100644 --- a/src/security/apparmor/meson.build +++ b/src/security/apparmor/meson.build @@ -34,8 +34,10 @@ install_data( install_dir: apparmor_dir / 'libvirt', ) -install_data( - 'usr.lib.libvirt.virt-aa-helper.local', - install_dir: apparmor_dir / 'local', - rename: 'usr.lib.libvirt.virt-aa-helper', -) +foreach name : apparmor_gen_profiles + install_data( + '@0@.local'.format(name), + install_dir: apparmor_dir / 'local', + rename: name, + ) +endforeach diff --git a/src/security/apparmor/usr.sbin.libvirtd.in b/src/security/apparmor/usr.sbin.libvirtd.in index edb8dd8e26..41bdef53ec 100644 --- a/src/security/apparmor/usr.sbin.libvirtd.in +++ b/src/security/apparmor/usr.sbin.libvirtd.in @@ -139,4 +139,7 @@ profile libvirtd @sbindir@/libvirtd flags=(attach_disconnected) { /usr/{lib,lib64,lib/qemu,libexec}/qemu-bridge-helper rmix, } + + # Site-specific additions and overrides. See local/README for details. + include if exists <local/usr.sbin.libvirtd> } diff --git a/src/security/apparmor/usr.sbin.libvirtd.local b/src/security/apparmor/usr.sbin.libvirtd.local new file mode 100644 index 0000000000..3716400022 --- /dev/null +++ b/src/security/apparmor/usr.sbin.libvirtd.local @@ -0,0 +1 @@ +# Site-specific additions and overrides for 'usr.sbin.libvirtd' diff --git a/src/security/apparmor/usr.sbin.virtqemud.in b/src/security/apparmor/usr.sbin.virtqemud.in index f269c60809..3ebdbf2a8f 100644 --- a/src/security/apparmor/usr.sbin.virtqemud.in +++ b/src/security/apparmor/usr.sbin.virtqemud.in @@ -132,4 +132,7 @@ profile virtqemud @sbindir@/virtqemud flags=(attach_disconnected) { /usr/{lib,lib64,lib/qemu,libexec}/qemu-bridge-helper rmix, } + + # Site-specific additions and overrides. See local/README for details. + include if exists <local/usr.sbin.virtqemud> } diff --git a/src/security/apparmor/usr.sbin.virtqemud.local b/src/security/apparmor/usr.sbin.virtqemud.local new file mode 100644 index 0000000000..2ac68bb069 --- /dev/null +++ b/src/security/apparmor/usr.sbin.virtqemud.local @@ -0,0 +1 @@ +# Site-specific additions and overrides for 'usr.sbin.virtqemud' diff --git a/src/security/apparmor/usr.sbin.virtxend.in b/src/security/apparmor/usr.sbin.virtxend.in index 72e0d801e5..719766a0c1 100644 --- a/src/security/apparmor/usr.sbin.virtxend.in +++ b/src/security/apparmor/usr.sbin.virtxend.in @@ -52,4 +52,7 @@ profile virtxend @sbindir@/virtxend flags=(attach_disconnected) { @libexecdir@/libvirt_iohelper ix, /etc/libvirt/hooks/** rmix, /etc/xen/scripts/** rmix, + + # Site-specific additions and overrides. See local/README for details. + include if exists <local/usr.sbin.virtxend> } diff --git a/src/security/apparmor/usr.sbin.virtxend.local b/src/security/apparmor/usr.sbin.virtxend.local new file mode 100644 index 0000000000..2ade86d4df --- /dev/null +++ b/src/security/apparmor/usr.sbin.virtxend.local @@ -0,0 +1 @@ +# Site-specific additions and overrides for 'usr.sbin.virtxend' -- 2.40.1

5 22

[libvirt PATCH 0/8] apparmor: Improve overrides, fix 2.x compatibility
by Andrea Bolognani 29 Jun '23

29 Jun '23

An alternative to Jim's attempt[1]. See [2] for the discussion leading up to these changes. [1] https://listman.redhat.com/archives/libvir-list/2023-June/240531.html [2] https://listman.redhat.com/archives/libvir-list/2023-June/240251.html Andrea Bolognani (8): meson: Detect AppArmor 3.x apparmor: Allow version-specific bits in profiles apparmor: Allow version-specific bits in abstractions too apparmor: Only support passt on 3.x apparmor: Make abstractions extensible apparmor: Improve virt-aa-helper include apparmor: Make all profiles extensible NEWS: Mention overrides for AppArmor profiles and abstractions NEWS.rst | 8 +++ meson.build | 3 + .../apparmor/{libvirt-lxc => libvirt-lxc.in} | 4 ++ .../{libvirt-qemu => libvirt-qemu.in} | 6 ++ src/security/apparmor/meson.build | 68 ++++++++++++++++--- .../usr.lib.libvirt.virt-aa-helper.in | 5 ++ src/security/apparmor/usr.sbin.libvirtd.in | 4 ++ src/security/apparmor/usr.sbin.virtqemud.in | 4 ++ src/security/apparmor/usr.sbin.virtxend.in | 4 ++ 9 files changed, 96 insertions(+), 10 deletions(-) rename src/security/apparmor/{libvirt-lxc => libvirt-lxc.in} (98%) rename src/security/apparmor/{libvirt-qemu => libvirt-qemu.in} (98%) -- 2.41.0

2 10

[PATCH V2 0/3] apparmor: Add support for local profile customizations
by Jim Fehlig 29 Jun '23

29 Jun '23

This is a stab at a V2 of https://listman.redhat.com/archives/libvir-list/2023-June/240219.html That patch was ACKed and committed, but reverted before the 9.5.0 release since it could be problematic with older apparmor 2.x versions still supported by libvirt. Andrea suggested copies of the profiles for apparmor 2.x and 3.x. This series takes that approach, with patch 1 making an identical copy of the src/security/apparmor directory. Patches 2 and 3 then adjust the profiles accordingly. My approach to copying the existing directory does introduce some duplicate files in the tree, but otherwise it's minimally disruptive and will be easy to rip out when upstream libvirt no longer needs to support apparmor 2.x. FYI, so far I've only tested with apparmor 3.x, but I did push the changes to my fork with CI enabled https://gitlab.com/jfehlig/libvirt/-/pipelines/915347878 Thanks for comments/suggestions! Jim Fehlig (3): apparmor: Create version specific apparmor profiles apparmor: Remove support for passt from apparmor 2.x apparmor: Add support for local profile customizations meson.build | 6 +- src/security/apparmor-2/TEMPLATE.lxc | 15 + src/security/apparmor-2/TEMPLATE.qemu | 9 + src/security/apparmor-2/libvirt-lxc | 118 ++++++++ src/security/apparmor-2/libvirt-qemu | 256 ++++++++++++++++++ src/security/apparmor-2/meson.build | 41 +++ .../usr.lib.libvirt.virt-aa-helper.in | 75 +++++ .../usr.lib.libvirt.virt-aa-helper.local | 1 + src/security/apparmor-2/usr.sbin.libvirtd.in | 142 ++++++++++ src/security/apparmor-2/usr.sbin.virtqemud.in | 135 +++++++++ src/security/apparmor-2/usr.sbin.virtxend.in | 55 ++++ src/security/apparmor/libvirt-lxc | 3 + src/security/apparmor/libvirt-qemu | 3 + src/security/apparmor/usr.sbin.libvirtd.in | 5 +- src/security/apparmor/usr.sbin.virtqemud.in | 3 + src/security/apparmor/usr.sbin.virtxend.in | 3 + src/security/meson.build | 3 + 17 files changed, 871 insertions(+), 2 deletions(-) create mode 100644 src/security/apparmor-2/TEMPLATE.lxc create mode 100644 src/security/apparmor-2/TEMPLATE.qemu create mode 100644 src/security/apparmor-2/libvirt-lxc create mode 100644 src/security/apparmor-2/libvirt-qemu create mode 100644 src/security/apparmor-2/meson.build create mode 100644 src/security/apparmor-2/usr.lib.libvirt.virt-aa-helper.in create mode 100644 src/security/apparmor-2/usr.lib.libvirt.virt-aa-helper.local create mode 100644 src/security/apparmor-2/usr.sbin.libvirtd.in create mode 100644 src/security/apparmor-2/usr.sbin.virtqemud.in create mode 100644 src/security/apparmor-2/usr.sbin.virtxend.in -- 2.41.0

3 6

[PATCH] nodedev: transient mdev update on nodeDeviceCreateXML
by Boris Fiuczynski 29 Jun '23

29 Jun '23

Update the optional mdev attributes on the new created nodedev object as they otherwise would not get set until the next mdevctl update cycle. Resolves: https://bugzilla.redhat.com/show_bug.cgi?id=2143158 Signed-off-by: Boris Fiuczynski <fiuczy(a)linux.ibm.com> --- src/node_device/node_device_driver.c | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/src/node_device/node_device_driver.c b/src/node_device/node_device_driver.c index a2d0600560..5134d246f3 100644 --- a/src/node_device/node_device_driver.c +++ b/src/node_device/node_device_driver.c @@ -847,6 +847,9 @@ static virNodeDevicePtr nodeDeviceCreateXMLMdev(virConnectPtr conn, virNodeDeviceDef *def) { + virNodeDeviceObj *obj; + virNodeDeviceDef *new_def; + virNodeDevicePtr device; g_autofree char *uuid = NULL; if (!def->parent) { @@ -864,8 +867,19 @@ nodeDeviceCreateXMLMdev(virConnectPtr conn, def->caps->data.mdev.uuid = g_steal_pointer(&uuid); } - return nodeDeviceFindNewMediatedDevice(conn, def->caps->data.mdev.uuid, - def->caps->data.mdev.parent_addr); + device = nodeDeviceFindNewMediatedDevice(conn, def->caps->data.mdev.uuid, + def->caps->data.mdev.parent_addr); + /* check on def for attributes and try update */ + if (def->caps->data.mdev.nattributes > 0) { + /* ignore failures as mdevctl updates will recover later */ + if (!(obj = nodeDeviceObjFindByName(device->name))) + return device; + new_def = virNodeDeviceObjGetDef(obj); + nodeDeviceDefCopyFromMdevctl(new_def, def); + virNodeDeviceObjEndAPI(&obj); + } + + return device; } -- 2.41.0

2 4

[PATCH] Revert "lxc_fuse: Tell FUSE that /proc/meminfo is nonseekable"
by Michal Privoznik 29 Jun '23

29 Jun '23

After v8.1.0-61-g030faee28d it is no longer necessary to make the /proc/meminfo file nonseekable as our code that fills the file with spoofed values can handle seeking just fine. Previously, `free(1)` was okay with failed lseek(), but this was ages ago and meanwhile the procps project moved to creating a library and moved the file parsing code under an exported function. In attempt to make the function callable multiple times, it can lseek() multiple times and failure to do so is fatal. This reverts commit 766495508650bebd5f4ac23224ecd0a2ee2ca9eb. Resolves: https://gitlab.com/libvirt/libvirt/-/issues/492 Signed-off-by: Michal Privoznik <mprivozn(a)redhat.com> --- Ideally, I'd like to merge this in the time for upcoming release, but I don't insist. It's a bug fix after all ;-) src/lxc/lxc_fuse.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/lxc/lxc_fuse.c b/src/lxc/lxc_fuse.c index a11cbbaff7..44bdd1412d 100644 --- a/src/lxc/lxc_fuse.c +++ b/src/lxc/lxc_fuse.c @@ -163,7 +163,6 @@ lxcProcOpen(const char *path, return -EACCES; fi->direct_io = 1; - fi->nonseekable = 1; return 0; } -- 2.39.3

2 1