[libvirt] [PATCH RFC] virhook: Adding inotify support to virhook.
Libvirtd only support hooks when the daemon is started. Hooks cannot be loaded when the daemon is already running. So, to load a hook you need to restart the service everytime. Now, the inotify support enables the option of create a hook and run it even if libvirtd was started. Cc: Carlos Castilho <ccasti@br.ibm.com> Signed-off-by: Julio Faracco <jcfaracco@gmail.com> --- daemon/libvirtd.c | 1 + src/libvirt_private.syms | 1 + src/util/virhook.c | 165 ++++++++++++++++++++++++++++++++++++++++++++++- src/util/virhook.h | 10 +++ 4 files changed, 175 insertions(+), 2 deletions(-) diff --git a/daemon/libvirtd.c b/daemon/libvirtd.c index 95c1b1c..56175d1 100644 --- a/daemon/libvirtd.c +++ b/daemon/libvirtd.c @@ -1622,6 +1622,7 @@ int main(int argc, char **argv) { 0, "shutdown", NULL, NULL); cleanup: + virHookCleanUp(); virNetlinkEventServiceStopAll(); virObjectUnref(remoteProgram); virObjectUnref(lxcProgram); diff --git a/src/libvirt_private.syms b/src/libvirt_private.syms index 6a77e46..c8ad816 100644 --- a/src/libvirt_private.syms +++ b/src/libvirt_private.syms @@ -1648,6 +1648,7 @@ virHashValueFree; virHookCall; virHookInitialize; virHookPresent; +virHookCleanUp; # util/virhostdev.h diff --git a/src/util/virhook.c b/src/util/virhook.c index facd74a..d5fc928 100644 --- a/src/util/virhook.c +++ b/src/util/virhook.c @@ -26,6 +26,7 @@ #include <sys/types.h> #include <sys/wait.h> #include <sys/stat.h> +#include <sys/inotify.h> #include <unistd.h> #include <stdlib.h> #include <stdio.h> @@ -45,6 +46,10 @@ VIR_LOG_INIT("util.hook"); #define LIBVIRT_HOOK_DIR SYSCONFDIR "/libvirt/hooks" +#define virHookInstall(driver) virHooksFound |= (1 << driver); + +#define virHookUninstall(driver) virHooksFound ^= (1 << driver); + VIR_ENUM_DECL(virHookDriver) VIR_ENUM_DECL(virHookDaemonOp) VIR_ENUM_DECL(virHookSubop) @@ -109,6 +114,8 @@ VIR_ENUM_IMPL(virHookLibxlOp, VIR_HOOK_LIBXL_OP_LAST, static int virHooksFound = -1; +static virHookInotifyPtr virHooksInotify = NULL; + /** * virHookCheck: * @driver: the driver name "daemon", "qemu", "lxc"... @@ -153,6 +160,121 @@ virHookCheck(int no, const char *driver) return ret; } +/** + * virHookInotifyEvent: + * @fd: inotify file descriptor. + * + * Identifies file events at libvirt's hook directory. + * Install or uninstall hooks on demand. Acording file manipulation. + */ +static void +virHookInotifyEvent(int watch ATTRIBUTE_UNUSED, + int fd, + int events ATTRIBUTE_UNUSED, + void *data ATTRIBUTE_UNUSED) +{ + char buf[1024]; + struct inotify_event *e; + int got; + int driver; + char *tmp, *name; + + VIR_DEBUG("inotify event in virHookInotify()"); + +reread: + got = read(fd, buf, sizeof(buf)); + if (got == -1) { + if (errno == EINTR) + goto reread; + return; + } + + tmp = buf; + while (got) { + if (got < sizeof(struct inotify_event)) + return; + + VIR_WARNINGS_NO_CAST_ALIGN + e = (struct inotify_event *)tmp; + VIR_WARNINGS_RESET + + tmp += sizeof(struct inotify_event); + got -= sizeof(struct inotify_event); + + if (got < e->len) + return; + + tmp += e->len; + got -= e->len; + + name = (char *)&(e->name); + + /* Removing hook file. */ + if (e->mask & (IN_DELETE | IN_MOVED_FROM)) { + if ((driver = virHookDriverTypeFromString(name)) < 0) { + VIR_DEBUG("Invalid hook name for %s", name); + return; + } + + virHookUninstall(driver); + } + + /* Creating hook file. */ + if (e->mask & (IN_CREATE | IN_CLOSE_WRITE | IN_MOVED_TO)) { + if ((driver = virHookDriverTypeFromString(name)) < 0) { + VIR_DEBUG("Invalid hook name for %s", name); + return; + } + + virHookInstall(driver); + } + } +} + +/** + * virHookInotifyInit: + * + * Initialize inotify hooks support. + * Enable hooks installation on demand. + * + * Returns 0 if inotify was successfully installed, -1 in case of failure. + */ +static int +virHookInotifyInit(void) { + + if (VIR_ALLOC(virHooksInotify) < 0) + goto error; + + if ((virHooksInotify->inotifyFD = inotify_init()) < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", _("cannot initialize inotify")); + goto error; + } + + if ((virHooksInotify->inotifyWatch = + inotify_add_watch(virHooksInotify->inotifyFD, + LIBVIRT_HOOK_DIR, + IN_CREATE | IN_MODIFY | IN_DELETE)) < 0) { + virReportSystemError(errno, _("Failed to create inotify watch on %s"), + LIBVIRT_HOOK_DIR); + goto error; + } + + if ((virHooksInotify->inotifyHandler = + virEventAddHandle(virHooksInotify->inotifyFD, + VIR_EVENT_HANDLE_READABLE, + virHookInotifyEvent, NULL, NULL)) < 0) { + VIR_DEBUG("Failed to add inotify handle in virHook."); + goto error; + } + + return 0; + +error: + virHookCleanUp(); + return -1; +} + + /* * virHookInitialize: * @@ -174,10 +296,14 @@ virHookInitialize(void) return -1; if (res == 1) { - virHooksFound |= (1 << i); + virHookInstall(i); ret++; } } + + if (virHookInotifyInit() < 0) + VIR_INFO("Disabling hooks inotify support."); + return ret; } @@ -309,7 +435,12 @@ virHookCall(int driver, if (output) virCommandSetOutputBuffer(cmd, output); - ret = virCommandRun(cmd, NULL); + ret = virHookCheck(driver, virHookDriverTypeToString(driver)); + + if (ret > 0) { + ret = virCommandRun(cmd, NULL); + } + if (ret < 0) { /* Convert INTERNAL_ERROR into known error. */ virReportError(VIR_ERR_HOOK_SCRIPT_FAILED, "%s", @@ -322,3 +453,33 @@ virHookCall(int driver, return ret; } + +/** + * virHookCall: + * + * Release all structures and data used in virhooks. + * + * Returns: 0 if the execution succeeded + */ +int +virHookCleanUp(void) +{ + if (!virHooksInotify) + return -1; + + if ((virHooksInotify->inotifyFD >= 0) && + (virHooksInotify->inotifyWatch >= 0)) + if (inotify_rm_watch(virHooksInotify->inotifyFD, + virHooksInotify->inotifyWatch) < 0) + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", _("Cannot remove inotify watcher.")); + + if (virHooksInotify->inotifyHandler >= 0) + virEventRemoveHandle(virHooksInotify->inotifyHandler); + + VIR_FORCE_CLOSE(virHooksInotify->inotifyFD); + VIR_FREE(virHooksInotify); + + virHooksFound = -1; + + return 0; +} diff --git a/src/util/virhook.h b/src/util/virhook.h index 205249c..47a32c7 100644 --- a/src/util/virhook.h +++ b/src/util/virhook.h @@ -100,6 +100,14 @@ typedef enum { VIR_HOOK_LIBXL_OP_LAST, } virHookLibxlOpType; +struct _virHookInotify { + int inotifyFD; + int inotifyWatch; + int inotifyHandler; +}; + +typedef struct _virHookInotify *virHookInotifyPtr; + int virHookInitialize(void); int virHookPresent(int driver); @@ -107,4 +115,6 @@ int virHookPresent(int driver); int virHookCall(int driver, const char *id, int op, int sub_op, const char *extra, const char *input, char **output); +int virHookCleanUp(void); + #endif /* __VIR_HOOKS_H__ */ -- 2.7.4
Hi guys, As I mentioned, we are working in a project that introduces a hook for QEMU. And when someone installs the RPM/DEB package with this hook, we need to restart the libvirtd service everytime. It is really annoying for users. In my opinion, we can include an #if and #else to compile with inotify support. But, I don't see any good reason to add. Julio Cesar Faracco 2016-09-12 21:21 GMT-03:00 Julio Faracco <jcfaracco@gmail.com>:
Libvirtd only support hooks when the daemon is started. Hooks cannot be loaded when the daemon is already running. So, to load a hook you need to restart the service everytime. Now, the inotify support enables the option of create a hook and run it even if libvirtd was started.
Cc: Carlos Castilho <ccasti@br.ibm.com> Signed-off-by: Julio Faracco <jcfaracco@gmail.com> --- daemon/libvirtd.c | 1 + src/libvirt_private.syms | 1 + src/util/virhook.c | 165 ++++++++++++++++++++++++++++++++++++++++++++++- src/util/virhook.h | 10 +++ 4 files changed, 175 insertions(+), 2 deletions(-)
diff --git a/daemon/libvirtd.c b/daemon/libvirtd.c index 95c1b1c..56175d1 100644 --- a/daemon/libvirtd.c +++ b/daemon/libvirtd.c @@ -1622,6 +1622,7 @@ int main(int argc, char **argv) { 0, "shutdown", NULL, NULL);
cleanup: + virHookCleanUp(); virNetlinkEventServiceStopAll(); virObjectUnref(remoteProgram); virObjectUnref(lxcProgram); diff --git a/src/libvirt_private.syms b/src/libvirt_private.syms index 6a77e46..c8ad816 100644 --- a/src/libvirt_private.syms +++ b/src/libvirt_private.syms @@ -1648,6 +1648,7 @@ virHashValueFree; virHookCall; virHookInitialize; virHookPresent; +virHookCleanUp;
# util/virhostdev.h diff --git a/src/util/virhook.c b/src/util/virhook.c index facd74a..d5fc928 100644 --- a/src/util/virhook.c +++ b/src/util/virhook.c @@ -26,6 +26,7 @@ #include <sys/types.h> #include <sys/wait.h> #include <sys/stat.h> +#include <sys/inotify.h> #include <unistd.h> #include <stdlib.h> #include <stdio.h> @@ -45,6 +46,10 @@ VIR_LOG_INIT("util.hook");
#define LIBVIRT_HOOK_DIR SYSCONFDIR "/libvirt/hooks"
+#define virHookInstall(driver) virHooksFound |= (1 << driver); + +#define virHookUninstall(driver) virHooksFound ^= (1 << driver); + VIR_ENUM_DECL(virHookDriver) VIR_ENUM_DECL(virHookDaemonOp) VIR_ENUM_DECL(virHookSubop) @@ -109,6 +114,8 @@ VIR_ENUM_IMPL(virHookLibxlOp, VIR_HOOK_LIBXL_OP_LAST,
static int virHooksFound = -1;
+static virHookInotifyPtr virHooksInotify = NULL; + /** * virHookCheck: * @driver: the driver name "daemon", "qemu", "lxc"... @@ -153,6 +160,121 @@ virHookCheck(int no, const char *driver) return ret; }
+/** + * virHookInotifyEvent: + * @fd: inotify file descriptor. + * + * Identifies file events at libvirt's hook directory. + * Install or uninstall hooks on demand. Acording file manipulation. + */ +static void +virHookInotifyEvent(int watch ATTRIBUTE_UNUSED, + int fd, + int events ATTRIBUTE_UNUSED, + void *data ATTRIBUTE_UNUSED) +{ + char buf[1024]; + struct inotify_event *e; + int got; + int driver; + char *tmp, *name; + + VIR_DEBUG("inotify event in virHookInotify()"); + +reread: + got = read(fd, buf, sizeof(buf)); + if (got == -1) { + if (errno == EINTR) + goto reread; + return; + } + + tmp = buf; + while (got) { + if (got < sizeof(struct inotify_event)) + return; + + VIR_WARNINGS_NO_CAST_ALIGN + e = (struct inotify_event *)tmp; + VIR_WARNINGS_RESET + + tmp += sizeof(struct inotify_event); + got -= sizeof(struct inotify_event); + + if (got < e->len) + return; + + tmp += e->len; + got -= e->len; + + name = (char *)&(e->name); + + /* Removing hook file. */ + if (e->mask & (IN_DELETE | IN_MOVED_FROM)) { + if ((driver = virHookDriverTypeFromString(name)) < 0) { + VIR_DEBUG("Invalid hook name for %s", name); + return; + } + + virHookUninstall(driver); + } + + /* Creating hook file. */ + if (e->mask & (IN_CREATE | IN_CLOSE_WRITE | IN_MOVED_TO)) { + if ((driver = virHookDriverTypeFromString(name)) < 0) { + VIR_DEBUG("Invalid hook name for %s", name); + return; + } + + virHookInstall(driver); + } + } +} + +/** + * virHookInotifyInit: + * + * Initialize inotify hooks support. + * Enable hooks installation on demand. + * + * Returns 0 if inotify was successfully installed, -1 in case of failure. + */ +static int +virHookInotifyInit(void) { + + if (VIR_ALLOC(virHooksInotify) < 0) + goto error; + + if ((virHooksInotify->inotifyFD = inotify_init()) < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", _("cannot initialize inotify")); + goto error; + } + + if ((virHooksInotify->inotifyWatch = + inotify_add_watch(virHooksInotify->inotifyFD, + LIBVIRT_HOOK_DIR, + IN_CREATE | IN_MODIFY | IN_DELETE)) < 0) { + virReportSystemError(errno, _("Failed to create inotify watch on %s"), + LIBVIRT_HOOK_DIR); + goto error; + } + + if ((virHooksInotify->inotifyHandler = + virEventAddHandle(virHooksInotify->inotifyFD, + VIR_EVENT_HANDLE_READABLE, + virHookInotifyEvent, NULL, NULL)) < 0) { + VIR_DEBUG("Failed to add inotify handle in virHook."); + goto error; + } + + return 0; + +error: + virHookCleanUp(); + return -1; +} + + /* * virHookInitialize: * @@ -174,10 +296,14 @@ virHookInitialize(void) return -1;
if (res == 1) { - virHooksFound |= (1 << i); + virHookInstall(i); ret++; } } + + if (virHookInotifyInit() < 0) + VIR_INFO("Disabling hooks inotify support."); + return ret; }
@@ -309,7 +435,12 @@ virHookCall(int driver, if (output) virCommandSetOutputBuffer(cmd, output);
- ret = virCommandRun(cmd, NULL); + ret = virHookCheck(driver, virHookDriverTypeToString(driver)); + + if (ret > 0) { + ret = virCommandRun(cmd, NULL); + } + if (ret < 0) { /* Convert INTERNAL_ERROR into known error. */ virReportError(VIR_ERR_HOOK_SCRIPT_FAILED, "%s", @@ -322,3 +453,33 @@ virHookCall(int driver,
return ret; } + +/** + * virHookCall: + * + * Release all structures and data used in virhooks. + * + * Returns: 0 if the execution succeeded + */ +int +virHookCleanUp(void) +{ + if (!virHooksInotify) + return -1; + + if ((virHooksInotify->inotifyFD >= 0) && + (virHooksInotify->inotifyWatch >= 0)) + if (inotify_rm_watch(virHooksInotify->inotifyFD, + virHooksInotify->inotifyWatch) < 0) + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", _("Cannot remove inotify watcher.")); + + if (virHooksInotify->inotifyHandler >= 0) + virEventRemoveHandle(virHooksInotify->inotifyHandler); + + VIR_FORCE_CLOSE(virHooksInotify->inotifyFD); + VIR_FREE(virHooksInotify); + + virHooksFound = -1; + + return 0; +} diff --git a/src/util/virhook.h b/src/util/virhook.h index 205249c..47a32c7 100644 --- a/src/util/virhook.h +++ b/src/util/virhook.h @@ -100,6 +100,14 @@ typedef enum { VIR_HOOK_LIBXL_OP_LAST, } virHookLibxlOpType;
+struct _virHookInotify { + int inotifyFD; + int inotifyWatch; + int inotifyHandler; +}; + +typedef struct _virHookInotify *virHookInotifyPtr; + int virHookInitialize(void);
int virHookPresent(int driver); @@ -107,4 +115,6 @@ int virHookPresent(int driver); int virHookCall(int driver, const char *id, int op, int sub_op, const char *extra, const char *input, char **output);
+int virHookCleanUp(void); + #endif /* __VIR_HOOKS_H__ */ -- 2.7.4
On 13.09.2016 02:25, Julio Faracco wrote:
Hi guys,
As I mentioned, we are working in a project that introduces a hook for QEMU. And when someone installs the RPM/DEB package with this hook, we need to restart the libvirtd service everytime. It is really annoying for users.
Well, RPM packages have %post macro where package maintainer can put some scripts. Well, commands to run. For instance: /bin/systemctl try-restart libvirtd.service >/dev/null 2>&1 || : can be there. I bet DEB packages have similar option.
In my opinion, we can include an #if and #else to compile with inotify support. But, I don't see any good reason to add.
Well, one of the reasons might be that other systems than Linux where libvirt is built would fail after this patch. inotify is Linux specific and for instance *BSD kernels don't have it. So we need to check for it in configure phase. Michal
Hi Michal! Sorry, guys.... I was on vacation during this time. Thanks to pointing all suggestions. Basically, we are restarting libvirt to install properly the hook that our package installs. But it is very annoying to users. Generally, they are using Virtual Machines when they receive a package update and after the installation, They lost the current connection to libvirt because the service was restarted. So, think in 10K users using some Virtual Machines and losing connection (virt-viewer, virt-manager, etc) while they are updating the system. Unfortunately, our hooks setup some network configurations. So, the hooks needs to be loaded. We can't wait for the users shutdown/reboot his computer. :-( But really, I don't want to introduce big changes (including documentation) to fix only a load problem. -- Julio Cesar Faracco 2016-09-19 5:09 GMT-03:00 Michal Privoznik <mprivozn@redhat.com>:
On 13.09.2016 02:25, Julio Faracco wrote:
Hi guys,
As I mentioned, we are working in a project that introduces a hook for QEMU. And when someone installs the RPM/DEB package with this hook, we need to restart the libvirtd service everytime. It is really annoying for users.
Well, RPM packages have %post macro where package maintainer can put some scripts. Well, commands to run. For instance:
/bin/systemctl try-restart libvirtd.service >/dev/null 2>&1 || :
can be there. I bet DEB packages have similar option.
In my opinion, we can include an #if and #else to compile with inotify support. But, I don't see any good reason to add.
Well, one of the reasons might be that other systems than Linux where libvirt is built would fail after this patch. inotify is Linux specific and for instance *BSD kernels don't have it. So we need to check for it in configure phase.
Michal
On 13.09.2016 02:21, Julio Faracco wrote:
Libvirtd only support hooks when the daemon is started. Hooks cannot be loaded when the daemon is already running. So, to load a hook you need to restart the service everytime. Now, the inotify support enables the option of create a hook and run it even if libvirtd was started.
Cc: Carlos Castilho <ccasti@br.ibm.com> Signed-off-by: Julio Faracco <jcfaracco@gmail.com>
Interesting approach. The only thing I am worried about is that it might trick users into thinking we will implement something similar for editing XML files for domains. That is, if they edit /etc/libvirt/qemu/myFavouriteDomain.xml, libvirt will monitor the changes and re-read the file if needed.
--- daemon/libvirtd.c | 1 + src/libvirt_private.syms | 1 + src/util/virhook.c | 165 ++++++++++++++++++++++++++++++++++++++++++++++- src/util/virhook.h | 10 +++ 4 files changed, 175 insertions(+), 2 deletions(-)
The documentation is missing. There is docs/hooks.html.in file which covers this topic. Moreover, 'make all syntax-check check' is your friend ;-) I'm not gonna report problems found by syntax-check - now that you know we have such feature, you can fix those issues yourself.
diff --git a/daemon/libvirtd.c b/daemon/libvirtd.c index 95c1b1c..56175d1 100644 --- a/daemon/libvirtd.c +++ b/daemon/libvirtd.c @@ -1622,6 +1622,7 @@ int main(int argc, char **argv) { 0, "shutdown", NULL, NULL);
cleanup: + virHookCleanUp(); virNetlinkEventServiceStopAll(); virObjectUnref(remoteProgram); virObjectUnref(lxcProgram); diff --git a/src/libvirt_private.syms b/src/libvirt_private.syms index 6a77e46..c8ad816 100644 --- a/src/libvirt_private.syms +++ b/src/libvirt_private.syms @@ -1648,6 +1648,7 @@ virHashValueFree; virHookCall; virHookInitialize; virHookPresent; +virHookCleanUp;
# util/virhostdev.h diff --git a/src/util/virhook.c b/src/util/virhook.c index facd74a..d5fc928 100644 --- a/src/util/virhook.c +++ b/src/util/virhook.c @@ -26,6 +26,7 @@ #include <sys/types.h> #include <sys/wait.h> #include <sys/stat.h> +#include <sys/inotify.h> #include <unistd.h> #include <stdlib.h> #include <stdio.h> @@ -45,6 +46,10 @@ VIR_LOG_INIT("util.hook");
#define LIBVIRT_HOOK_DIR SYSCONFDIR "/libvirt/hooks"
+#define virHookInstall(driver) virHooksFound |= (1 << driver); + +#define virHookUninstall(driver) virHooksFound ^= (1 << driver); + VIR_ENUM_DECL(virHookDriver) VIR_ENUM_DECL(virHookDaemonOp) VIR_ENUM_DECL(virHookSubop) @@ -109,6 +114,8 @@ VIR_ENUM_IMPL(virHookLibxlOp, VIR_HOOK_LIBXL_OP_LAST,
static int virHooksFound = -1;
+static virHookInotifyPtr virHooksInotify = NULL; + /** * virHookCheck: * @driver: the driver name "daemon", "qemu", "lxc"... @@ -153,6 +160,121 @@ virHookCheck(int no, const char *driver) return ret; }
+/** + * virHookInotifyEvent: + * @fd: inotify file descriptor. + * + * Identifies file events at libvirt's hook directory. + * Install or uninstall hooks on demand. Acording file manipulation. + */ +static void +virHookInotifyEvent(int watch ATTRIBUTE_UNUSED, + int fd, + int events ATTRIBUTE_UNUSED, + void *data ATTRIBUTE_UNUSED) +{ + char buf[1024];
man page suggests using slightly larger buffer.
+ struct inotify_event *e; + int got; + int driver; + char *tmp, *name; + + VIR_DEBUG("inotify event in virHookInotify()"); + +reread: + got = read(fd, buf, sizeof(buf));
@fd is blocking. We don't want to read from a blocking FD in our event loop (as it might block the whole event loop).
+ if (got == -1) { + if (errno == EINTR) + goto reread; + return; + }
We have a wrapper for that - saferead(). Also, it would be nice if an error is reported on failed read. Just imagine you're given the daemon logs with bug description 'libvirt hooks do not work with inotify'.
+ + tmp = buf; + while (got) { + if (got < sizeof(struct inotify_event)) + return; + + VIR_WARNINGS_NO_CAST_ALIGN + e = (struct inotify_event *)tmp; + VIR_WARNINGS_RESET + + tmp += sizeof(struct inotify_event); + got -= sizeof(struct inotify_event); + + if (got < e->len) + return;
Wait, what? This discards partially read event. Moreover, in the next iteration we start reading from wrong offset in the inotify_event struct and thus poison the well. I think we need to come up with better mechanism for reading events. Also - should we allocate receiving buffer for events dynamically on demand?
+ + tmp += e->len; + got -= e->len; + + name = (char *)&(e->name);
This does not feel right. e->name is type of ' char[]'.
+ + /* Removing hook file. */ + if (e->mask & (IN_DELETE | IN_MOVED_FROM)) { + if ((driver = virHookDriverTypeFromString(name)) < 0) { + VIR_DEBUG("Invalid hook name for %s", name); + return; + } + + virHookUninstall(driver); + } + + /* Creating hook file. */ + if (e->mask & (IN_CREATE | IN_CLOSE_WRITE | IN_MOVED_TO)) { + if ((driver = virHookDriverTypeFromString(name)) < 0) { + VIR_DEBUG("Invalid hook name for %s", name); + return; + } + + virHookInstall(driver); + } + } +}
Well, this just sets some internal variable. It doesn't reload the list of hooks or something. Moreover, these hooks are meant to be run at the daemon startup. Therefore I think you should stick to what I suggest in the other e-mail. Michal
participants (2)
-
Julio Faracco -
Michal Privoznik