[PATCH] qemu: migration: Call hook script on source host at migration start
Currently, the qemu hook script with the "migrate" operation is only called on the destination host at the beginning of incoming migration. This makes it impossible for the source host to prepare for migration, for example to change storage locks from exclusive to shared mode when using shared LVM storage with lvmlockd. Add a hook call on the source host at the beginning of outgoing migration. The hook is called with "source" as the extra argument to distinguish it from the destination hook which uses "-". The same change is also applied to the libxl driver for consistency. Resolves: https://gitlab.com/libvirt/libvirt/-/issues/37 Signed-off-by: Guy Godfroy <guy.godfroy@gugod.fr> --- docs/hooks.rst | 36 ++++++++++++++++++++++++++++++++---- src/libxl/libxl_migration.c | 17 +++++++++++++++++ src/qemu/qemu_migration.c | 17 +++++++++++++++++ 3 files changed, 66 insertions(+), 4 deletions(-) diff --git a/docs/hooks.rst b/docs/hooks.rst index e1745b8cc7..4c7cac56ba 100644 --- a/docs/hooks.rst +++ b/docs/hooks.rst @@ -244,7 +244,7 @@ operation. There is no specific operation to indicate a "restart" is occurring. - :since:`Since 0.9.11`, the qemu hook script is also called at the beginning - of incoming migration. It is called as: + of incoming migration on the **destination** host. It is called as: :: @@ -257,6 +257,18 @@ operation. There is no specific operation to indicate a "restart" is occurring. is not valid, incoming migration will be canceled. This hook may be used, e.g., to change location of disk images for incoming domains. +- :since:`Since 12.1.0`, the qemu hook script is also called at the beginning + of outgoing migration on the **source** host. It is called as: + + :: + + /etc/libvirt/hooks/qemu guest_name migrate begin source + + with domain XML sent to standard input of the script. Unlike the destination + hook, this script does not act as a filter and any output is ignored. If the + script returns failure, the migration will be canceled. This hook may be used, + e.g., to change storage lock mode from exclusive to shared before migration. + - :since:`Since 1.2.9`, the qemu hook script is also called when restoring a saved image either via the API or automatically when restoring a managed save machine. It is called as: @@ -417,7 +429,7 @@ operation. There is no specific operation to indicate a "restart" is occurring. /etc/libvirt/hooks/libxl guest_name release end - - :since:`Since 2.1.0`, the libxl hook script is also called at the beginning - of incoming migration. It is called as: + of incoming migration on the **destination** host. It is called as: :: @@ -430,6 +442,18 @@ operation. There is no specific operation to indicate a "restart" is occurring. is not valid, incoming migration will be canceled. This hook may be used, e.g., to change location of disk images for incoming domains. +- :since:`Since 12.1.0`, the libxl hook script is also called at the beginning + of outgoing migration on the **source** host. It is called as: + + :: + + /etc/libvirt/hooks/libxl guest_name migrate begin source + + with domain XML sent to standard input of the script. Unlike the destination + hook, this script does not act as a filter and any output is ignored. If the + script returns failure, the migration will be canceled. This hook may be used, + e.g., to change storage lock mode from exclusive to shared before migration. + - :since:`Since 6.5.0`, you can also place several hook scripts in the directory ``/etc/libvirt/hooks/libxl.d/``. They are executed in alphabetical order after main script. In this case each script also acts as filter and can @@ -566,10 +590,14 @@ Migration of a QEMU guest involves running hook scripts on both the source and destination hosts: #. At the beginning of the migration, the *qemu* hook script on the - **destination** host is executed with the "migrate" operation. + **source** host is executed with the "migrate" operation and "source" + as the extra argument (:since:`since 12.1.0`). This allows the source + host to prepare for migration, e.g., changing storage locks. +#. Then, the *qemu* hook script on the **destination** host is executed + with the "migrate" operation. #. Before QEMU process is spawned, the two operations ("prepare" and "start") called for domain start are executed on **destination** host. -#. If both of these hook script executions exit successfully (exit status 0), +#. If all of these hook script executions exit successfully (exit status 0), the migration continues. Any other exit code indicates failure, and the migration is aborted. #. The QEMU guest is then migrated to the destination host. diff --git a/src/libxl/libxl_migration.c b/src/libxl/libxl_migration.c index f5dee7627b..76f3c7ceb7 100644 --- a/src/libxl/libxl_migration.c +++ b/src/libxl/libxl_migration.c @@ -411,6 +411,23 @@ libxlDomainMigrationSrcBegin(virConnectPtr conn, if (!libxlDomainMigrationIsAllowed(def)) goto endjob; + /* Call hook to allow source host to prepare for migration */ + if (virHookPresent(VIR_HOOK_DRIVER_LIBXL)) { + g_autofree char *hookxml = NULL; + int hookret; + + if (!(hookxml = virDomainDefFormat(def, driver->xmlopt, + VIR_DOMAIN_DEF_FORMAT_SECURE))) + goto endjob; + + hookret = virHookCall(VIR_HOOK_DRIVER_LIBXL, def->name, + VIR_HOOK_LIBXL_OP_MIGRATE, VIR_HOOK_SUBOP_BEGIN, + "source", hookxml, NULL); + + if (hookret < 0) + goto endjob; + } + xml = virDomainDefFormat(def, driver->xmlopt, VIR_DOMAIN_DEF_FORMAT_SECURE); /* Valid xml means success! EndJob in the confirm phase */ if (xml) diff --git a/src/qemu/qemu_migration.c b/src/qemu/qemu_migration.c index 6dd022163b..60e370319c 100644 --- a/src/qemu/qemu_migration.c +++ b/src/qemu/qemu_migration.c @@ -2791,6 +2791,23 @@ qemuMigrationSrcBeginPhase(virQEMUDriver *driver, if (!qemuMigrationSrcIsAllowed(vm, true, vm->job->asyncJob, flags)) return NULL; + /* Call hook to allow source host to prepare for migration */ + if (virHookPresent(VIR_HOOK_DRIVER_QEMU)) { + g_autofree char *xml = NULL; + int hookret; + + if (!(xml = qemuDomainDefFormatLive(driver, priv->qemuCaps, vm->def, + priv->origCPU, false, true))) + return NULL; + + hookret = virHookCall(VIR_HOOK_DRIVER_QEMU, vm->def->name, + VIR_HOOK_QEMU_OP_MIGRATE, VIR_HOOK_SUBOP_BEGIN, + "source", xml, NULL); + + if (hookret < 0) + return NULL; + } + if (!(flags & (VIR_MIGRATE_UNSAFE | VIR_MIGRATE_OFFLINE)) && !qemuMigrationSrcIsSafe(vm, migrate_disks, flags)) return NULL; -- 2.34.1
On Mon, Jan 26, 2026 at 11:05:13 +0100, Guy Godfroy via Devel wrote:
Currently, the qemu hook script with the "migrate" operation is only called on the destination host at the beginning of incoming migration. This makes it impossible for the source host to prepare for migration, for example to change storage locks from exclusive to shared mode when using shared LVM storage with lvmlockd.
Add a hook call on the source host at the beginning of outgoing migration. The hook is called with "source" as the extra argument to distinguish it from the destination hook which uses "-".
Well, this has IMO still the potential to break things. I suggest that the second argument to the hook program is something else than 'migrate'. Perhaps 'migrate-outgoing' to prevent breaking programs which ignore the third argument. I'm also worried a bit about the usability of the hook for cases when the migration fails ...
The same change is also applied to the libxl driver for consistency.
Resolves: https://gitlab.com/libvirt/libvirt/-/issues/37 Signed-off-by: Guy Godfroy <guy.godfroy@gugod.fr> --- docs/hooks.rst | 36 ++++++++++++++++++++++++++++++++---- src/libxl/libxl_migration.c | 17 +++++++++++++++++ src/qemu/qemu_migration.c | 17 +++++++++++++++++ 3 files changed, 66 insertions(+), 4 deletions(-)
diff --git a/docs/hooks.rst b/docs/hooks.rst index e1745b8cc7..4c7cac56ba 100644 --- a/docs/hooks.rst +++ b/docs/hooks.rst @@ -244,7 +244,7 @@ operation. There is no specific operation to indicate a "restart" is occurring.
- :since:`Since 0.9.11`, the qemu hook script is also called at the beginning - of incoming migration. It is called as: + of incoming migration on the **destination** host. It is called as:
(These doc changes make sense on its own. Ideally separate them to another patch)
::
@@ -257,6 +257,18 @@ operation. There is no specific operation to indicate a "restart" is occurring. is not valid, incoming migration will be canceled. This hook may be used, e.g., to change location of disk images for incoming domains.
+- :since:`Since 12.1.0`, the qemu hook script is also called at the beginning + of outgoing migration on the **source** host. It is called as: + + :: + + /etc/libvirt/hooks/qemu guest_name migrate begin source + + with domain XML sent to standard input of the script. Unlike the destination + hook, this script does not act as a filter and any output is ignored. If the + script returns failure, the migration will be canceled. This hook may be used, + e.g., to change storage lock mode from exclusive to shared before migration. + - :since:`Since 1.2.9`, the qemu hook script is also called when restoring a saved image either via the API or automatically when restoring a managed save machine. It is called as: @@ -417,7 +429,7 @@ operation. There is no specific operation to indicate a "restart" is occurring. /etc/libvirt/hooks/libxl guest_name release end -
- :since:`Since 2.1.0`, the libxl hook script is also called at the beginning - of incoming migration. It is called as: + of incoming migration on the **destination** host. It is called as:
::
@@ -430,6 +442,18 @@ operation. There is no specific operation to indicate a "restart" is occurring. is not valid, incoming migration will be canceled. This hook may be used, e.g., to change location of disk images for incoming domains.
+- :since:`Since 12.1.0`, the libxl hook script is also called at the beginning + of outgoing migration on the **source** host. It is called as: + + :: + + /etc/libvirt/hooks/libxl guest_name migrate begin source + + with domain XML sent to standard input of the script. Unlike the destination + hook, this script does not act as a filter and any output is ignored. If the + script returns failure, the migration will be canceled. This hook may be used, + e.g., to change storage lock mode from exclusive to shared before migration. + - :since:`Since 6.5.0`, you can also place several hook scripts in the directory ``/etc/libvirt/hooks/libxl.d/``. They are executed in alphabetical order after main script. In this case each script also acts as filter and can @@ -566,10 +590,14 @@ Migration of a QEMU guest involves running hook scripts on both the source and destination hosts:
#. At the beginning of the migration, the *qemu* hook script on the - **destination** host is executed with the "migrate" operation. + **source** host is executed with the "migrate" operation and "source" + as the extra argument (:since:`since 12.1.0`). This allows the source + host to prepare for migration, e.g., changing storage locks. +#. Then, the *qemu* hook script on the **destination** host is executed + with the "migrate" operation. #. Before QEMU process is spawned, the two operations ("prepare" and "start") called for domain start are executed on **destination** host. -#. If both of these hook script executions exit successfully (exit status 0), +#. If all of these hook script executions exit successfully (exit status 0), the migration continues. Any other exit code indicates failure, and the migration is aborted. #. The QEMU guest is then migrated to the destination host. diff --git a/src/libxl/libxl_migration.c b/src/libxl/libxl_migration.c index f5dee7627b..76f3c7ceb7 100644 --- a/src/libxl/libxl_migration.c +++ b/src/libxl/libxl_migration.c @@ -411,6 +411,23 @@ libxlDomainMigrationSrcBegin(virConnectPtr conn, if (!libxlDomainMigrationIsAllowed(def)) goto endjob;
+ /* Call hook to allow source host to prepare for migration */ + if (virHookPresent(VIR_HOOK_DRIVER_LIBXL)) { + g_autofree char *hookxml = NULL; + int hookret; + + if (!(hookxml = virDomainDefFormat(def, driver->xmlopt, + VIR_DOMAIN_DEF_FORMAT_SECURE))) + goto endjob; + + hookret = virHookCall(VIR_HOOK_DRIVER_LIBXL, def->name, + VIR_HOOK_LIBXL_OP_MIGRATE, VIR_HOOK_SUBOP_BEGIN, + "source", hookxml, NULL); + + if (hookret < 0) + goto endjob; + } + xml = virDomainDefFormat(def, driver->xmlopt, VIR_DOMAIN_DEF_FORMAT_SECURE); /* Valid xml means success! EndJob in the confirm phase */ if (xml) diff --git a/src/qemu/qemu_migration.c b/src/qemu/qemu_migration.c index 6dd022163b..60e370319c 100644 --- a/src/qemu/qemu_migration.c +++ b/src/qemu/qemu_migration.c @@ -2791,6 +2791,23 @@ qemuMigrationSrcBeginPhase(virQEMUDriver *driver, if (!qemuMigrationSrcIsAllowed(vm, true, vm->job->asyncJob, flags)) return NULL;
+ /* Call hook to allow source host to prepare for migration */ + if (virHookPresent(VIR_HOOK_DRIVER_QEMU)) { + g_autofree char *xml = NULL; + int hookret; + + if (!(xml = qemuDomainDefFormatLive(driver, priv->qemuCaps, vm->def, + priv->origCPU, false, true))) + return NULL; + + hookret = virHookCall(VIR_HOOK_DRIVER_QEMU, vm->def->name, + VIR_HOOK_QEMU_OP_MIGRATE, VIR_HOOK_SUBOP_BEGIN, + "source", xml, NULL); + + if (hookret < 0) + return NULL; + }
... specifically this is *very* early in the migration. So early that ...
+ if (!(flags & (VIR_MIGRATE_UNSAFE | VIR_MIGRATE_OFFLINE)) && !qemuMigrationSrcIsSafe(vm, migrate_disks, flags))
... we didn't do even the pre-checks if migration will happen. That is not the problem per-se, but there is no hook action if the migration fails, in which case the hook program will very likely want to undo what it changed. At the very least the source migration hook, or however it's called really needs another parameter which will for this invocation be e.g. 'begin'. We can later add a 'fail' action if needed. I've seen also ideas of having a hook call during switchover phase so that one could then be added too if we get a compelling reason to have it. (all hook calls add migration latency so a switchover hook will need to be implemented in a different way). Also don't forget to bump version numbers to at least 12.3.0 when you'll re-send.
Guy Godfroy (4): docs: hooks: Clarify that the migrate operation is on the destination host util: hook: Add migrate-outgoing operation for qemu and libxl drivers qemu, libxl: migration: Call hook script on source host at migration start qemu, libxl: migration: Call hook script on source host when migration fails docs/hooks.rst | 63 ++++++++++++++++++++++++++++++++++--- src/libxl/libxl_migration.c | 39 +++++++++++++++++++++++ src/qemu/qemu_migration.c | 29 +++++++++++++++++ src/util/virhook.c | 2 ++ src/util/virhook.h | 2 ++ 5 files changed, 130 insertions(+), 5 deletions(-) -- 2.53.0
The documentation for the "migrate" hook operation described it as being called at the beginning of incoming migration, but didn't explicitly mention that this happens on the destination host. Add "on the **destination** host" to make this clear. Signed-off-by: Guy Godfroy <guy.godfroy@gugod.fr> --- docs/hooks.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/hooks.rst b/docs/hooks.rst index e1745b8cc7..0310e0d82c 100644 --- a/docs/hooks.rst +++ b/docs/hooks.rst @@ -244,7 +244,7 @@ operation. There is no specific operation to indicate a "restart" is occurring. - :since:`Since 0.9.11`, the qemu hook script is also called at the beginning - of incoming migration. It is called as: + of incoming migration on the **destination** host. It is called as: :: @@ -417,7 +417,7 @@ operation. There is no specific operation to indicate a "restart" is occurring. /etc/libvirt/hooks/libxl guest_name release end - - :since:`Since 2.1.0`, the libxl hook script is also called at the beginning - of incoming migration. It is called as: + of incoming migration on the **destination** host. It is called as: :: -- 2.53.0
Add VIR_HOOK_QEMU_OP_MIGRATE_OUTGOING and VIR_HOOK_LIBXL_OP_MIGRATE_OUTGOING enum values with corresponding "migrate-outgoing" string mappings. This new operation will be used for hooks called on the source host during outgoing migration, distinct from the existing "migrate" operation which is used for destination host hooks. Signed-off-by: Guy Godfroy <guy.godfroy@gugod.fr> --- src/util/virhook.c | 2 ++ src/util/virhook.h | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/util/virhook.c b/src/util/virhook.c index d012bb1825..7f7e253a67 100644 --- a/src/util/virhook.c +++ b/src/util/virhook.c @@ -84,6 +84,7 @@ VIR_ENUM_IMPL(virHookQemuOp, "reconnect", "attach", "restore", + "migrate-outgoing", ); VIR_ENUM_IMPL(virHookLxcOp, @@ -115,6 +116,7 @@ VIR_ENUM_IMPL(virHookLibxlOp, "migrate", "started", "reconnect", + "migrate-outgoing", ); VIR_ENUM_IMPL(virHookBhyveOp, diff --git a/src/util/virhook.h b/src/util/virhook.h index d8237c837e..a7536fb01e 100644 --- a/src/util/virhook.h +++ b/src/util/virhook.h @@ -60,6 +60,7 @@ typedef enum { VIR_HOOK_QEMU_OP_RECONNECT, /* domain is being reconnected by libvirt */ VIR_HOOK_QEMU_OP_ATTACH, /* domain is being attached to be libvirt */ VIR_HOOK_QEMU_OP_RESTORE, /* domain is being restored */ + VIR_HOOK_QEMU_OP_MIGRATE_OUTGOING, /* domain is being migrated out */ VIR_HOOK_QEMU_OP_LAST, } virHookQemuOpType; @@ -94,6 +95,7 @@ typedef enum { VIR_HOOK_LIBXL_OP_MIGRATE, /* domain is being migrated */ VIR_HOOK_LIBXL_OP_STARTED, /* domain has started */ VIR_HOOK_LIBXL_OP_RECONNECT, /* domain is being reconnected by libvirt */ + VIR_HOOK_LIBXL_OP_MIGRATE_OUTGOING, /* domain is being migrated out */ VIR_HOOK_LIBXL_OP_LAST, } virHookLibxlOpType; -- 2.53.0
Currently, the hook script with the "migrate" operation is only called on the destination host at the beginning of incoming migration. This makes it impossible for the source host to prepare for migration, for example to change storage locks from exclusive to shared mode when using shared LVM storage with lvmlockd. Add a hook call on the source host at the beginning of outgoing migration using the new "migrate-outgoing" operation. The hook is called with "source" as the extra argument and "begin" as the sub-operation. The script is not used as a filter; its output is ignored. If the script returns failure, the migration is canceled. The same change is also applied to the libxl driver for consistency. Signed-off-by: Guy Godfroy <guy.godfroy@gugod.fr> --- docs/hooks.rst | 32 ++++++++++++++++++++++++++++++-- src/libxl/libxl_migration.c | 17 +++++++++++++++++ src/qemu/qemu_migration.c | 17 +++++++++++++++++ 3 files changed, 64 insertions(+), 2 deletions(-) diff --git a/docs/hooks.rst b/docs/hooks.rst index 0310e0d82c..cff1711536 100644 --- a/docs/hooks.rst +++ b/docs/hooks.rst @@ -257,6 +257,18 @@ operation. There is no specific operation to indicate a "restart" is occurring. is not valid, incoming migration will be canceled. This hook may be used, e.g., to change location of disk images for incoming domains. +- :since:`Since 12.3.0`, the qemu hook script is also called at the beginning + of outgoing migration on the **source** host. It is called as: + + :: + + /etc/libvirt/hooks/qemu guest_name migrate-outgoing begin source + + with domain XML sent to standard input of the script. Unlike the destination + hook, this script does not act as a filter and any output is ignored. If the + script returns failure, the migration will be canceled. This hook may be used, + e.g., to change storage lock mode from exclusive to shared before migration. + - :since:`Since 1.2.9`, the qemu hook script is also called when restoring a saved image either via the API or automatically when restoring a managed save machine. It is called as: @@ -430,6 +442,18 @@ operation. There is no specific operation to indicate a "restart" is occurring. is not valid, incoming migration will be canceled. This hook may be used, e.g., to change location of disk images for incoming domains. +- :since:`Since 12.3.0`, the libxl hook script is also called at the beginning + of outgoing migration on the **source** host. It is called as: + + :: + + /etc/libvirt/hooks/libxl guest_name migrate-outgoing begin source + + with domain XML sent to standard input of the script. Unlike the destination + hook, this script does not act as a filter and any output is ignored. If the + script returns failure, the migration will be canceled. This hook may be used, + e.g., to change storage lock mode from exclusive to shared before migration. + - :since:`Since 6.5.0`, you can also place several hook scripts in the directory ``/etc/libvirt/hooks/libxl.d/``. They are executed in alphabetical order after main script. In this case each script also acts as filter and can @@ -566,10 +590,14 @@ Migration of a QEMU guest involves running hook scripts on both the source and destination hosts: #. At the beginning of the migration, the *qemu* hook script on the - **destination** host is executed with the "migrate" operation. + **source** host is executed with the "migrate-outgoing" operation and + "source" as the extra argument (:since:`since 12.3.0`). This allows the + source host to prepare for migration, e.g., changing storage locks. +#. Then, the *qemu* hook script on the **destination** host is executed + with the "migrate" operation. #. Before QEMU process is spawned, the two operations ("prepare" and "start") called for domain start are executed on **destination** host. -#. If both of these hook script executions exit successfully (exit status 0), +#. If all of these hook script executions exit successfully (exit status 0), the migration continues. Any other exit code indicates failure, and the migration is aborted. #. The QEMU guest is then migrated to the destination host. diff --git a/src/libxl/libxl_migration.c b/src/libxl/libxl_migration.c index f5dee7627b..be51cfd316 100644 --- a/src/libxl/libxl_migration.c +++ b/src/libxl/libxl_migration.c @@ -411,6 +411,23 @@ libxlDomainMigrationSrcBegin(virConnectPtr conn, if (!libxlDomainMigrationIsAllowed(def)) goto endjob; + /* Call hook to allow source host to prepare for migration */ + if (virHookPresent(VIR_HOOK_DRIVER_LIBXL)) { + g_autofree char *hookxml = NULL; + int hookret; + + if (!(hookxml = virDomainDefFormat(def, driver->xmlopt, + VIR_DOMAIN_DEF_FORMAT_SECURE))) + goto endjob; + + hookret = virHookCall(VIR_HOOK_DRIVER_LIBXL, def->name, + VIR_HOOK_LIBXL_OP_MIGRATE_OUTGOING, VIR_HOOK_SUBOP_BEGIN, + "source", hookxml, NULL); + + if (hookret < 0) + goto endjob; + } + xml = virDomainDefFormat(def, driver->xmlopt, VIR_DOMAIN_DEF_FORMAT_SECURE); /* Valid xml means success! EndJob in the confirm phase */ if (xml) diff --git a/src/qemu/qemu_migration.c b/src/qemu/qemu_migration.c index fec808ccfb..e9ce2d8b8b 100644 --- a/src/qemu/qemu_migration.c +++ b/src/qemu/qemu_migration.c @@ -2870,6 +2870,23 @@ qemuMigrationSrcBeginPhase(virQEMUDriver *driver, vm->newDef && !qemuDomainVcpuHotplugIsInOrder(vm->newDef))) cookieFlags |= QEMU_MIGRATION_COOKIE_CPU_HOTPLUG; + /* Call hook to allow source host to prepare for migration */ + if (virHookPresent(VIR_HOOK_DRIVER_QEMU)) { + g_autofree char *xml = NULL; + int hookret; + + if (!(xml = qemuDomainDefFormatLive(driver, priv->qemuCaps, vm->def, + priv->origCPU, false, true))) + return NULL; + + hookret = virHookCall(VIR_HOOK_DRIVER_QEMU, vm->def->name, + VIR_HOOK_QEMU_OP_MIGRATE_OUTGOING, VIR_HOOK_SUBOP_BEGIN, + "source", xml, NULL); + + if (hookret < 0) + return NULL; + } + if (qemuBlockNodesEnsureActive(vm, vm->job->asyncJob) < 0) return NULL; -- 2.53.0
When outgoing migration fails after the "migrate-outgoing begin source" hook was executed, the source host has no way to be notified and cannot undo changes made in the begin hook (e.g., restoring exclusive storage locks after switching to shared mode). Add a hook call on the source host when migration fails, using the "migrate-outgoing" operation with "end" as the sub-operation and "failed" as the extra argument. The hook output and return code are ignored since migration has already failed. For qemu, the fail hook is called in the confirm phase when retcode != 0. For libxl, the fail hook is called in both the perform phase (when perform fails, since confirm is not called) and the confirm phase when cancelled is true. Resolves: https://gitlab.com/libvirt/libvirt/-/issues/37 Signed-off-by: Guy Godfroy <guy.godfroy@gugod.fr> --- docs/hooks.rst | 27 ++++++++++++++++++++++++++- src/libxl/libxl_migration.c | 22 ++++++++++++++++++++++ src/qemu/qemu_migration.c | 12 ++++++++++++ 3 files changed, 60 insertions(+), 1 deletion(-) diff --git a/docs/hooks.rst b/docs/hooks.rst index cff1711536..4fe02b7c28 100644 --- a/docs/hooks.rst +++ b/docs/hooks.rst @@ -269,6 +269,17 @@ operation. There is no specific operation to indicate a "restart" is occurring. script returns failure, the migration will be canceled. This hook may be used, e.g., to change storage lock mode from exclusive to shared before migration. +- :since:`Since 12.3.0`, when migration fails on the source host, the qemu + hook script is called to allow the source host to undo any changes made in + the begin hook. It is called as: + + :: + + /etc/libvirt/hooks/qemu guest_name migrate-outgoing end failed + + with domain XML sent to standard input of the script. Any output and the + return code of the script are ignored. + - :since:`Since 1.2.9`, the qemu hook script is also called when restoring a saved image either via the API or automatically when restoring a managed save machine. It is called as: @@ -454,6 +465,17 @@ operation. There is no specific operation to indicate a "restart" is occurring. script returns failure, the migration will be canceled. This hook may be used, e.g., to change storage lock mode from exclusive to shared before migration. +- :since:`Since 12.3.0`, when migration fails on the source host, the libxl + hook script is called to allow the source host to undo any changes made in + the begin hook. It is called as: + + :: + + /etc/libvirt/hooks/libxl guest_name migrate-outgoing end failed + + with domain XML sent to standard input of the script. Any output and the + return code of the script are ignored. + - :since:`Since 6.5.0`, you can also place several hook scripts in the directory ``/etc/libvirt/hooks/libxl.d/``. They are executed in alphabetical order after main script. In this case each script also acts as filter and can @@ -599,7 +621,10 @@ destination hosts: called for domain start are executed on **destination** host. #. If all of these hook script executions exit successfully (exit status 0), the migration continues. Any other exit code indicates failure, and the - migration is aborted. + migration is aborted. If migration is aborted, the *qemu* hook script on + the **source** host is executed with the "migrate-outgoing" operation, + "end" sub-operation, and "failed" as the extra argument + (:since:`since 12.3.0`). The return code is ignored. #. The QEMU guest is then migrated to the destination host. #. Unless an error occurs during the migration process, the *qemu* hook script on the **source** host is then executed with the "stopped" and "release" diff --git a/src/libxl/libxl_migration.c b/src/libxl/libxl_migration.c index be51cfd316..4e7e2bdbe9 100644 --- a/src/libxl/libxl_migration.c +++ b/src/libxl/libxl_migration.c @@ -1237,6 +1237,17 @@ libxlDomainMigrationSrcPerform(libxlDriverPrivate *driver, VIR_WARN("Unable to release lease on %s", vm->def->name); } } else { + /* Call hook to notify source host that migration failed */ + if (virHookPresent(VIR_HOOK_DRIVER_LIBXL)) { + g_autofree char *hookxml = NULL; + + if ((hookxml = virDomainDefFormat(vm->def, driver->xmlopt, + VIR_DOMAIN_DEF_FORMAT_SECURE))) + virHookCall(VIR_HOOK_DRIVER_LIBXL, vm->def->name, + VIR_HOOK_LIBXL_OP_MIGRATE_OUTGOING, VIR_HOOK_SUBOP_END, + "failed", hookxml, NULL); + } + /* * Confirm phase will not be executed if perform fails. End the * job started in begin phase. @@ -1358,6 +1369,17 @@ libxlDomainMigrationSrcConfirm(libxlDriverPrivate *driver, virObjectEvent *event = NULL; if (cancelled) { + /* Call hook to notify source host that migration failed */ + if (virHookPresent(VIR_HOOK_DRIVER_LIBXL)) { + g_autofree char *hookxml = NULL; + + if ((hookxml = virDomainDefFormat(vm->def, driver->xmlopt, + VIR_DOMAIN_DEF_FORMAT_SECURE))) + virHookCall(VIR_HOOK_DRIVER_LIBXL, vm->def->name, + VIR_HOOK_LIBXL_OP_MIGRATE_OUTGOING, VIR_HOOK_SUBOP_END, + "failed", hookxml, NULL); + } + /* Resume lock process that was paused in MigrationSrcPerform */ virDomainLockProcessResume(driver->lockManager, "xen:///system", diff --git a/src/qemu/qemu_migration.c b/src/qemu/qemu_migration.c index e9ce2d8b8b..afb5161ea3 100644 --- a/src/qemu/qemu_migration.c +++ b/src/qemu/qemu_migration.c @@ -4218,6 +4218,18 @@ qemuMigrationSrcConfirmPhase(virQEMUDriver *driver, qemuDomainSetMaxMemLock(vm, 0, &priv->preMigrationMemlock); } + /* Call hook to notify source host that migration failed */ + if (virHookPresent(VIR_HOOK_DRIVER_QEMU)) { + g_autofree char *xml = NULL; + + if ((xml = qemuDomainDefFormatLive(driver, priv->qemuCaps, + vm->def, priv->origCPU, + false, true))) + virHookCall(VIR_HOOK_DRIVER_QEMU, vm->def->name, + VIR_HOOK_QEMU_OP_MIGRATE_OUTGOING, VIR_HOOK_SUBOP_END, + "failed", xml, NULL); + } + qemuDomainSaveStatus(vm); virErrorRestore(&orig_err); } -- 2.53.0
Guy Godfroy (4): docs: hooks: Clarify that the migrate operation is on the destination host util: hook: Add migrate-outgoing operation for qemu and libxl drivers qemu, libxl: migration: Call hook script on source host at migration start qemu, libxl: migration: Call hook script on source host when migration fails docs/hooks.rst | 63 ++++++++++++++++++++++++++++++++++--- src/libxl/libxl_migration.c | 39 +++++++++++++++++++++++ src/qemu/qemu_migration.c | 29 +++++++++++++++++ src/util/virhook.c | 2 ++ src/util/virhook.h | 2 ++ 5 files changed, 130 insertions(+), 5 deletions(-) -- 2.53.0
The documentation for the "migrate" hook operation described it as being called at the beginning of incoming migration, but didn't explicitly mention that this happens on the destination host. Add "on the **destination** host" to make this clear. Signed-off-by: Guy Godfroy <guy.godfroy@gugod.fr> --- docs/hooks.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/hooks.rst b/docs/hooks.rst index e1745b8cc7..0310e0d82c 100644 --- a/docs/hooks.rst +++ b/docs/hooks.rst @@ -244,7 +244,7 @@ operation. There is no specific operation to indicate a "restart" is occurring. - :since:`Since 0.9.11`, the qemu hook script is also called at the beginning - of incoming migration. It is called as: + of incoming migration on the **destination** host. It is called as: :: @@ -417,7 +417,7 @@ operation. There is no specific operation to indicate a "restart" is occurring. /etc/libvirt/hooks/libxl guest_name release end - - :since:`Since 2.1.0`, the libxl hook script is also called at the beginning - of incoming migration. It is called as: + of incoming migration on the **destination** host. It is called as: :: -- 2.53.0
Add VIR_HOOK_QEMU_OP_MIGRATE_OUTGOING and VIR_HOOK_LIBXL_OP_MIGRATE_OUTGOING enum values with corresponding "migrate-outgoing" string mappings. This new operation will be used for hooks called on the source host during outgoing migration, distinct from the existing "migrate" operation which is used for destination host hooks. Signed-off-by: Guy Godfroy <guy.godfroy@gugod.fr> --- src/util/virhook.c | 2 ++ src/util/virhook.h | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/util/virhook.c b/src/util/virhook.c index d012bb1825..7f7e253a67 100644 --- a/src/util/virhook.c +++ b/src/util/virhook.c @@ -84,6 +84,7 @@ VIR_ENUM_IMPL(virHookQemuOp, "reconnect", "attach", "restore", + "migrate-outgoing", ); VIR_ENUM_IMPL(virHookLxcOp, @@ -115,6 +116,7 @@ VIR_ENUM_IMPL(virHookLibxlOp, "migrate", "started", "reconnect", + "migrate-outgoing", ); VIR_ENUM_IMPL(virHookBhyveOp, diff --git a/src/util/virhook.h b/src/util/virhook.h index d8237c837e..a7536fb01e 100644 --- a/src/util/virhook.h +++ b/src/util/virhook.h @@ -60,6 +60,7 @@ typedef enum { VIR_HOOK_QEMU_OP_RECONNECT, /* domain is being reconnected by libvirt */ VIR_HOOK_QEMU_OP_ATTACH, /* domain is being attached to be libvirt */ VIR_HOOK_QEMU_OP_RESTORE, /* domain is being restored */ + VIR_HOOK_QEMU_OP_MIGRATE_OUTGOING, /* domain is being migrated out */ VIR_HOOK_QEMU_OP_LAST, } virHookQemuOpType; @@ -94,6 +95,7 @@ typedef enum { VIR_HOOK_LIBXL_OP_MIGRATE, /* domain is being migrated */ VIR_HOOK_LIBXL_OP_STARTED, /* domain has started */ VIR_HOOK_LIBXL_OP_RECONNECT, /* domain is being reconnected by libvirt */ + VIR_HOOK_LIBXL_OP_MIGRATE_OUTGOING, /* domain is being migrated out */ VIR_HOOK_LIBXL_OP_LAST, } virHookLibxlOpType; -- 2.53.0
Currently, the hook script with the "migrate" operation is only called on the destination host at the beginning of incoming migration. This makes it impossible for the source host to prepare for migration, for example to change storage locks from exclusive to shared mode when using shared LVM storage with lvmlockd. Add a hook call on the source host at the beginning of outgoing migration using the new "migrate-outgoing" operation. The hook is called with "source" as the extra argument and "begin" as the sub-operation. The script is not used as a filter; its output is ignored. If the script returns failure, the migration is canceled. The same change is also applied to the libxl driver for consistency. Signed-off-by: Guy Godfroy <guy.godfroy@gugod.fr> --- docs/hooks.rst | 32 ++++++++++++++++++++++++++++++-- src/libxl/libxl_migration.c | 17 +++++++++++++++++ src/qemu/qemu_migration.c | 17 +++++++++++++++++ 3 files changed, 64 insertions(+), 2 deletions(-) diff --git a/docs/hooks.rst b/docs/hooks.rst index 0310e0d82c..cff1711536 100644 --- a/docs/hooks.rst +++ b/docs/hooks.rst @@ -257,6 +257,18 @@ operation. There is no specific operation to indicate a "restart" is occurring. is not valid, incoming migration will be canceled. This hook may be used, e.g., to change location of disk images for incoming domains. +- :since:`Since 12.3.0`, the qemu hook script is also called at the beginning + of outgoing migration on the **source** host. It is called as: + + :: + + /etc/libvirt/hooks/qemu guest_name migrate-outgoing begin source + + with domain XML sent to standard input of the script. Unlike the destination + hook, this script does not act as a filter and any output is ignored. If the + script returns failure, the migration will be canceled. This hook may be used, + e.g., to change storage lock mode from exclusive to shared before migration. + - :since:`Since 1.2.9`, the qemu hook script is also called when restoring a saved image either via the API or automatically when restoring a managed save machine. It is called as: @@ -430,6 +442,18 @@ operation. There is no specific operation to indicate a "restart" is occurring. is not valid, incoming migration will be canceled. This hook may be used, e.g., to change location of disk images for incoming domains. +- :since:`Since 12.3.0`, the libxl hook script is also called at the beginning + of outgoing migration on the **source** host. It is called as: + + :: + + /etc/libvirt/hooks/libxl guest_name migrate-outgoing begin source + + with domain XML sent to standard input of the script. Unlike the destination + hook, this script does not act as a filter and any output is ignored. If the + script returns failure, the migration will be canceled. This hook may be used, + e.g., to change storage lock mode from exclusive to shared before migration. + - :since:`Since 6.5.0`, you can also place several hook scripts in the directory ``/etc/libvirt/hooks/libxl.d/``. They are executed in alphabetical order after main script. In this case each script also acts as filter and can @@ -566,10 +590,14 @@ Migration of a QEMU guest involves running hook scripts on both the source and destination hosts: #. At the beginning of the migration, the *qemu* hook script on the - **destination** host is executed with the "migrate" operation. + **source** host is executed with the "migrate-outgoing" operation and + "source" as the extra argument (:since:`since 12.3.0`). This allows the + source host to prepare for migration, e.g., changing storage locks. +#. Then, the *qemu* hook script on the **destination** host is executed + with the "migrate" operation. #. Before QEMU process is spawned, the two operations ("prepare" and "start") called for domain start are executed on **destination** host. -#. If both of these hook script executions exit successfully (exit status 0), +#. If all of these hook script executions exit successfully (exit status 0), the migration continues. Any other exit code indicates failure, and the migration is aborted. #. The QEMU guest is then migrated to the destination host. diff --git a/src/libxl/libxl_migration.c b/src/libxl/libxl_migration.c index f5dee7627b..be51cfd316 100644 --- a/src/libxl/libxl_migration.c +++ b/src/libxl/libxl_migration.c @@ -411,6 +411,23 @@ libxlDomainMigrationSrcBegin(virConnectPtr conn, if (!libxlDomainMigrationIsAllowed(def)) goto endjob; + /* Call hook to allow source host to prepare for migration */ + if (virHookPresent(VIR_HOOK_DRIVER_LIBXL)) { + g_autofree char *hookxml = NULL; + int hookret; + + if (!(hookxml = virDomainDefFormat(def, driver->xmlopt, + VIR_DOMAIN_DEF_FORMAT_SECURE))) + goto endjob; + + hookret = virHookCall(VIR_HOOK_DRIVER_LIBXL, def->name, + VIR_HOOK_LIBXL_OP_MIGRATE_OUTGOING, VIR_HOOK_SUBOP_BEGIN, + "source", hookxml, NULL); + + if (hookret < 0) + goto endjob; + } + xml = virDomainDefFormat(def, driver->xmlopt, VIR_DOMAIN_DEF_FORMAT_SECURE); /* Valid xml means success! EndJob in the confirm phase */ if (xml) diff --git a/src/qemu/qemu_migration.c b/src/qemu/qemu_migration.c index fec808ccfb..e9ce2d8b8b 100644 --- a/src/qemu/qemu_migration.c +++ b/src/qemu/qemu_migration.c @@ -2870,6 +2870,23 @@ qemuMigrationSrcBeginPhase(virQEMUDriver *driver, vm->newDef && !qemuDomainVcpuHotplugIsInOrder(vm->newDef))) cookieFlags |= QEMU_MIGRATION_COOKIE_CPU_HOTPLUG; + /* Call hook to allow source host to prepare for migration */ + if (virHookPresent(VIR_HOOK_DRIVER_QEMU)) { + g_autofree char *xml = NULL; + int hookret; + + if (!(xml = qemuDomainDefFormatLive(driver, priv->qemuCaps, vm->def, + priv->origCPU, false, true))) + return NULL; + + hookret = virHookCall(VIR_HOOK_DRIVER_QEMU, vm->def->name, + VIR_HOOK_QEMU_OP_MIGRATE_OUTGOING, VIR_HOOK_SUBOP_BEGIN, + "source", xml, NULL); + + if (hookret < 0) + return NULL; + } + if (qemuBlockNodesEnsureActive(vm, vm->job->asyncJob) < 0) return NULL; -- 2.53.0
When outgoing migration fails after the "migrate-outgoing begin source" hook was executed, the source host has no way to be notified and cannot undo changes made in the begin hook (e.g., restoring exclusive storage locks after switching to shared mode). Add a hook call on the source host when migration fails, using the "migrate-outgoing" operation with "end" as the sub-operation and "failed" as the extra argument. The hook output and return code are ignored since migration has already failed. For qemu, the fail hook is called in the confirm phase when retcode != 0. For libxl, the fail hook is called in both the perform phase (when perform fails, since confirm is not called) and the confirm phase when cancelled is true. Resolves: https://gitlab.com/libvirt/libvirt/-/issues/37 Signed-off-by: Guy Godfroy <guy.godfroy@gugod.fr> --- docs/hooks.rst | 27 ++++++++++++++++++++++++++- src/libxl/libxl_migration.c | 22 ++++++++++++++++++++++ src/qemu/qemu_migration.c | 12 ++++++++++++ 3 files changed, 60 insertions(+), 1 deletion(-) diff --git a/docs/hooks.rst b/docs/hooks.rst index cff1711536..4fe02b7c28 100644 --- a/docs/hooks.rst +++ b/docs/hooks.rst @@ -269,6 +269,17 @@ operation. There is no specific operation to indicate a "restart" is occurring. script returns failure, the migration will be canceled. This hook may be used, e.g., to change storage lock mode from exclusive to shared before migration. +- :since:`Since 12.3.0`, when migration fails on the source host, the qemu + hook script is called to allow the source host to undo any changes made in + the begin hook. It is called as: + + :: + + /etc/libvirt/hooks/qemu guest_name migrate-outgoing end failed + + with domain XML sent to standard input of the script. Any output and the + return code of the script are ignored. + - :since:`Since 1.2.9`, the qemu hook script is also called when restoring a saved image either via the API or automatically when restoring a managed save machine. It is called as: @@ -454,6 +465,17 @@ operation. There is no specific operation to indicate a "restart" is occurring. script returns failure, the migration will be canceled. This hook may be used, e.g., to change storage lock mode from exclusive to shared before migration. +- :since:`Since 12.3.0`, when migration fails on the source host, the libxl + hook script is called to allow the source host to undo any changes made in + the begin hook. It is called as: + + :: + + /etc/libvirt/hooks/libxl guest_name migrate-outgoing end failed + + with domain XML sent to standard input of the script. Any output and the + return code of the script are ignored. + - :since:`Since 6.5.0`, you can also place several hook scripts in the directory ``/etc/libvirt/hooks/libxl.d/``. They are executed in alphabetical order after main script. In this case each script also acts as filter and can @@ -599,7 +621,10 @@ destination hosts: called for domain start are executed on **destination** host. #. If all of these hook script executions exit successfully (exit status 0), the migration continues. Any other exit code indicates failure, and the - migration is aborted. + migration is aborted. If migration is aborted, the *qemu* hook script on + the **source** host is executed with the "migrate-outgoing" operation, + "end" sub-operation, and "failed" as the extra argument + (:since:`since 12.3.0`). The return code is ignored. #. The QEMU guest is then migrated to the destination host. #. Unless an error occurs during the migration process, the *qemu* hook script on the **source** host is then executed with the "stopped" and "release" diff --git a/src/libxl/libxl_migration.c b/src/libxl/libxl_migration.c index be51cfd316..4e7e2bdbe9 100644 --- a/src/libxl/libxl_migration.c +++ b/src/libxl/libxl_migration.c @@ -1237,6 +1237,17 @@ libxlDomainMigrationSrcPerform(libxlDriverPrivate *driver, VIR_WARN("Unable to release lease on %s", vm->def->name); } } else { + /* Call hook to notify source host that migration failed */ + if (virHookPresent(VIR_HOOK_DRIVER_LIBXL)) { + g_autofree char *hookxml = NULL; + + if ((hookxml = virDomainDefFormat(vm->def, driver->xmlopt, + VIR_DOMAIN_DEF_FORMAT_SECURE))) + virHookCall(VIR_HOOK_DRIVER_LIBXL, vm->def->name, + VIR_HOOK_LIBXL_OP_MIGRATE_OUTGOING, VIR_HOOK_SUBOP_END, + "failed", hookxml, NULL); + } + /* * Confirm phase will not be executed if perform fails. End the * job started in begin phase. @@ -1358,6 +1369,17 @@ libxlDomainMigrationSrcConfirm(libxlDriverPrivate *driver, virObjectEvent *event = NULL; if (cancelled) { + /* Call hook to notify source host that migration failed */ + if (virHookPresent(VIR_HOOK_DRIVER_LIBXL)) { + g_autofree char *hookxml = NULL; + + if ((hookxml = virDomainDefFormat(vm->def, driver->xmlopt, + VIR_DOMAIN_DEF_FORMAT_SECURE))) + virHookCall(VIR_HOOK_DRIVER_LIBXL, vm->def->name, + VIR_HOOK_LIBXL_OP_MIGRATE_OUTGOING, VIR_HOOK_SUBOP_END, + "failed", hookxml, NULL); + } + /* Resume lock process that was paused in MigrationSrcPerform */ virDomainLockProcessResume(driver->lockManager, "xen:///system", diff --git a/src/qemu/qemu_migration.c b/src/qemu/qemu_migration.c index e9ce2d8b8b..afb5161ea3 100644 --- a/src/qemu/qemu_migration.c +++ b/src/qemu/qemu_migration.c @@ -4218,6 +4218,18 @@ qemuMigrationSrcConfirmPhase(virQEMUDriver *driver, qemuDomainSetMaxMemLock(vm, 0, &priv->preMigrationMemlock); } + /* Call hook to notify source host that migration failed */ + if (virHookPresent(VIR_HOOK_DRIVER_QEMU)) { + g_autofree char *xml = NULL; + + if ((xml = qemuDomainDefFormatLive(driver, priv->qemuCaps, + vm->def, priv->origCPU, + false, true))) + virHookCall(VIR_HOOK_DRIVER_QEMU, vm->def->name, + VIR_HOOK_QEMU_OP_MIGRATE_OUTGOING, VIR_HOOK_SUBOP_END, + "failed", xml, NULL); + } + qemuDomainSaveStatus(vm); virErrorRestore(&orig_err); } -- 2.53.0
I believe i messed up and send twice the same patches, I am sorry. I do not often submit patches using git send-email.
participants (3)
-
Guy Godfroy -
guy.godfroy＠gugod.fr
-
Peter Krempa