[PATCH 00/11] ch_driver: Add basic SAVE and RESTORE VM operations support to CH driver

save, managedsave and restore operations for ch_driver are now supported for domains without any network, hostdev config defined. The input `path` to save and restore commands should be a directory path as cloud-hypervisor expects dir path where it saves VM state and other config files. Purna Pavan Chandra Aekkaladevi (11): ch_driver: Support Save, Restore VM actions from monitor ch_driver: Pass virCHDriverConfig to virCHMonitorNew ch_driver: Add domainSave, domainSaveFlags callbacks ch_driver: Add domainManagedSave callback ch_driver: Implement more save callbacks ch_driver: Refactor virCHProcessStart ch_driver: Implement domain restore callbacks ch_driver: cleanup any stale managed save dir before VM creation ch_driver: Add additional validation for save/restore docs: Update doc for virDomainSave and virDomainRestore NEWS: Mention save & restore support for ch driver NEWS.rst | 5 + src/ch/ch_conf.c | 6 + src/ch/ch_conf.h | 12 + src/ch/ch_driver.c | 515 ++++++++++++++++++++++++++++++++++++++++++- src/ch/ch_monitor.c | 97 +++++++- src/ch/ch_monitor.h | 6 +- src/ch/ch_process.c | 102 +++++++-- src/ch/ch_process.h | 4 + src/libvirt-domain.c | 10 +- 9 files changed, 730 insertions(+), 27 deletions(-) -- 2.34.1

Implement folowing API calls from CH monitor * vmm.snapshot -> to save a domain * vmm.restore -> to restore saved domain Signed-off-by: Purna Pavan Chandra Aekkaladevi <paekkaladevi@linux.microsoft.com> --- src/ch/ch_monitor.c | 87 +++++++++++++++++++++++++++++++++++++++++++++ src/ch/ch_monitor.h | 4 +++ 2 files changed, 91 insertions(+) diff --git a/src/ch/ch_monitor.c b/src/ch/ch_monitor.c index fefbf7e67a..facbff002e 100644 --- a/src/ch/ch_monitor.c +++ b/src/ch/ch_monitor.c @@ -457,6 +457,22 @@ virCHMonitorBuildVMJson(virCHDriver *driver, virDomainDef *vmdef, return 0; } +static int +virCHMonitorBuildKeyValueStringJson(char **jsonstr, + const char *key, + const char *value) +{ + g_autoptr(virJSONValue) content = virJSONValueNewObject(); + + if (virJSONValueObjectAppendString(content, key, value) < 0) + return -1; + + if (!(*jsonstr = virJSONValueToString(content, false))) + return -1; + + return 0; +} + static int chMonitorCreateSocket(const char *socket_path) { @@ -896,6 +912,77 @@ virCHMonitorResumeVM(virCHMonitor *mon) return virCHMonitorPutNoContent(mon, URL_VM_RESUME); } +static int +virCHMonitorSaveRestoreVM(virCHMonitor *mon, const char *path, bool save) +{ + g_autofree char *url = NULL; + int responseCode = 0; + int ret = -1; + g_autofree char *payload = NULL; + g_autofree char *path_url = NULL; + struct curl_slist *headers = NULL; + struct curl_data data = {0}; + + if (save) + url = g_strdup_printf("%s/%s", URL_ROOT, URL_VM_SAVE); + else + url = g_strdup_printf("%s/%s", URL_ROOT, URL_VM_RESTORE); + + headers = curl_slist_append(headers, "Accept: application/json"); + headers = curl_slist_append(headers, "Content-Type: application/json"); + + path_url = g_strdup_printf("file://%s", path); + if (save) { + if (virCHMonitorBuildKeyValueStringJson(&payload, "destination_url", path_url) != 0) + return -1; + } else { + if (virCHMonitorBuildKeyValueStringJson(&payload, "source_url", path_url) != 0) + return -1; + } + + VIR_WITH_OBJECT_LOCK_GUARD(mon) { + /* reset all options of a libcurl session handle at first */ + curl_easy_reset(mon->handle); + + curl_easy_setopt(mon->handle, CURLOPT_UNIX_SOCKET_PATH, mon->socketpath); + curl_easy_setopt(mon->handle, CURLOPT_URL, url); + curl_easy_setopt(mon->handle, CURLOPT_CUSTOMREQUEST, "PUT"); + curl_easy_setopt(mon->handle, CURLOPT_HTTPHEADER, headers); + curl_easy_setopt(mon->handle, CURLOPT_POSTFIELDS, payload); + curl_easy_setopt(mon->handle, CURLOPT_WRITEFUNCTION, curl_callback); + curl_easy_setopt(mon->handle, CURLOPT_WRITEDATA, (void *)&data); + + responseCode = virCHMonitorCurlPerform(mon->handle); + } + + if (responseCode == 200 || responseCode == 204) { + ret = 0; + } else { + data.content = g_realloc(data.content, data.size + 1); + data.content[data.size] = 0; + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + data.content); + g_free(data.content); + } + + /* reset the libcurl handle to avoid leaking a stack pointer to data */ + curl_easy_reset(mon->handle); + curl_slist_free_all(headers); + return ret; +} + +int +virCHMonitorSaveVM(virCHMonitor *mon, const char *to) +{ + return virCHMonitorSaveRestoreVM(mon, to, true); +} + +int +virCHMonitorRestoreVM(virCHMonitor *mon, const char *from) +{ + return virCHMonitorSaveRestoreVM(mon, from, false); +} + /** * virCHMonitorGetInfo: * @mon: Pointer to the monitor diff --git a/src/ch/ch_monitor.h b/src/ch/ch_monitor.h index 47b4e7abbd..3e0befe5d0 100644 --- a/src/ch/ch_monitor.h +++ b/src/ch/ch_monitor.h @@ -37,6 +37,8 @@ #define URL_VM_Suspend "vm.pause" #define URL_VM_RESUME "vm.resume" #define URL_VM_INFO "vm.info" +#define URL_VM_SAVE "vm.snapshot" +#define URL_VM_RESTORE "vm.restore" #define VIRCH_THREAD_NAME_LEN 16 @@ -110,6 +112,8 @@ int virCHMonitorShutdownVM(virCHMonitor *mon); int virCHMonitorRebootVM(virCHMonitor *mon); int virCHMonitorSuspendVM(virCHMonitor *mon); int virCHMonitorResumeVM(virCHMonitor *mon); +int virCHMonitorSaveVM(virCHMonitor *mon, const char *to); +int virCHMonitorRestoreVM(virCHMonitor *mon, const char *from); int virCHMonitorGetInfo(virCHMonitor *mon, virJSONValue **info); void virCHMonitorCPUInfoFree(virCHMonitorCPUInfo *cpus); -- 2.34.1

Pass virCHDriverConfig to VirCHMonitorNew instead of just stateDir so that the cfg can be used for any additional purposes. Signed-off-by: Purna Pavan Chandra Aekkaladevi <paekkaladevi@linux.microsoft.com> --- src/ch/ch_monitor.c | 3 ++- src/ch/ch_monitor.h | 2 +- src/ch/ch_process.c | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/ch/ch_monitor.c b/src/ch/ch_monitor.c index facbff002e..939fa13667 100644 --- a/src/ch/ch_monitor.c +++ b/src/ch/ch_monitor.c @@ -529,10 +529,11 @@ chMonitorCreateSocket(const char *socket_path) } virCHMonitor * -virCHMonitorNew(virDomainObj *vm, const char *socketdir) +virCHMonitorNew(virDomainObj *vm, virCHDriverConfig *cfg) { g_autoptr(virCHMonitor) mon = NULL; g_autoptr(virCommand) cmd = NULL; + const char *socketdir = cfg->stateDir; int socket_fd = 0; if (virCHMonitorInitialize() < 0) diff --git a/src/ch/ch_monitor.h b/src/ch/ch_monitor.h index 3e0befe5d0..ea6b2a771b 100644 --- a/src/ch/ch_monitor.h +++ b/src/ch/ch_monitor.h @@ -101,7 +101,7 @@ struct _virCHMonitor { virCHMonitorThreadInfo *threads; }; -virCHMonitor *virCHMonitorNew(virDomainObj *vm, const char *socketdir); +virCHMonitor *virCHMonitorNew(virDomainObj *vm, virCHDriverConfig *cfg); void virCHMonitorClose(virCHMonitor *mon); G_DEFINE_AUTOPTR_CLEANUP_FUNC(virCHMonitor, virCHMonitorClose); diff --git a/src/ch/ch_process.c b/src/ch/ch_process.c index 7488b1d65d..4b360413fb 100644 --- a/src/ch/ch_process.c +++ b/src/ch/ch_process.c @@ -51,7 +51,7 @@ virCHProcessConnectMonitor(virCHDriver *driver, virCHMonitor *monitor = NULL; virCHDriverConfig *cfg = virCHDriverGetConfig(driver); - monitor = virCHMonitorNew(vm, cfg->stateDir); + monitor = virCHMonitorNew(vm, cfg); virObjectUnref(cfg); return monitor; -- 2.34.1

Implemented save callbacks. CH's vmm.snapshot API is called to save the domain state. The path passed to these callbacks has to be of directory as CH takes dir as input to snapshot and saves multiple files under it. Signed-off-by: Purna Pavan Chandra Aekkaladevi <paekkaladevi@linux.microsoft.com> --- src/ch/ch_conf.h | 11 ++++ src/ch/ch_driver.c | 144 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 155 insertions(+) diff --git a/src/ch/ch_conf.h b/src/ch/ch_conf.h index 579eca894e..4b4c3345b6 100644 --- a/src/ch/ch_conf.h +++ b/src/ch/ch_conf.h @@ -81,6 +81,17 @@ struct _virCHDriver ebtablesContext *ebtables; }; +#define CH_SAVE_MAGIC "libvirt-xml\n \0 \r" +#define CH_SAVE_XML "libvirt-save.xml" + +typedef struct _CHSaveXMLHeader CHSaveXMLHeader; +struct _CHSaveXMLHeader { + char magic[sizeof(CH_SAVE_MAGIC)-1]; + uint32_t xmlLen; + /* 20 bytes used, pad up to 64 bytes */ + uint32_t unused[11]; +}; + virCaps *virCHDriverCapsInit(void); virCaps *virCHDriverGetCapabilities(virCHDriver *driver, bool refresh); diff --git a/src/ch/ch_driver.c b/src/ch/ch_driver.c index 2601eea44b..24f697d2e1 100644 --- a/src/ch/ch_driver.c +++ b/src/ch/ch_driver.c @@ -19,6 +19,7 @@ */ #include <config.h> +#include <fcntl.h> #include "ch_capabilities.h" #include "ch_conf.h" @@ -34,6 +35,7 @@ #include "virerror.h" #include "virlog.h" #include "virobject.h" +#include "virfile.h" #include "virtypedparam.h" #include "virutil.h" #include "viruuid.h" @@ -621,6 +623,146 @@ chDomainDestroy(virDomainPtr dom) return chDomainDestroyFlags(dom, 0); } +/** + * chDoDomainSave: + * @driver: pointer to driver structure + * @vm: pointer to virtual machine structure. Must be locked before invocation. + * @to_dir: directory path (CH needs directory input) to save the domain + * @managed: whether the VM is managed or not + * + * Checks if the domain is running or paused, then suspends it and saves it + * using CH's vmm.snapshot API. CH creates multiple files for config, memory, + * device state into @to_dir. + * + * Returns 0 on success or -1 in case of error + */ +static int +chDoDomainSave(virCHDriver *driver, + virDomainObj *vm, + const char *to_dir, + bool managed) +{ + g_autoptr(virCHDriverConfig) cfg = virCHDriverGetConfig(driver); + virCHDomainObjPrivate *priv = vm->privateData; + CHSaveXMLHeader hdr; + g_autofree char *to = NULL; + g_autofree char *xml = NULL; + uint32_t xml_len; + VIR_AUTOCLOSE fd = -1; + int ret = -1; + + virDomainState domainState = virDomainObjGetState(vm, NULL); + if (domainState == VIR_DOMAIN_RUNNING) { + if (virCHMonitorSuspendVM(priv->monitor) < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("failed to suspend domain before saving")); + goto end; + } + virDomainObjSetState(vm, VIR_DOMAIN_PAUSED, VIR_DOMAIN_PAUSED_SAVE); + } else if (domainState != VIR_DOMAIN_PAUSED) { + virReportError(VIR_ERR_OPERATION_INVALID, "%s", + _("only can save running/paused domain")); + goto end; + } + + if (virDirCreate(to_dir, 0770, cfg->user, cfg->group, + VIR_DIR_CREATE_ALLOW_EXIST) < 0) { + virReportSystemError(errno, _("Failed to create SAVE dir %1$s"), to_dir); + goto end; + } + + to = g_strdup_printf("%s/%s", to_dir, CH_SAVE_XML); + if ((fd = virFileOpenAs(to, O_CREAT|O_TRUNC|O_WRONLY, S_IRUSR|S_IWUSR, + cfg->user, cfg->group, 0)) < 0) { + virReportSystemError(-fd, + _("Failed to create/open domain save xml file '%1$s'"), + to); + goto end; + } + + if ((xml = virDomainDefFormat(vm->def, driver->xmlopt, 0)) == NULL) + goto end; + xml_len = strlen(xml) + 1; + + memset(&hdr, 0, sizeof(hdr)); + memcpy(hdr.magic, CH_SAVE_MAGIC, sizeof(hdr.magic)); + hdr.xmlLen = xml_len; + + if (safewrite(fd, &hdr, sizeof(hdr)) != sizeof(hdr)) { + virReportSystemError(errno, "%s", _("Failed to write file header")); + goto end; + } + + if (safewrite(fd, xml, xml_len) != xml_len) { + virReportSystemError(errno, "%s", _("Failed to write xml definition")); + goto end; + } + + if (virCHMonitorSaveVM(priv->monitor, to_dir) < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", _("Failed to save domain")); + goto end; + } + + if (virCHProcessStop(driver, vm, VIR_DOMAIN_SHUTOFF_SAVED) < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("Failed to shutoff after domain save")); + goto end; + } + + vm->hasManagedSave = managed; + ret = 0; + + end: + return ret; +} + +static int +chDomainSaveFlags(virDomainPtr dom, const char *to, const char *dxml, unsigned int flags) +{ + virCHDriver *driver = dom->conn->privateData; + virDomainObj *vm = NULL; + int ret = -1; + + virCheckFlags(0, -1); + if (dxml) { + virReportError(VIR_ERR_ARGUMENT_UNSUPPORTED, "%s", + _("xml modification unsupported")); + return -1; + } + + if (!(vm = virCHDomainObjFromDomain(dom))) + goto cleanup; + + if (virDomainSaveFlagsEnsureACL(dom->conn, vm->def) < 0) + goto cleanup; + + if (virDomainObjBeginJob(vm, VIR_JOB_MODIFY) < 0) + goto cleanup; + + if (virDomainObjCheckActive(vm) < 0) + goto endjob; + + if (chDoDomainSave(driver, vm, to, false) < 0) + goto endjob; + + /* Remove if VM is not persistent */ + virCHDomainRemoveInactive(driver, vm); + ret = 0; + + endjob: + virDomainObjEndJob(vm); + + cleanup: + virDomainObjEndAPI(&vm); + return ret; +} + +static int +chDomainSave(virDomainPtr dom, const char *to) +{ + return chDomainSaveFlags(dom, to, NULL, 0); +} + static virDomainPtr chDomainLookupByID(virConnectPtr conn, int id) { @@ -1743,6 +1885,8 @@ static virHypervisorDriver chHypervisorDriver = { .nodeGetCPUMap = chNodeGetCPUMap, /* 8.0.0 */ .domainSetNumaParameters = chDomainSetNumaParameters, /* 8.1.0 */ .domainGetNumaParameters = chDomainGetNumaParameters, /* 8.1.0 */ + .domainSave = chDomainSave, /* 10.2.0 */ + .domainSaveFlags = chDomainSaveFlags, /* 10.2.0 */ }; static virConnectDriver chConnectDriver = { -- 2.34.1

Create libvirt managed saveDir and pass it to CH to save the VM Signed-off-by: Purna Pavan Chandra Aekkaladevi <paekkaladevi@linux.microsoft.com> --- src/ch/ch_conf.c | 6 ++++++ src/ch/ch_conf.h | 1 + src/ch/ch_driver.c | 51 +++++++++++++++++++++++++++++++++++++++++++++ src/ch/ch_monitor.c | 7 +++++++ 4 files changed, 65 insertions(+) diff --git a/src/ch/ch_conf.c b/src/ch/ch_conf.c index a7b2285886..cab97639c4 100644 --- a/src/ch/ch_conf.c +++ b/src/ch/ch_conf.c @@ -148,10 +148,12 @@ virCHDriverConfigNew(bool privileged) if (privileged) { cfg->logDir = g_strdup_printf("%s/log/libvirt/ch", LOCALSTATEDIR); cfg->stateDir = g_strdup_printf("%s/libvirt/ch", RUNSTATEDIR); + cfg->saveDir = g_strdup_printf("%s/lib/libvirt/ch/save", LOCALSTATEDIR); } else { g_autofree char *rundir = NULL; g_autofree char *cachedir = NULL; + g_autofree char *configbasedir = NULL; cachedir = virGetUserCacheDirectory(); @@ -159,6 +161,9 @@ virCHDriverConfigNew(bool privileged) rundir = virGetUserRuntimeDirectory(); cfg->stateDir = g_strdup_printf("%s/ch/run", rundir); + + configbasedir = virGetUserConfigDirectory(); + cfg->saveDir = g_strdup_printf("%s/ch/save", configbasedir); } return cfg; @@ -175,6 +180,7 @@ virCHDriverConfigDispose(void *obj) { virCHDriverConfig *cfg = obj; + g_free(cfg->saveDir); g_free(cfg->stateDir); g_free(cfg->logDir); } diff --git a/src/ch/ch_conf.h b/src/ch/ch_conf.h index 4b4c3345b6..a77cad7a2a 100644 --- a/src/ch/ch_conf.h +++ b/src/ch/ch_conf.h @@ -37,6 +37,7 @@ struct _virCHDriverConfig { char *stateDir; char *logDir; + char *saveDir; int cgroupControllers; diff --git a/src/ch/ch_driver.c b/src/ch/ch_driver.c index 24f697d2e1..be6e25c8e2 100644 --- a/src/ch/ch_driver.c +++ b/src/ch/ch_driver.c @@ -178,6 +178,14 @@ static char *chConnectGetCapabilities(virConnectPtr conn) return xml; } +static char * +chDomainManagedSavePath(virCHDriver *driver, virDomainObj *vm) +{ + g_autoptr(virCHDriverConfig) cfg = virCHDriverGetConfig(driver); + return g_strdup_printf("%s/%s.save", cfg->saveDir, vm->def->name); +} + + /** * chDomainCreateXML: * @conn: pointer to connection @@ -763,6 +771,48 @@ chDomainSave(virDomainPtr dom, const char *to) return chDomainSaveFlags(dom, to, NULL, 0); } +static int +chDomainManagedSave(virDomainPtr dom, unsigned int flags) +{ + virCHDriver *driver = dom->conn->privateData; + virDomainObj *vm = NULL; + g_autofree char *to = NULL; + int ret = -1; + + virCheckFlags(0, -1); + + if (!(vm = virCHDomainObjFromDomain(dom))) + goto cleanup; + + if (virDomainManagedSaveEnsureACL(dom->conn, vm->def) < 0) + goto cleanup; + + if (virDomainObjBeginJob(vm, VIR_JOB_MODIFY) < 0) + goto cleanup; + + if (virDomainObjCheckActive(vm) < 0) + goto endjob; + + if (!vm->persistent) { + virReportError(VIR_ERR_OPERATION_INVALID, "%s", + _("cannot do managed save for transient domain")); + goto endjob; + } + + to = chDomainManagedSavePath(driver, vm); + if (chDoDomainSave(driver, vm, to, true) < 0) + goto endjob; + + ret = 0; + + endjob: + virDomainObjEndJob(vm); + + cleanup: + virDomainObjEndAPI(&vm); + return ret; +} + static virDomainPtr chDomainLookupByID(virConnectPtr conn, int id) { @@ -1887,6 +1937,7 @@ static virHypervisorDriver chHypervisorDriver = { .domainGetNumaParameters = chDomainGetNumaParameters, /* 8.1.0 */ .domainSave = chDomainSave, /* 10.2.0 */ .domainSaveFlags = chDomainSaveFlags, /* 10.2.0 */ + .domainManagedSave = chDomainManagedSave, /* 10.2.0 */ }; static virConnectDriver chConnectDriver = { diff --git a/src/ch/ch_monitor.c b/src/ch/ch_monitor.c index 939fa13667..7b6b77de1c 100644 --- a/src/ch/ch_monitor.c +++ b/src/ch/ch_monitor.c @@ -557,6 +557,13 @@ virCHMonitorNew(virDomainObj *vm, virCHDriverConfig *cfg) return NULL; } + if (g_mkdir_with_parents(cfg->saveDir, 0777) < 0) { + virReportSystemError(errno, + _("Cannot create save directory '%1$s'"), + cfg->saveDir); + return NULL; + } + cmd = virCommandNew(vm->def->emulator); virCommandSetUmask(cmd, 0x002); socket_fd = chMonitorCreateSocket(mon->socketpath); -- 2.34.1

Following callbacks have been implemented * domainSaveImageGetXMLDesc * domainManagedSaveRemove * domainManagedSaveGetXMLDesc * domainHasManagedSaveImage Signed-off-by: Purna Pavan Chandra Aekkaladevi <paekkaladevi@linux.microsoft.com> --- src/ch/ch_driver.c | 180 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 180 insertions(+) diff --git a/src/ch/ch_driver.c b/src/ch/ch_driver.c index be6e25c8e2..577544c941 100644 --- a/src/ch/ch_driver.c +++ b/src/ch/ch_driver.c @@ -771,6 +771,99 @@ chDomainSave(virDomainPtr dom, const char *to) return chDomainSaveFlags(dom, to, NULL, 0); } +static char * +chDomainSaveXMLRead(int fd) +{ + g_autofree char *xml = NULL; + CHSaveXMLHeader hdr; + + if (saferead(fd, &hdr, sizeof(hdr)) != sizeof(hdr)) { + virReportError(VIR_ERR_OPERATION_FAILED, "%s", + _("failed to read CHSaveXMLHeader header")); + return NULL; + } + + if (memcmp(hdr.magic, CH_SAVE_MAGIC, sizeof(hdr.magic))) { + virReportError(VIR_ERR_INVALID_ARG, "%s", + _("save image magic is incorrect")); + return NULL; + } + + if (hdr.xmlLen <= 0) { + virReportError(VIR_ERR_OPERATION_FAILED, + _("invalid XML length: %1$d"), hdr.xmlLen); + return NULL; + } + + xml = g_new0(char, hdr.xmlLen); + + if (saferead(fd, xml, hdr.xmlLen) != hdr.xmlLen) { + virReportError(VIR_ERR_OPERATION_FAILED, "%s", + _("failed to read XML")); + return NULL; + } + + return g_steal_pointer(&xml); +} + +static int chDomainSaveImageRead(virCHDriver *driver, + const char *path, + virDomainDef **ret_def) +{ + g_autoptr(virCHDriverConfig) cfg = virCHDriverGetConfig(driver); + g_autoptr(virDomainDef) def = NULL; + g_autofree char *from = NULL; + g_autofree char *xml = NULL; + VIR_AUTOCLOSE fd = -1; + int ret = -1; + + from = g_strdup_printf("%s/%s", path, CH_SAVE_XML); + if ((fd = virFileOpenAs(from, O_RDONLY, 0, cfg->user, cfg->group, 0)) < 0) { + virReportSystemError(errno, + _("Failed to open domain save file '%1$s'"), + from); + goto end; + } + + if (!(xml = chDomainSaveXMLRead(fd))) + goto end; + + if (!(def = virDomainDefParseString(xml, driver->xmlopt, NULL, + VIR_DOMAIN_DEF_PARSE_INACTIVE | + VIR_DOMAIN_DEF_PARSE_SKIP_VALIDATE))) + goto end; + + *ret_def = g_steal_pointer(&def); + ret = 0; + + end: + return ret; +} + +static char * +chDomainSaveImageGetXMLDesc(virConnectPtr conn, + const char *path, + unsigned int flags) +{ + virCHDriver *driver = conn->privateData; + g_autoptr(virDomainDef) def = NULL; + char *ret = NULL; + + virCheckFlags(VIR_DOMAIN_SAVE_IMAGE_XML_SECURE, NULL); + + if (chDomainSaveImageRead(driver, path, &def) < 0) + goto cleanup; + + if (virDomainSaveImageGetXMLDescEnsureACL(conn, def) < 0) + goto cleanup; + + ret = virDomainDefFormat(def, driver->xmlopt, + virDomainDefFormatConvertXMLFlags(flags)); + + cleanup: + return ret; +} + static int chDomainManagedSave(virDomainPtr dom, unsigned int flags) { @@ -813,6 +906,89 @@ chDomainManagedSave(virDomainPtr dom, unsigned int flags) return ret; } +static int +chDomainManagedSaveRemove(virDomainPtr dom, unsigned int flags) +{ + virCHDriver *driver = dom->conn->privateData; + virDomainObj *vm; + int ret = -1; + g_autofree char *path = NULL; + + virCheckFlags(0, -1); + + if (!(vm = virCHDomainObjFromDomain(dom))) + return -1; + + if (virDomainManagedSaveRemoveEnsureACL(dom->conn, vm->def) < 0) + goto cleanup; + + path = chDomainManagedSavePath(driver, vm); + + if (virFileDeleteTree(path) < 0) { + virReportSystemError(errno, + _("Failed to remove managed save path '%1$s'"), + path); + goto cleanup; + } + + vm->hasManagedSave = false; + ret = 0; + + cleanup: + virDomainObjEndAPI(&vm); + return ret; +} + +static char * +chDomainManagedSaveGetXMLDesc(virDomainPtr dom, unsigned int flags) +{ + virCHDriver *driver = dom->conn->privateData; + virDomainObj *vm = NULL; + g_autoptr(virDomainDef) def = NULL; + char *ret = NULL; + g_autofree char *path = NULL; + + virCheckFlags(VIR_DOMAIN_SAVE_IMAGE_XML_SECURE, NULL); + + if (!(vm = virCHDomainObjFromDomain(dom))) + goto cleanup; + + path = chDomainManagedSavePath(driver, vm); + if (chDomainSaveImageRead(driver, path, &def) < 0) + goto cleanup; + + if (virDomainManagedSaveGetXMLDescEnsureACL(dom->conn, def, flags) < 0) + goto cleanup; + + ret = virDomainDefFormat(def, driver->xmlopt, + virDomainDefFormatConvertXMLFlags(flags)); + + cleanup: + virDomainObjEndAPI(&vm); + return ret; +} + +static int +chDomainHasManagedSaveImage(virDomainPtr dom, unsigned int flags) +{ + virDomainObj *vm = NULL; + int ret = -1; + + virCheckFlags(0, -1); + + if (!(vm = virCHDomainObjFromDomain(dom))) + return -1; + + if (virDomainHasManagedSaveImageEnsureACL(dom->conn, vm->def) < 0) + goto cleanup; + + ret = vm->hasManagedSave; + + cleanup: + virDomainObjEndAPI(&vm); + return ret; +} + static virDomainPtr chDomainLookupByID(virConnectPtr conn, int id) { @@ -1937,7 +2113,11 @@ static virHypervisorDriver chHypervisorDriver = { .domainGetNumaParameters = chDomainGetNumaParameters, /* 8.1.0 */ .domainSave = chDomainSave, /* 10.2.0 */ .domainSaveFlags = chDomainSaveFlags, /* 10.2.0 */ + .domainSaveImageGetXMLDesc = chDomainSaveImageGetXMLDesc, /* 10.2.0 */ .domainManagedSave = chDomainManagedSave, /* 10.2.0 */ + .domainManagedSaveRemove = chDomainManagedSaveRemove, /* 10.2.0 */ + .domainManagedSaveGetXMLDesc = chDomainManagedSaveGetXMLDesc, /* 10.2.0 */ + .domainHasManagedSaveImage = chDomainHasManagedSaveImage, /* 10.2.0 */ }; static virConnectDriver chConnectDriver = { -- 2.34.1

Signed-off-by: Purna Pavan Chandra Aekkaladevi <paekkaladevi@linux.microsoft.com> --- src/ch/ch_process.c | 47 ++++++++++++++++++++++++++++----------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/src/ch/ch_process.c b/src/ch/ch_process.c index 4b360413fb..02749adfb6 100644 --- a/src/ch/ch_process.c +++ b/src/ch/ch_process.c @@ -460,6 +460,34 @@ virCHProcessSetupVcpus(virDomainObj *vm) return 0; } +static int +virCHProcessSetup(virDomainObj *vm) +{ + virCHDomainObjPrivate *priv = vm->privateData; + + virCHDomainRefreshThreadInfo(vm); + + VIR_DEBUG("Setting emulator tuning/settings"); + if (virCHProcessSetupEmulatorThreads(vm) < 0) + return -1; + + VIR_DEBUG("Setting iothread tuning/settings"); + if (virCHProcessSetupIOThreads(vm) < 0) + return -1; + + VIR_DEBUG("Setting global CPU cgroup (if required)"); + if (virDomainCgroupSetupGlobalCpuCgroup(vm, + priv->cgroup) < 0) + return -1; + + VIR_DEBUG("Setting vCPU tuning/settings"); + if (virCHProcessSetupVcpus(vm) < 0) + return -1; + + virCHProcessUpdateInfo(vm); + return 0; +} + #define PKT_TIMEOUT_MS 500 /* ms */ @@ -763,26 +791,9 @@ virCHProcessStart(virCHDriver *driver, goto cleanup; } - virCHDomainRefreshThreadInfo(vm); - - VIR_DEBUG("Setting emulator tuning/settings"); - if (virCHProcessSetupEmulatorThreads(vm) < 0) - goto cleanup; - - VIR_DEBUG("Setting iothread tuning/settings"); - if (virCHProcessSetupIOThreads(vm) < 0) - goto cleanup; - - VIR_DEBUG("Setting global CPU cgroup (if required)"); - if (virDomainCgroupSetupGlobalCpuCgroup(vm, - priv->cgroup) < 0) + if (virCHProcessSetup(vm) < 0) goto cleanup; - VIR_DEBUG("Setting vCPU tuning/settings"); - if (virCHProcessSetupVcpus(vm) < 0) - goto cleanup; - - virCHProcessUpdateInfo(vm); virDomainObjSetState(vm, VIR_DOMAIN_RUNNING, reason); return 0; -- 2.34.1

Following callbacks have been implemented * domainRestore * domainRestoreFlags The path parameter to these callbacks has to be of the directory where libvirt has performed save. Additionally, call restore in `domainCreate` if the domain has managedsave. Signed-off-by: Purna Pavan Chandra Aekkaladevi <paekkaladevi@linux.microsoft.com> --- src/ch/ch_driver.c | 96 ++++++++++++++++++++++++++++++++++++++++++++- src/ch/ch_process.c | 53 +++++++++++++++++++++++++ src/ch/ch_process.h | 4 ++ 3 files changed, 152 insertions(+), 1 deletion(-) diff --git a/src/ch/ch_driver.c b/src/ch/ch_driver.c index 577544c941..218e2ec56f 100644 --- a/src/ch/ch_driver.c +++ b/src/ch/ch_driver.c @@ -252,6 +252,8 @@ chDomainCreateWithFlags(virDomainPtr dom, unsigned int flags) { virCHDriver *driver = dom->conn->privateData; virDomainObj *vm; + virCHDomainObjPrivate *priv; + g_autofree char *managed_save_path = NULL; int ret = -1; virCheckFlags(0, -1); @@ -265,8 +267,34 @@ chDomainCreateWithFlags(virDomainPtr dom, unsigned int flags) if (virDomainObjBeginJob(vm, VIR_JOB_MODIFY) < 0) goto cleanup; - ret = virCHProcessStart(driver, vm, VIR_DOMAIN_RUNNING_BOOTED); + if (vm->hasManagedSave) { + priv = vm->privateData; + managed_save_path = chDomainManagedSavePath(driver, vm); + if (virCHProcessStartRestore(driver, vm, managed_save_path) < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("failed to restore domain from managed save")); + goto endjob; + } + if (virCHMonitorResumeVM(priv->monitor) < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("failed to resume domain after restore from managed save")); + goto endjob; + } + virDomainObjSetState(vm, VIR_DOMAIN_RUNNING, VIR_DOMAIN_RUNNING_RESTORED); + /* cleanup the save dir after restore */ + if (virFileDeleteTree(managed_save_path) < 0) { + virReportSystemError(errno, + _("Failed to remove managed save path '%1$s'"), + managed_save_path); + goto endjob; + } + vm->hasManagedSave = false; + ret = 0; + } else { + ret = virCHProcessStart(driver, vm, VIR_DOMAIN_RUNNING_BOOTED); + } + endjob: virDomainObjEndJob(vm); cleanup: @@ -989,6 +1017,70 @@ chDomainHasManagedSaveImage(virDomainPtr dom, unsigned int flags) return ret; } +static int +chDomainRestoreFlags(virConnectPtr conn, + const char *from, + const char *dxml, + unsigned int flags) +{ + virCHDriver *driver = conn->privateData; + virDomainObj *vm = NULL; + virCHDomainObjPrivate *priv; + g_autoptr(virDomainDef) def = NULL; + int ret = -1; + + virCheckFlags(0, -1); + + if (dxml) { + virReportError(VIR_ERR_ARGUMENT_UNSUPPORTED, "%s", + _("xml modification unsupported")); + return -1; + } + + if (chDomainSaveImageRead(driver, from, &def) < 0) + goto cleanup; + + if (virDomainRestoreFlagsEnsureACL(conn, def) < 0) + goto cleanup; + + if (!(vm = virDomainObjListAdd(driver->domains, &def, + driver->xmlopt, + VIR_DOMAIN_OBJ_LIST_ADD_LIVE | + VIR_DOMAIN_OBJ_LIST_ADD_CHECK_LIVE, + NULL))) + goto cleanup; + + if (virDomainObjBeginJob(vm, VIR_JOB_MODIFY) < 0) + goto cleanup; + + if (virCHProcessStartRestore(driver, vm, from) < 0) + goto endjob; + + priv = vm->privateData; + if (virCHMonitorResumeVM(priv->monitor) < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("failed to resume domain after restore")); + goto endjob; + } + virDomainObjSetState(vm, VIR_DOMAIN_RUNNING, VIR_DOMAIN_RUNNING_RESTORED); + ret = 0; + + endjob: + virDomainObjEndJob(vm); + + cleanup: + if (vm && ret < 0) + virCHDomainRemoveInactive(driver, vm); + virDomainObjEndAPI(&vm); + return ret; +} + +static int +chDomainRestore(virConnectPtr conn, const char *from) +{ + return chDomainRestoreFlags(conn, from, NULL, 0); +} + static virDomainPtr chDomainLookupByID(virConnectPtr conn, int id) { @@ -2118,6 +2210,8 @@ static virHypervisorDriver chHypervisorDriver = { .domainManagedSaveRemove = chDomainManagedSaveRemove, /* 10.2.0 */ .domainManagedSaveGetXMLDesc = chDomainManagedSaveGetXMLDesc, /* 10.2.0 */ .domainHasManagedSaveImage = chDomainHasManagedSaveImage, /* 10.2.0 */ + .domainRestore = chDomainRestore, /* 10.2.0 */ + .domainRestoreFlags = chDomainRestoreFlags, /* 10.2.0 */ }; static virConnectDriver chConnectDriver = { diff --git a/src/ch/ch_process.c b/src/ch/ch_process.c index 02749adfb6..081fc5af52 100644 --- a/src/ch/ch_process.c +++ b/src/ch/ch_process.c @@ -852,3 +852,56 @@ virCHProcessStop(virCHDriver *driver G_GNUC_UNUSED, return 0; } + +/** + * virCHProcessStartRestore: + * @driver: pointer to driver structure + * @vm: pointer to virtual machine structure + * @from: directory path to restore the VM from + * + * Starts Cloud-Hypervisor process with the restored VM + * + * Returns 0 on success or -1 in case of error + */ +int +virCHProcessStartRestore(virCHDriver *driver, virDomainObj *vm, const char *from) +{ + virCHDomainObjPrivate *priv = vm->privateData; + g_autoptr(virCHDriverConfig) cfg = virCHDriverGetConfig(priv->driver); + + if (!priv->monitor) { + /* Get the first monitor connection if not already */ + if (!(priv->monitor = virCHProcessConnectMonitor(driver, vm))) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("failed to create connection to CH socket")); + return -1; + } + } + + vm->pid = priv->monitor->pid; + vm->def->id = vm->pid; + priv->machineName = virCHDomainGetMachineName(vm); + + if (virCHMonitorRestoreVM(priv->monitor, from) < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("failed to restore domain")); + return -1; + } + + /* Pass 0, NULL as restore only works without networking support */ + if (virDomainCgroupSetupCgroup("ch", vm, + 0, NULL, /* nnicindexes, nicindexes */ + &priv->cgroup, + cfg->cgroupControllers, + 0, /*maxThreadsPerProc*/ + priv->driver->privileged, + priv->machineName) < 0) + return -1; + + if (virCHProcessSetup(vm) < 0) + return -1; + + virDomainObjSetState(vm, VIR_DOMAIN_PAUSED, VIR_DOMAIN_PAUSED_FROM_SNAPSHOT); + + return 0; +} diff --git a/src/ch/ch_process.h b/src/ch/ch_process.h index 800e3f4e23..38bfce3b7f 100644 --- a/src/ch/ch_process.h +++ b/src/ch/ch_process.h @@ -32,3 +32,7 @@ int virCHProcessStop(virCHDriver *driver, int virCHProcessSetupVcpu(virDomainObj *vm, unsigned int vcpuid); + +int virCHProcessStartRestore(virCHDriver *driver, + virDomainObj *vm, + const char *from); -- 2.34.1

There are chances that libvirt process is killed and it resulting in stale managed save dirs. So check for it, and cleanup it there's any. Signed-off-by: Purna Pavan Chandra Aekkaladevi <paekkaladevi@linux.microsoft.com> --- src/ch/ch_driver.c | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/ch/ch_driver.c b/src/ch/ch_driver.c index 218e2ec56f..09cd6b90e7 100644 --- a/src/ch/ch_driver.c +++ b/src/ch/ch_driver.c @@ -206,6 +206,7 @@ chDomainCreateXML(virConnectPtr conn, virDomainObj *vm = NULL; virDomainPtr dom = NULL; unsigned int parse_flags = VIR_DOMAIN_DEF_PARSE_INACTIVE; + g_autofree char *managed_save_path = NULL; virCheckFlags(VIR_DOMAIN_START_VALIDATE, NULL); @@ -228,6 +229,15 @@ chDomainCreateXML(virConnectPtr conn, NULL))) goto cleanup; + /* cleanup if there's any stale managedsave dir */ + managed_save_path = chDomainManagedSavePath(driver, vm); + if (virFileDeleteTree(managed_save_path) < 0) { + virReportSystemError(errno, + _("Failed to cleanup stale managed save dir '%1$s'"), + managed_save_path); + goto cleanup; + } + if (virDomainObjBeginJob(vm, VIR_JOB_MODIFY) < 0) goto cleanup; @@ -315,6 +325,7 @@ chDomainDefineXMLFlags(virConnectPtr conn, const char *xml, unsigned int flags) g_autoptr(virDomainDef) vmdef = NULL; virDomainObj *vm = NULL; virDomainPtr dom = NULL; + g_autofree char *managed_save_path = NULL; unsigned int parse_flags = VIR_DOMAIN_DEF_PARSE_INACTIVE; virCheckFlags(VIR_DOMAIN_DEFINE_VALIDATE, NULL); @@ -337,6 +348,15 @@ chDomainDefineXMLFlags(virConnectPtr conn, const char *xml, unsigned int flags) 0, NULL))) goto cleanup; + /* cleanup if there's any stale managedsave dir */ + managed_save_path = chDomainManagedSavePath(driver, vm); + if (virFileDeleteTree(managed_save_path) < 0) { + virReportSystemError(errno, + _("Failed to cleanup stale managed save dir '%1$s'"), + managed_save_path); + goto cleanup; + } + vm->persistent = 1; dom = virGetDomain(conn, vm->def->name, vm->def->uuid, vm->def->id); -- 2.34.1

Save & Restore are supported without any network and hostdev config defined. So, add a validation for it before performing save. Signed-off-by: Purna Pavan Chandra Aekkaladevi <paekkaladevi@linux.microsoft.com> --- src/ch/ch_driver.c | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/src/ch/ch_driver.c b/src/ch/ch_driver.c index 09cd6b90e7..3d5b8d211a 100644 --- a/src/ch/ch_driver.c +++ b/src/ch/ch_driver.c @@ -679,6 +679,26 @@ chDomainDestroy(virDomainPtr dom) return chDomainDestroyFlags(dom, 0); } +static int +chDomainSaveAdditionalValidation(virDomainDef *vmdef) +{ + /* + SAVE and RESTORE are functional only without any networking and + device passthrough configuration + */ + if (vmdef->nnets > 0) { + virReportError(VIR_ERR_OPERATION_INVALID, "%s", + _("cannot save domain with network interfaces")); + return -1; + } + if (vmdef->nhostdevs > 0) { + virReportError(VIR_ERR_OPERATION_INVALID, "%s", + _("cannot save domain with host devices")); + return -1; + } + return 0; +} + /** * chDoDomainSave: * @driver: pointer to driver structure @@ -701,13 +721,17 @@ chDoDomainSave(virCHDriver *driver, g_autoptr(virCHDriverConfig) cfg = virCHDriverGetConfig(driver); virCHDomainObjPrivate *priv = vm->privateData; CHSaveXMLHeader hdr; + virDomainState domainState; g_autofree char *to = NULL; g_autofree char *xml = NULL; uint32_t xml_len; VIR_AUTOCLOSE fd = -1; int ret = -1; - virDomainState domainState = virDomainObjGetState(vm, NULL); + if (chDomainSaveAdditionalValidation(vm->def) < 0) + goto end; + + domainState = virDomainObjGetState(vm, NULL); if (domainState == VIR_DOMAIN_RUNNING) { if (virCHMonitorSuspendVM(priv->monitor) < 0) { virReportError(VIR_ERR_INTERNAL_ERROR, "%s", -- 2.34.1

ch_driver expects path to be of a dir for save/restore. So, update the documentation at global API as well. Signed-off-by: Purna Pavan Chandra Aekkaladevi <paekkaladevi@linux.microsoft.com> --- src/libvirt-domain.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/libvirt-domain.c b/src/libvirt-domain.c index 9e3c61b66a..7c6b93963c 100644 --- a/src/libvirt-domain.c +++ b/src/libvirt-domain.c @@ -852,11 +852,11 @@ virDomainPMWakeup(virDomainPtr dom, /** * virDomainSave: * @domain: a domain object - * @to: path for the output file + * @to: path for the output save file / directory * - * This method will suspend a domain and save its memory contents to - * a file on disk. After the call, if successful, the domain is not - * listed as running anymore (this ends the life of a transient domain). + * This method will suspend a domain and save its memory contents to a file or + * direcotry (based on the vmm) on disk. After the call, if successful,the domain + * is not listed as running anymore (this ends the life of a transient domain). * Use virDomainRestore() to restore a domain after saving. * * See virDomainSaveFlags() and virDomainSaveParams() for more control. @@ -1053,7 +1053,7 @@ virDomainSaveParams(virDomainPtr domain, /** * virDomainRestore: * @conn: pointer to the hypervisor connection - * @from: path to the input file + * @from: path to the input save file / directory * * This method will restore a domain saved to disk by virDomainSave(). * -- 2.34.1

Signed-off-by: Purna Pavan Chandra Aekkaladevi <paekkaladevi@linux.microsoft.com> --- NEWS.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/NEWS.rst b/NEWS.rst index 44b775b546..e25265e1b0 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -16,6 +16,11 @@ v10.2.0 (unreleased) * **Removed features** * **New features** + * ch: Basic save and restore support for ch driver + + The ch driver now supports basic save and restore operations. This is functional + on domains without any network, host device config defined. The `path` parameter + for save and restore should be a directory. * **Improvements** -- 2.34.1

Hi, Bumping this up for review. Thanks, Pavan On Mon, Mar 11, 2024 at 09:43:56AM +0000, Purna Pavan Chandra Aekkaladevi wrote:
save, managedsave and restore operations for ch_driver are now supported for domains without any network, hostdev config defined. The input `path` to save and restore commands should be a directory path as cloud-hypervisor expects dir path where it saves VM state and other config files.
Purna Pavan Chandra Aekkaladevi (11): ch_driver: Support Save, Restore VM actions from monitor ch_driver: Pass virCHDriverConfig to virCHMonitorNew ch_driver: Add domainSave, domainSaveFlags callbacks ch_driver: Add domainManagedSave callback ch_driver: Implement more save callbacks ch_driver: Refactor virCHProcessStart ch_driver: Implement domain restore callbacks ch_driver: cleanup any stale managed save dir before VM creation ch_driver: Add additional validation for save/restore docs: Update doc for virDomainSave and virDomainRestore NEWS: Mention save & restore support for ch driver
NEWS.rst | 5 + src/ch/ch_conf.c | 6 + src/ch/ch_conf.h | 12 + src/ch/ch_driver.c | 515 ++++++++++++++++++++++++++++++++++++++++++- src/ch/ch_monitor.c | 97 +++++++- src/ch/ch_monitor.h | 6 +- src/ch/ch_process.c | 102 +++++++-- src/ch/ch_process.h | 4 + src/libvirt-domain.c | 10 +- 9 files changed, 730 insertions(+), 27 deletions(-)
-- 2.34.1

On 3/11/24 10:43, Purna Pavan Chandra Aekkaladevi wrote:
save, managedsave and restore operations for ch_driver are now supported for domains without any network, hostdev config defined. The input `path` to save and restore commands should be a directory path as cloud-hypervisor expects dir path where it saves VM state and other config files.
Purna Pavan Chandra Aekkaladevi (11): ch_driver: Support Save, Restore VM actions from monitor ch_driver: Pass virCHDriverConfig to virCHMonitorNew ch_driver: Add domainSave, domainSaveFlags callbacks ch_driver: Add domainManagedSave callback ch_driver: Implement more save callbacks ch_driver: Refactor virCHProcessStart ch_driver: Implement domain restore callbacks ch_driver: cleanup any stale managed save dir before VM creation ch_driver: Add additional validation for save/restore docs: Update doc for virDomainSave and virDomainRestore NEWS: Mention save & restore support for ch driver
NEWS.rst | 5 + src/ch/ch_conf.c | 6 + src/ch/ch_conf.h | 12 + src/ch/ch_driver.c | 515 ++++++++++++++++++++++++++++++++++++++++++- src/ch/ch_monitor.c | 97 +++++++- src/ch/ch_monitor.h | 6 +- src/ch/ch_process.c | 102 +++++++-- src/ch/ch_process.h | 4 + src/libvirt-domain.c | 10 +- 9 files changed, 730 insertions(+), 27 deletions(-)
Reviewed-by: Michal Privoznik <mprivozn@redhat.com> and merged. Michal
participants (2)
-
Michal Prívozník
-
Purna Pavan Chandra Aekkaladevi