On 5/20/25 13:12, Stefan Kober wrote:

--- src/ch/ch_conf.h | 4 + src/ch/ch_domain.h | 2 + src/ch/ch_driver.c | 362 +++++++++++++++++++++++++++++- src/ch/ch_monitor.c | 156 +++++++++++++ src/ch/ch_monitor.h | 8 + src/ch/ch_process.c | 136 ++++++++++- src/ch/ch_process.h | 6 + src/hypervisor/domain_interface.c | 1 + src/libvirt-domain.c | 15 +- 9 files changed, 680 insertions(+), 10 deletions(-)

There's quite a lot happening in just this one patch. Nevertheless, couple of overall comments: 1) Do NOT change VIR_DEBUG() to VIR_WARN(). It cloggs logs for no additional value. If you want to see debug logs there is plenty of ways to do that: https://libvirt.org/kbase/debuglogs.html 2) In general we tend to use the following pattern for functions: int f(..) { int ret = -1; if (cond) goto cleanup; if (otherCond) goto cleanup; ret = 0; cleanup: cleanupCode(); return ret; } See comments below.

diff --git a/src/ch/ch_conf.h b/src/ch/ch_conf.h index b08573476e..40d639eb2a 100644 --- a/src/ch/ch_conf.h +++ b/src/ch/ch_conf.h @@ -26,6 +26,7 @@ #include "ch_capabilities.h" #include "virebtables.h" #include "object_event.h" +#include "virportallocator.h"

#define CH_DRIVER_NAME "CH" #define CH_CMD "cloud-hypervisor" @@ -90,6 +91,9 @@ struct _virCHDriver

/* Immutable pointer, self-locking APIs */ virObjectEventState *domainEventState; + + /* Immutable pointer, immutable object */ + virPortAllocatorRange *migrationPorts; };

#define CH_SAVE_MAGIC "libvirt-xml\n \0 \r" diff --git a/src/ch/ch_domain.h b/src/ch/ch_domain.h index 69a657f6af..f9ad18b518 100644 --- a/src/ch/ch_domain.h +++ b/src/ch/ch_domain.h @@ -25,6 +25,7 @@ #include "virchrdev.h" #include "vircgroup.h" #include "virdomainjob.h" +#include "virthread.h"

typedef struct _virCHDomainObjPrivate virCHDomainObjPrivate; @@ -32,6 +33,7 @@ struct _virCHDomainObjPrivate { virChrdevs *chrdevs; virCHDriver *driver; virCHMonitor *monitor; + virThread *migrationDstReceiveThr; char *machineName; virBitmap *autoCpuset; virBitmap *autoNodeset; diff --git a/src/ch/ch_driver.c b/src/ch/ch_driver.c index 3bdcf66ebd..c53607251b 100644 --- a/src/ch/ch_driver.c +++ b/src/ch/ch_driver.c @@ -29,20 +29,25 @@ #include "ch_process.h" #include "domain_cgroup.h" #include "domain_event.h" +#include "domain_interface.h" #include "datatypes.h" #include "driver.h" +#include "viralloc.h" #include "viraccessapicheck.h" #include "virchrdev.h" #include "virerror.h" #include "virlog.h" #include "virobject.h" #include "virfile.h" +#include "virtime.h" #include "virtypedparam.h" #include "virutil.h" #include "viruuid.h" #include "virnuma.h" #include "virhostmem.h"

+#include "util/virportallocator.h" + #define VIR_FROM_THIS VIR_FROM_CH

VIR_LOG_INIT("ch.ch_driver"); @@ -1453,6 +1458,13 @@ chStateInitialize(bool privileged, if (!(ch_driver->domainEventState = virObjectEventStateNew())) goto cleanup;

+ /* Allocate bitmap for migration port reservation */ + if (!(ch_driver->migrationPorts = + virPortAllocatorRangeNew(_("migration"), + 49152, + 49216)))

These magic constants should be declared as macros somewhere in ch_conf.h.

+ goto cleanup; + if ((rv = chExtractVersion(ch_driver)) < 0) { if (rv == -2) ret = VIR_DRV_STATE_INIT_SKIPPED; @@ -1495,8 +1507,9 @@ chConnectSupportsFeature(virConnectPtr conn, _("Global feature %1$d should have already been handled"), feature); return -1; - case VIR_DRV_FEATURE_MIGRATION_V2: case VIR_DRV_FEATURE_MIGRATION_V3: + return 1; + case VIR_DRV_FEATURE_MIGRATION_V2: case VIR_DRV_FEATURE_MIGRATION_P2P: case VIR_DRV_FEATURE_MIGRATE_CHANGE_PROTECTION: case VIR_DRV_FEATURE_XML_MIGRATABLE: @@ -2338,6 +2351,348 @@ chDomainInterfaceAddresses(virDomain *dom, return ret; }

+/******************************************************************* + * Migration Protocol Version 3 + *******************************************************************/ + +static char * +chDomainMigrateBegin3(virDomainPtr domain, + const char *xmlin, + char **cookieout, + int *cookieoutlen, + unsigned long flags, + const char *dname, + unsigned long resource G_GNUC_UNUSED) +{ + virDomainObj *vm; + char *xml = NULL; + virCHDriver *driver = domain->conn->privateData; + + VIR_WARN("chDomainMigrateBegin3 %p %s %p %p %lu %s", + domain, xmlin, cookieout, cookieoutlen, flags, dname);

I don't see any plausible reason for VIR_WARN() here. We do not need to warn user about arguments. VIR_DEBUG() would be a better fit, but not really - the only caller of this function (virDomainMigrateBegin3()) already did that.

+ if (!(vm = virCHDomainObjFromDomain(domain))) + return NULL; + + if (virDomainMigrateBegin3EnsureACL(domain->conn, vm->def) < 0) { + virDomainObjEndAPI(&vm); + return NULL; + } + + // Copied from libxl_migration.c:386

We use C89 style of comments.

+ if (virDomainObjBeginJob(vm, VIR_JOB_MODIFY) < 0) + goto cleanup; + + xml = virDomainDefFormat(vm->def, driver->xmlopt, VIR_DOMAIN_DEF_FORMAT_SECURE); + + if (xml) { + VIR_WARN("chDomainMigrateBegin3 success. xml: %s", xml); + goto cleanup; + } + + return NULL; + cleanup: + virDomainObjEndAPI(&vm); + return xml; +} + +static +virDomainDef * +chMigrationAnyPrepareDef(virCHDriver *driver, + const char *dom_xml, + const char *dname) +{ + virDomainDef *def; + char *name = NULL; + + if (!dom_xml) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("no domain XML passed")); + return NULL; + } + + if (!(def = virDomainDefParseString(dom_xml, driver->xmlopt, + NULL, + VIR_DOMAIN_DEF_PARSE_INACTIVE))) + goto cleanup; + + if (dname) { + VIR_FREE(name);

I bet what you've meant to say here is: VIR_FREE(def->name); which renders 'name' variable unused.

+ def->name = g_strdup(dname); + } + + cleanup: + return def; +} + +typedef struct _chMigrationDstArgs { + unsigned int port; + virCHDomainObjPrivate *priv; +} chMigrationDstArgs; + +static void +chDoMigrateDstReceive(void *opaque) +{ + chMigrationDstArgs *args = opaque; + virCHDomainObjPrivate *priv = args->priv; + g_autofree char* rcv_uri = NULL; + + VIR_WARN("In thread. %u %p", args->port, args->priv); + if (!priv->monitor) { + VIR_ERROR(_("VMs monitor not initialized")); + } + + rcv_uri = g_strdup_printf("tcp:0.0.0.0:%d", args->port); + + if (virCHMonitorMigrationReceive(priv->monitor, rcv_uri) < 0) { + VIR_WARN("Migration receive failed."); + } + + VIR_WARN("Migration thread finished its duty"); +} + +/** + * Runs on the destination and prepares the empty cloud hypervisor process to + * receive the migration. + * Allocates some tcp port number to use as a migration channel. + */ +static int +chDomainMigratePrepare3(virConnectPtr dconn, + const char *cookiein, + int cookieinlen, + char **cookieout, + int *cookieoutlen, + const char *uri_in, + char **uri_out, + unsigned long flags, + const char *dname, + unsigned long resource G_GNUC_UNUSED, + const char *dom_xml) +{ + virCHDriver *driver = dconn->privateData; + virDomainObj *vm = NULL; + virCHDomainObjPrivate *priv = NULL; + chMigrationDstArgs *args = g_new0(chMigrationDstArgs, 1);

This is allocated but never freed.

+ unsigned short port = 0; + g_autofree char *hostname = NULL; + const char *threadname = "mig-ch"; + g_autoptr(virDomainDef) def = NULL; + int rc = 0; + const char *incFormat = "%s:%s:%d"; // seems to differ for AF_INET6 + + VIR_WARN("chDomainMigratePrepare3 %p %s %u %p %p %s %p %lu %s %s", + dconn, cookiein, cookieinlen, cookieout, cookieoutlen, uri_in, uri_out, flags, dname, dom_xml); + + if (virDomainMigratePrepare3EnsureACL(dconn, def) < 0) + return -1; + + if (!(def = chMigrationAnyPrepareDef(driver, dom_xml, dname))) + return -1; + + VIR_WARN("Got DomainDef prepared successfully"); + + if (virPortAllocatorAcquire(driver->migrationPorts, &port) < 0) { + rc = -1; + goto cleanup; + } + VIR_WARN("Got port %i", port); + + if ((hostname = virGetHostname()) == NULL) { + rc = -1; + goto cleanup; + } + + *uri_out = g_strdup_printf(incFormat, "tcp", hostname, port); + VIR_WARN("uri out %s", *uri_out); + + if (!(vm = virDomainObjListAdd(driver->domains, &def, + driver->xmlopt, + VIR_DOMAIN_OBJ_LIST_ADD_LIVE | + VIR_DOMAIN_OBJ_LIST_ADD_CHECK_LIVE, + NULL))) + { + rc = -1; + VIR_WARN("Could not add Domain Obj to List"); + goto cleanup; + } + + if (virCHProcessInit(driver, vm) < 0) { + rc = -1; + VIR_WARN("Could not init process"); + goto cleanup; + } + + VIR_WARN("Try creating migration thread"); + priv = vm->privateData; + args->port = port; + args->priv = priv; + + // VM receiving is blocking which we cannot do here, because it would block + // the Libvirt migration protocol. + // Prepare a thread to receive the migration data + // VIR_FREE(priv->migrationDstReceiveThr);

This VIR_FREE() must be called in virCHDomainObjPrivateFree() at least.

+ priv->migrationDstReceiveThr = g_new0(virThread, 1); + if (virThreadCreateFull(priv->migrationDstReceiveThr, true, + chDoMigrateDstReceive, + threadname, + false, + args) < 0) { + virReportError(VIR_ERR_OPERATION_FAILED, "%s", + _("Failed to create thread for receiving migration data")); + goto cleanup; + } + + VIR_WARN("Finished creating migration thread"); + + + VIR_WARN("Fin migrationPrepare"); + + + cleanup: + virDomainObjEndAPI(&vm); + return rc; +}

This is the place where I acquired so much of changed lines that I stopped my review. Please see the diff at EOL which I think should be squashed in. It covers also other fixes that I haven't raised explicitly here.

+ +static int +chDomainMigratePerform3(virDomainPtr dom, + const char *xmlin, + const char *cookiein, + int cookieinlen, + char **cookieout, + int *cookieoutlen, + const char *dconnuri, + const char *uri, + unsigned long flags, + const char *dname, + unsigned long resource) +{ + size_t i; + virDomainObj *vm; + g_autoptr(virCHDriverConfig) cfg = virCHDriverGetConfig(dom->conn->privateData); + virCHDomainObjPrivate *priv = NULL; + g_autofree char *id = g_strdup_printf("bla"); + VIR_WARN("chDomainMigratePerform3 %p %s %s %u %p %p %s %s %lu %s %lu", + dom, xmlin, cookiein, cookieinlen, cookieout, cookieoutlen, dconnuri, uri, flags, dname, resource); + + if (!(vm = virCHDomainObjFromDomain(dom))) + return -1; + + priv = vm->privateData; + + if (!priv->monitor) { + VIR_ERROR(_("VMs monitor not initialized")); + goto cleanup; + } + + if (virDomainMigratePerform3EnsureACL(dom->conn, vm->def) < 0) { + goto cleanup; + } + + // Net device id hardcoded currently + // We need to remove network devices from the VM before live migration. + // Libvirt pre-allocates network devices and passes only the FD to CHV. CHV + // is not able to migrate those devices. + // See following CHV issue: https://github.com/cloud-hypervisor/cloud-hypervisor/issues/7054 + /* de-activate netdevs after stopping vm */ + ignore_value(virDomainInterfaceStopDevices(vm->def)); + for (i = 0; i < vm->def->nnets; i++) { + virDomainInterfaceDeleteDevice(vm->def, vm->def->nets[i], false, cfg->stateDir); + id = g_strdup_printf("net_%lu", i); + if (virCHMonitorRemoveDevice(priv->monitor, id) < 0) { + VIR_WARN("Could not remove net device. Continue to migrate regardless."); + } + } + + if (virCHMonitorMigrationSend(priv->monitor, uri) < 0) { + VIR_WARN("Migration send failed."); + goto cleanup; + } + + cleanup: + virDomainObjEndAPI(&vm); + return 0; +} + +static virDomainPtr +chDomainMigrateFinish3(virConnectPtr dconn, + const char *dname, + const char *cookiein, + int cookieinlen, + char **cookieout, + int *cookieoutlen, + const char *dconnuri G_GNUC_UNUSED, + const char *uri G_GNUC_UNUSED, + unsigned long flags, + int cancelled) +{ + virCHDriver *driver = dconn->privateData; + virDomainObj *vm = NULL; + virDomainPtr dom = NULL; + + VIR_WARN("chDomainMigrateFinish3 %p %s %s %d %p %p %lu %d", + dconn, dname, cookiein, cookieinlen, cookieout, cookieoutlen, flags, cancelled); + + vm = virDomainObjListFindByName(driver->domains, dname); + if (!vm) { + virReportError(VIR_ERR_NO_DOMAIN, + _("no domain with matching name '%1$s'"), dname); + return NULL; + } + + if (virDomainMigrateFinish3EnsureACL(dconn, vm->def) < 0) { + virDomainObjEndAPI(&vm); + return NULL; + } + if (!(dom = virGetDomain(dconn, vm->def->name, vm->def->uuid, vm->def->id))) { + virDomainObjEndAPI(&vm); + VIR_WARN("virGetDomain failed."); + return NULL; + + } + if (virCHProcessUpdateInfo(vm) < 0) { + VIR_WARN("Could not update console info. Consider that non-fatal."); + } + + if (virCHProcessInitNetwork(driver, vm) < 0) { + VIR_WARN("Could not updatenetwork info. Consider that non-fatal."); + } + + virDomainObjSetState(vm, VIR_DOMAIN_RUNNING, VIR_DOMAIN_RUNNING_MIGRATED); + + virDomainObjEndAPI(&vm); + return dom; +} + +static int +chDomainMigrateConfirm3(virDomainPtr domain, + const char *cookiein, + int cookieinlen, + unsigned long flags, + int cancelled) +{ + virCHDriver *driver = domain->conn->privateData; + virObjectEvent *event = NULL; + virDomainObj *vm; + + VIR_WARN("chDomainMigrateConfirm3 %p %s %d %lu %d", + domain, cookiein, cookieinlen, flags, cancelled); + + if (!(vm = virCHDomainObjFromDomain(domain))) + return -1; + + if (virDomainMigrateConfirm3EnsureACL(domain->conn, vm->def) < 0) { + virDomainObjEndAPI(&vm); + return -1; + } + + // Code from chDestroyFlags + virDomainObjSetState(vm, VIR_DOMAIN_SHUTOFF, VIR_DOMAIN_SHUTOFF_MIGRATED); + virCHProcessStop(driver, vm, VIR_DOMAIN_SHUTOFF_DESTROYED); + virCHDomainRemoveInactive(driver, vm); + virDomainObjEndAPI(&vm); + + virObjectEventStateQueue(driver->domainEventState, event); + return 0; +}

/* Function Tables */ static virHypervisorDriver chHypervisorDriver = { @@ -2400,6 +2755,11 @@ static virHypervisorDriver chHypervisorDriver = { .connectDomainEventRegisterAny = chConnectDomainEventRegisterAny, /* 10.10.0 */ .connectDomainEventDeregisterAny = chConnectDomainEventDeregisterAny, /* 10.10.0 */ .domainInterfaceAddresses = chDomainInterfaceAddresses, /* 11.0.0 */ + .domainMigrateBegin3 = chDomainMigrateBegin3, /* 11.4.0 */ + .domainMigratePrepare3 = chDomainMigratePrepare3, /* 11.4.0 */ + .domainMigratePerform3 = chDomainMigratePerform3, /* 11.4.0 */ + .domainMigrateFinish3 = chDomainMigrateFinish3, /* 11.4.0 */ + .domainMigrateConfirm3 = chDomainMigrateConfirm3, /* 11.4.0 */ };

static virConnectDriver chConnectDriver = { diff --git a/src/ch/ch_monitor.c b/src/ch/ch_monitor.c index 3d3b4cb87d..595fa30be0 100644 --- a/src/ch/ch_monitor.c +++ b/src/ch/ch_monitor.c @@ -548,6 +548,7 @@ virCHMonitorBuildVMJson(virCHDriver *driver, virDomainDef *vmdef, if (!(*jsonstr = virJSONValueToString(content, false))) return -1;

+ VIR_WARN("Build VM JSON: \n %s \n", *jsonstr); return 0; }

@@ -684,6 +685,8 @@ virCHMonitorNew(virDomainObj *vm, virCHDriverConfig *cfg, int logfile) return NULL; }

+ VIR_WARN("Start emulator with cmd: %s", vm->def->emulator); + cmd = virCommandNew(vm->def->emulator); virCommandSetOutputFD(cmd, &logfile); virCommandSetErrorFD(cmd, &logfile); @@ -1163,6 +1166,159 @@ virCHMonitorSaveVM(virCHMonitor *mon, return ret; }

+int virCHMonitorRemoveDevice(virCHMonitor *mon, + const char* device_id) +{ + g_autofree char *url = NULL; + int responseCode = 0; + int ret = -1; + g_autofree char *payload = NULL; + struct curl_slist *headers = NULL; + struct curl_data data = {0}; + + url = g_strdup_printf("%s/%s", URL_ROOT, URL_VM_REMOVE_DEVICE); + + headers = curl_slist_append(headers, "Accept: application/json"); + headers = curl_slist_append(headers, "Content-Type: application/json"); + + if (virCHMonitorBuildKeyValueStringJson(&payload, "id", device_id) != 0) + return -1; + + VIR_WARN("Remove device id %s json %s", device_id, payload); + + VIR_WITH_OBJECT_LOCK_GUARD(mon) { + /* reset all options of a libcurl session handle at first */ + curl_easy_reset(mon->handle); + + curl_easy_setopt(mon->handle, CURLOPT_UNIX_SOCKET_PATH, mon->socketpath); + curl_easy_setopt(mon->handle, CURLOPT_URL, url); + curl_easy_setopt(mon->handle, CURLOPT_CUSTOMREQUEST, "PUT"); + curl_easy_setopt(mon->handle, CURLOPT_HTTPHEADER, headers); + curl_easy_setopt(mon->handle, CURLOPT_POSTFIELDS, payload); + curl_easy_setopt(mon->handle, CURLOPT_WRITEFUNCTION, curl_callback); + curl_easy_setopt(mon->handle, CURLOPT_WRITEDATA, (void *)&data); + + responseCode = virCHMonitorCurlPerform(mon->handle); + } + + if (responseCode == 200 || responseCode == 204) { + ret = 0; + } else { + data.content = g_realloc(data.content, data.size + 1); + data.content[data.size] = 0; + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + data.content); + g_free(data.content); + } + + /* reset the libcurl handle to avoid leaking a stack pointer to data */ + curl_easy_reset(mon->handle); + curl_slist_free_all(headers); + return ret; +} + +int virCHMonitorMigrationSend(virCHMonitor *mon, + const char *dst_uri) +{ + g_autofree char *url = NULL; + int responseCode = 0; + int ret = -1; + g_autofree char *payload = NULL; + struct curl_slist *headers = NULL; + struct curl_data data = {0}; + + url = g_strdup_printf("%s/%s", URL_ROOT, URL_VM_SEND_MIGRATION); + + headers = curl_slist_append(headers, "Accept: application/json"); + headers = curl_slist_append(headers, "Content-Type: application/json"); + + if (virCHMonitorBuildKeyValueStringJson(&payload, "destination_url", dst_uri) != 0) + return -1; + + VIR_WARN("Send VM to url %s json %s", dst_uri, payload); + + VIR_WITH_OBJECT_LOCK_GUARD(mon) { + /* reset all options of a libcurl session handle at first */ + curl_easy_reset(mon->handle); + + curl_easy_setopt(mon->handle, CURLOPT_UNIX_SOCKET_PATH, mon->socketpath); + curl_easy_setopt(mon->handle, CURLOPT_URL, url); + curl_easy_setopt(mon->handle, CURLOPT_CUSTOMREQUEST, "PUT"); + curl_easy_setopt(mon->handle, CURLOPT_HTTPHEADER, headers); + curl_easy_setopt(mon->handle, CURLOPT_POSTFIELDS, payload); + curl_easy_setopt(mon->handle, CURLOPT_WRITEFUNCTION, curl_callback); + curl_easy_setopt(mon->handle, CURLOPT_WRITEDATA, (void *)&data); + + responseCode = virCHMonitorCurlPerform(mon->handle); + } + + if (responseCode == 200 || responseCode == 204) { + ret = 0; + } else { + data.content = g_realloc(data.content, data.size + 1); + data.content[data.size] = 0; + virReportError(VIR_ERR_INTERNAL_ERROR, _("Error sending VM: '%1$s'"), + data.content); + g_free(data.content); + } + + /* reset the libcurl handle to avoid leaking a stack pointer to data */ + curl_easy_reset(mon->handle); + curl_slist_free_all(headers); + return ret; +} + +int virCHMonitorMigrationReceive(virCHMonitor *mon, + const char *rcv_uri) +{ + g_autofree char *url = NULL; + int responseCode = 0; + int ret = -1; + g_autofree char *payload = NULL; + struct curl_slist *headers = NULL; + struct curl_data data = {0}; + + url = g_strdup_printf("%s/%s", URL_ROOT, URL_VM_RECEIVE_MIGRATION); + + headers = curl_slist_append(headers, "Accept: application/json"); + headers = curl_slist_append(headers, "Content-Type: application/json"); + + if (virCHMonitorBuildKeyValueStringJson(&payload, "receiver_url", rcv_uri) != 0) + return -1; + + VIR_WARN("Receive VM from url %s json: %s", rcv_uri, payload); + + VIR_WITH_OBJECT_LOCK_GUARD(mon) { + /* reset all options of a libcurl session handle at first */ + curl_easy_reset(mon->handle); + + curl_easy_setopt(mon->handle, CURLOPT_UNIX_SOCKET_PATH, mon->socketpath); + curl_easy_setopt(mon->handle, CURLOPT_URL, url); + curl_easy_setopt(mon->handle, CURLOPT_CUSTOMREQUEST, "PUT"); + curl_easy_setopt(mon->handle, CURLOPT_HTTPHEADER, headers); + curl_easy_setopt(mon->handle, CURLOPT_POSTFIELDS, payload); + curl_easy_setopt(mon->handle, CURLOPT_WRITEFUNCTION, curl_callback); + curl_easy_setopt(mon->handle, CURLOPT_WRITEDATA, (void *)&data); + + responseCode = virCHMonitorCurlPerform(mon->handle); + } + + if (responseCode == 200 || responseCode == 204) { + ret = 0; + } else { + data.content = g_realloc(data.content, data.size + 1); + data.content[data.size] = 0; + virReportError(VIR_ERR_INTERNAL_ERROR, _("Error receiving VM: '%1$s'"), + data.content); + g_free(data.content); + } + + /* reset the libcurl handle to avoid leaking a stack pointer to data */ + curl_easy_reset(mon->handle); + curl_slist_free_all(headers); + return ret; +} + int virCHMonitorBuildRestoreJson(virDomainDef *vmdef, const char *from, diff --git a/src/ch/ch_monitor.h b/src/ch/ch_monitor.h index ffac9e938e..256ab0d8a1 100644 --- a/src/ch/ch_monitor.h +++ b/src/ch/ch_monitor.h @@ -40,6 +40,9 @@ #define URL_VM_INFO "vm.info" #define URL_VM_SAVE "vm.snapshot" #define URL_VM_RESTORE "vm.restore" +#define URL_VM_RECEIVE_MIGRATION "vm.receive-migration" +#define URL_VM_SEND_MIGRATION "vm.send-migration" +#define URL_VM_REMOVE_DEVICE "vm.remove-device"

#define VIRCH_THREAD_NAME_LEN 16

@@ -128,6 +131,11 @@ int virCHMonitorSuspendVM(virCHMonitor *mon); int virCHMonitorResumeVM(virCHMonitor *mon); int virCHMonitorSaveVM(virCHMonitor *mon, const char *to); +int virCHMonitorMigrationSend(virCHMonitor *mon, + const char *dst_uri); +int virCHMonitorMigrationReceive(virCHMonitor *mon, + const char *rcv_uri); +int virCHMonitorRemoveDevice(virCHMonitor *mon, const char* device_id); int virCHMonitorGetInfo(virCHMonitor *mon, virJSONValue **info);

size_t virCHMonitorGetThreadInfo(virCHMonitor *mon, bool refresh, diff --git a/src/ch/ch_process.c b/src/ch/ch_process.c index 95c808cb41..4d2bbc2b3d 100644 --- a/src/ch/ch_process.c +++ b/src/ch/ch_process.c @@ -135,6 +135,7 @@ int virCHProcessUpdateInfo(virDomainObj *vm) { g_autoptr(virJSONValue) info = NULL; + virCHDomainObjPrivate *priv = vm->privateData; if (virCHMonitorGetInfo(priv->monitor, &info) < 0) return -1; @@ -643,7 +644,7 @@ chProcessAddNetworkDevices(virCHDriver *driver, int **nicindexes, size_t *nnicindexes) { - size_t i; + size_t i, j; VIR_AUTOCLOSE mon_sockfd = -1; g_auto(virBuffer) buf = VIR_BUFFER_INITIALIZER; g_auto(virBuffer) http_headers = VIR_BUFFER_INITIALIZER; @@ -654,8 +655,10 @@ chProcessAddNetworkDevices(virCHDriver *driver, return -1; }

- if ((mon_sockfd = chMonitorSocketConnect(mon)) < 0) + if ((mon_sockfd = chMonitorSocketConnect(mon)) < 0) { + VIR_WARN("chProcessAddNetworkDevices failed"); return -1; + }

virBufferAddLit(&http_headers, "PUT /api/v1/vm.add-net HTTP/1.1\r\n"); virBufferAddLit(&http_headers, "Host: localhost\r\n"); @@ -681,24 +684,33 @@ chProcessAddNetworkDevices(virCHDriver *driver, if (virCHDomainValidateActualNetDef(vmdef->nets[i]) < 0) { virReportError(VIR_ERR_INTERNAL_ERROR, "%s", _("net definition failed validation")); + VIR_WARN("virCHDomainValidateActualNetDef failed."); return -1; }

tapfds = g_new0(int, tapfd_len); memset(tapfds, -1, (tapfd_len) * sizeof(int));

+ VIR_WARN("net type: %u", vmdef->nets[i]->type); /* Connect Guest interfaces */ if (virCHConnetNetworkInterfaces(driver, vmdef, vmdef->nets[i], tapfds, - nicindexes, nnicindexes) < 0) + nicindexes, nnicindexes) < 0) { + VIR_WARN("chProcessAddNetworkDevices failed."); return -1; + }

if (virCHMonitorBuildNetJson(vmdef->nets[i], i, &payload) < 0) { virReportError(VIR_ERR_INTERNAL_ERROR, "%s", _("Failed to build net json")); + VIR_WARN("virCHMonitorBuildNetJson failed."); return -1; }

- VIR_DEBUG("payload sent with net-add request to CH = %s", payload); + VIR_WARN("payload sent with net-add request to CH = %s", payload); + + for (j = 0; j < tapfd_len; j++) { + VIR_WARN("tapfd %lu : %d", j, tapfds[j]); + }

virBufferAsprintf(&buf, "%s", virBufferCurrentContent(&http_headers)); virBufferAsprintf(&buf, "Content-Length: %zu\r\n\r\n", strlen(payload)); @@ -892,6 +904,122 @@ virCHProcessPrepareDomain(virDomainObj *vm) return 0; }

+int virCHProcessInitNetwork(virCHDriver *driver, + virDomainObj *vm) +{ + int ret = -1; + virCHDomainObjPrivate *priv = vm->privateData; + g_autoptr(virCHDriverConfig) cfg = virCHDriverGetConfig(priv->driver); + g_autofree int *nicindexes = NULL; + size_t nnicindexes = 0; + + if (chProcessAddNetworkDevices(driver, priv->monitor, vm->def, + &nicindexes, &nnicindexes) < 0) { + VIR_WARN("Failed chProcessAddNetworkDevices"); + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("Failed while adding guest interfaces")); + goto cleanup; + } + + /* Bring up netdevs before starting CPUs */ + if (virDomainInterfaceStartDevices(vm->def) < 0) { + VIR_WARN("Failed virDomainInterfaceStartDevices"); + return -1; + } + + return 0; + + cleanup: + + return ret; +} + +/** + * A variant of virCHProcessStart that does not start the vCPU threads and the + * VM. Sets up the CH process along most configuration. + * Is used to setup CH in order to receive a live migration afterwards. + */ +int +virCHProcessInit(virCHDriver *driver, + virDomainObj *vm) +{

This function copied parts of virCHProcessStart. If there's a thing that needs to be changed then we have two places to change. I think virCHProcessStart() should call virCHProcessInit(). You can invent an additional argument to do hacks around incoming migration ().

+ int ret = -1; + virCHDomainObjPrivate *priv = vm->privateData; + g_autoptr(virCHDriverConfig) cfg = virCHDriverGetConfig(priv->driver); + g_autofree int *nicindexes = NULL; + size_t nnicindexes = 0; + g_autoptr(domainLogContext) logCtxt = NULL; + int logfile = -1; + + if (virDomainObjIsActive(vm)) { + virReportError(VIR_ERR_OPERATION_INVALID, "%s", + _("VM is already active")); + return -1; + } + + if (virCHProcessStartValidate(driver, vm) < 0) { + return -1; + } + + VIR_WARN("Creating domain log file for %s domain", vm->def->name); + if (!(logCtxt = domainLogContextNew(cfg->stdioLogD, cfg->logDir, + CH_DRIVER_NAME, + vm, driver->privileged, + vm->def->name))) { + virLastErrorPrefixMessage("%s", _("can't connect to virtlogd")); + return -1; + } + logfile = domainLogContextGetWriteFD(logCtxt); + + if (virCHProcessPrepareDomain(vm) < 0) { + return -1; + } + + if (virCHProcessPrepareHost(driver, vm) < 0) + return -1; + + if (!priv->monitor) { + /* And we can get the first monitor connection now too */ + if (!(priv->monitor = virCHProcessConnectMonitor(driver, vm, logfile))) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("failed to create connection to CH socket")); + goto cleanup; + } + + if (virCHMonitorCreateVM(driver, priv->monitor) < 0) { + VIR_WARN("Failed virCHMonitorCreateVM"); + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("failed to create guest VM")); + goto cleanup; + } + } + + vm->def->id = vm->pid; + priv->machineName = virCHDomainGetMachineName(vm); + + if (virDomainCgroupSetupCgroup("ch", vm, + nnicindexes, nicindexes, + &priv->cgroup, + cfg->cgroupControllers, + 0, /*maxThreadsPerProc*/ + priv->driver->privileged, + priv->machineName) < 0) + { + VIR_WARN("Failed virDomainCgroupSetupCgroup"); + goto cleanup; + } + + virDomainObjSetState(vm, VIR_DOMAIN_PAUSED, VIR_DOMAIN_PAUSED_MIGRATION); + + return 0; + + cleanup: + if (ret) + virCHProcessStop(driver, vm, VIR_DOMAIN_SHUTOFF_FAILED); + + return ret; +} + /** * virCHProcessStart: * @driver: pointer to driver structure diff --git a/src/ch/ch_process.h b/src/ch/ch_process.h index 7a6995b7cf..3a5a2d17d7 100644 --- a/src/ch/ch_process.h +++ b/src/ch/ch_process.h @@ -23,6 +23,9 @@ #include "ch_conf.h" #include "internal.h"

+int virCHProcessInit(virCHDriver *driver, + virDomainObj *vm); + int virCHProcessStart(virCHDriver *driver, virDomainObj *vm, virDomainRunningReason reason); @@ -38,3 +41,6 @@ int virCHProcessStartRestore(virCHDriver *driver, const char *from);

int virCHProcessUpdateInfo(virDomainObj *vm); + +int virCHProcessInitNetwork(virCHDriver *driver, + virDomainObj *vm); diff --git a/src/hypervisor/domain_interface.c b/src/hypervisor/domain_interface.c index 5bc698d272..a13fcfb7d2 100644 --- a/src/hypervisor/domain_interface.c +++ b/src/hypervisor/domain_interface.c @@ -82,6 +82,7 @@ virDomainInterfaceEthernetConnect(virDomainDef *def, bool template_ifname = false; const char *tunpath = "/dev/net/tun"; const char *auditdev = tunpath; + VIR_WARN("virDomainInterfaceEthernetConnect %s", net->ifname);

if (net->backend.tap) { tunpath = net->backend.tap; diff --git a/src/libvirt-domain.c b/src/libvirt-domain.c index 93e8f5b853..867876b23f 100644 --- a/src/libvirt-domain.c +++ b/src/libvirt-domain.c @@ -3307,6 +3307,11 @@ virDomainMigrateVersion3Full(virDomainPtr domain, "params=%p, nparams=%d, useParams=%d, flags=0x%x", dconn, NULLSTR(xmlin), NULLSTR(dname), NULLSTR(uri), bandwidth, params, nparams, useParams, flags); + VIR_WARN( + "dconn=%p, xmlin=%s, dname=%s, uri=%s, bandwidth=%llu, " + "params=%p, nparams=%d, useParams=%d, flags=0x%x", + dconn, NULLSTR(xmlin), NULLSTR(dname), NULLSTR(uri), + bandwidth, params, nparams, useParams, flags); VIR_TYPED_PARAMS_DEBUG(params, nparams);

virCheckNonEmptyOptStringArgReturn(dname, NULL); @@ -3337,7 +3342,7 @@ virDomainMigrateVersion3Full(virDomainPtr domain, if (ret) protection = VIR_MIGRATE_CHANGE_PROTECTION;

- VIR_DEBUG("Begin3 %p", domain->conn); + VIR_WARN("Begin3 %p", domain->conn); if (useParams) { dom_xml = domain->conn->driver->domainMigrateBegin3Params (domain, params, nparams, &cookieout, &cookieoutlen, @@ -3366,7 +3371,7 @@ virDomainMigrateVersion3Full(virDomainPtr domain, destflags = flags & ~(VIR_MIGRATE_ABORT_ON_ERROR | VIR_MIGRATE_AUTO_CONVERGE);

- VIR_DEBUG("Prepare3 %p flags=0x%x", dconn, destflags); + VIR_WARN("Prepare3 %p flags=0x%x", dconn, destflags); cookiein = g_steal_pointer(&cookieout); cookieinlen = cookieoutlen; cookieoutlen = 0; @@ -3427,7 +3432,7 @@ virDomainMigrateVersion3Full(virDomainPtr domain, * running, but in paused state until the destination can * confirm migration completion. */ - VIR_DEBUG("Perform3 %p uri=%s", domain->conn, uri); + VIR_WARN("Perform3 %p uri=%s", domain->conn, uri); VIR_FREE(cookiein); cookiein = g_steal_pointer(&cookieout); cookieinlen = cookieoutlen; @@ -3465,7 +3470,7 @@ virDomainMigrateVersion3Full(virDomainPtr domain, * send all migration data. Returns NULL for ddomain if * the dest was unable to complete migration. */ - VIR_DEBUG("Finish3 %p ret=%d", dconn, ret); + VIR_WARN("Finish3 %p ret=%d", dconn, ret); VIR_FREE(cookiein); cookiein = g_steal_pointer(&cookieout); cookieinlen = cookieoutlen; @@ -3540,7 +3545,7 @@ virDomainMigrateVersion3Full(virDomainPtr domain, * cancelled there. */ if (notify_source) { - VIR_DEBUG("Confirm3 %p ret=%d domain=%p", domain->conn, ret, domain); + VIR_WARN("Confirm3 %p ret=%d domain=%p", domain->conn, ret, domain); VIR_FREE(cookiein); cookiein = g_steal_pointer(&cookieout); cookieinlen = cookieoutlen;

diff --git c/src/ch/ch_conf.h w/src/ch/ch_conf.h index 40d639eb2a..042507fcc6 100644 --- c/src/ch/ch_conf.h +++ w/src/ch/ch_conf.h @@ -31,6 +31,9 @@ #define CH_DRIVER_NAME "CH" #define CH_CMD "cloud-hypervisor" +#define CH_MIGRATION_PORT_MIN 49152 +#define CH_MIGRATION_PORT_MAX 49216 + typedef struct _virCHDriver virCHDriver; typedef struct _virCHDriverConfig virCHDriverConfig; diff --git c/src/ch/ch_domain.c w/src/ch/ch_domain.c index 7231fdc49f..c15410e474 100644 --- c/src/ch/ch_domain.c +++ w/src/ch/ch_domain.c @@ -68,6 +68,11 @@ virCHDomainObjPrivateFree(void *data) virBitmapFree(priv->autoCpuset); virBitmapFree(priv->autoNodeset); virCgroupFree(priv->cgroup); + if (priv->migrationDstReceiveThr) { + virThreadCancel(priv->migrationDstReceiveThr); + virThreadJoin(priv->migrationDstReceiveThr); + g_free(priv->migrationDstReceiveThr); + } g_free(priv->pidfile); g_free(priv); } diff --git c/src/ch/ch_driver.c w/src/ch/ch_driver.c index c53607251b..a0df931c80 100644 --- c/src/ch/ch_driver.c +++ w/src/ch/ch_driver.c @@ -1458,11 +1458,10 @@ chStateInitialize(bool privileged, if (!(ch_driver->domainEventState = virObjectEventStateNew())) goto cleanup; - /* Allocate bitmap for migration port reservation */ - if (!(ch_driver->migrationPorts = - virPortAllocatorRangeNew(_("migration"), - 49152, - 49216))) + /* Allocate bitmap for migration port reservation */ + if (!(ch_driver->migrationPorts = virPortAllocatorRangeNew(_("migration"), + CH_MIGRATION_PORT_MIN, + CH_MIGRATION_PORT_MAX))) goto cleanup; if ((rv = chExtractVersion(ch_driver)) < 0) { @@ -2358,51 +2357,52 @@ chDomainInterfaceAddresses(virDomain *dom, static char * chDomainMigrateBegin3(virDomainPtr domain, const char *xmlin, - char **cookieout, - int *cookieoutlen, + char **cookieout G_GNUC_UNUSED, + int *cookieoutlen G_GNUC_UNUSED, unsigned long flags, - const char *dname, - unsigned long resource G_GNUC_UNUSED) + const char *dname G_GNUC_UNUSED, + unsigned long bandwidth) { + virCHDriver *driver = domain->conn->privateData; virDomainObj *vm; char *xml = NULL; - virCHDriver *driver = domain->conn->privateData; - VIR_WARN("chDomainMigrateBegin3 %p %s %p %p %lu %s", - domain, xmlin, cookieout, cookieoutlen, flags, dname); + virCheckFlags(0, NULL); + + if (xmlin) { + virReportError(VIR_ERR_ARGUMENT_UNSUPPORTED, "%s", + _("xml modification unsupported")); + return NULL; + } + + if (bandwidth != 0) { + virReportError(VIR_ERR_ARGUMENT_UNSUPPORTED, "%s", + _("migration bandwidth unsupported")); + return NULL; + } + if (!(vm = virCHDomainObjFromDomain(domain))) return NULL; - if (virDomainMigrateBegin3EnsureACL(domain->conn, vm->def) < 0) { - virDomainObjEndAPI(&vm); - return NULL; - } + if (virDomainMigrateBegin3EnsureACL(domain->conn, vm->def) < 0) + goto cleanup; - // Copied from libxl_migration.c:386 if (virDomainObjBeginJob(vm, VIR_JOB_MODIFY) < 0) goto cleanup; xml = virDomainDefFormat(vm->def, driver->xmlopt, VIR_DOMAIN_DEF_FORMAT_SECURE); - if (xml) { - VIR_WARN("chDomainMigrateBegin3 success. xml: %s", xml); - goto cleanup; - } - - return NULL; cleanup: virDomainObjEndAPI(&vm); return xml; } -static -virDomainDef * +static virDomainDef * chMigrationAnyPrepareDef(virCHDriver *driver, - const char *dom_xml, - const char *dname) + const char *dom_xml, + const char *dname) { virDomainDef *def; - char *name = NULL; if (!dom_xml) { virReportError(VIR_ERR_INTERNAL_ERROR, "%s", @@ -2412,15 +2412,15 @@ chMigrationAnyPrepareDef(virCHDriver *driver, if (!(def = virDomainDefParseString(dom_xml, driver->xmlopt, NULL, - VIR_DOMAIN_DEF_PARSE_INACTIVE))) - goto cleanup; + VIR_DOMAIN_DEF_PARSE_INACTIVE))) { + return NULL; + } if (dname) { - VIR_FREE(name); + VIR_FREE(def->name); def->name = g_strdup(dname); } - cleanup: return def; } @@ -2432,13 +2432,14 @@ typedef struct _chMigrationDstArgs { static void chDoMigrateDstReceive(void *opaque) { - chMigrationDstArgs *args = opaque; + g_autofree chMigrationDstArgs *args = opaque; virCHDomainObjPrivate *priv = args->priv; g_autofree char* rcv_uri = NULL; - VIR_WARN("In thread. %u %p", args->port, args->priv); + VIR_DEBUG("In thread. %u %p", args->port, args->priv); if (!priv->monitor) { VIR_ERROR(_("VMs monitor not initialized")); + return; } rcv_uri = g_strdup_printf("tcp:0.0.0.0:%d", args->port); @@ -2447,7 +2448,7 @@ chDoMigrateDstReceive(void *opaque) VIR_WARN("Migration receive failed."); } - VIR_WARN("Migration thread finished its duty"); + VIR_DEBUG("Migration thread finished its duty"); } /** @@ -2457,30 +2458,27 @@ chDoMigrateDstReceive(void *opaque) */ static int chDomainMigratePrepare3(virConnectPtr dconn, - const char *cookiein, - int cookieinlen, - char **cookieout, - int *cookieoutlen, - const char *uri_in, + const char *cookiein G_GNUC_UNUSED, + int cookieinlen G_GNUC_UNUSED, + char **cookieout G_GNUC_UNUSED, + int *cookieoutlen G_GNUC_UNUSED, + const char *uri_in G_GNUC_UNUSED, char **uri_out, unsigned long flags, const char *dname, - unsigned long resource G_GNUC_UNUSED, + unsigned long bandwidth G_GNUC_UNUSED, const char *dom_xml) { virCHDriver *driver = dconn->privateData; virDomainObj *vm = NULL; virCHDomainObjPrivate *priv = NULL; - chMigrationDstArgs *args = g_new0(chMigrationDstArgs, 1); + g_autofree chMigrationDstArgs *args = g_new0(chMigrationDstArgs, 1); unsigned short port = 0; g_autofree char *hostname = NULL; - const char *threadname = "mig-ch"; g_autoptr(virDomainDef) def = NULL; - int rc = 0; - const char *incFormat = "%s:%s:%d"; // seems to differ for AF_INET6 + int ret = -1; - VIR_WARN("chDomainMigratePrepare3 %p %s %u %p %p %s %p %lu %s %s", - dconn, cookiein, cookieinlen, cookieout, cookieoutlen, uri_in, uri_out, flags, dname, dom_xml); + virCheckFlags(0, -1); if (virDomainMigratePrepare3EnsureACL(dconn, def) < 0) return -1; @@ -2488,68 +2486,55 @@ chDomainMigratePrepare3(virConnectPtr dconn, if (!(def = chMigrationAnyPrepareDef(driver, dom_xml, dname))) return -1; - VIR_WARN("Got DomainDef prepared successfully"); + if (virPortAllocatorAcquire(driver->migrationPorts, &port) < 0) + return -1; - if (virPortAllocatorAcquire(driver->migrationPorts, &port) < 0) { - rc = -1; - goto cleanup; - } - VIR_WARN("Got port %i", port); + if (!(hostname = virGetHostname())) + return -1; - if ((hostname = virGetHostname()) == NULL) { - rc = -1; - goto cleanup; - } - - *uri_out = g_strdup_printf(incFormat, "tcp", hostname, port); - VIR_WARN("uri out %s", *uri_out); + /* Seems to differ for AF_INET6 */ + *uri_out = g_strdup_printf("tcp:%s:%d", hostname, port); + VIR_DEBUG("uri out %s", *uri_out); if (!(vm = virDomainObjListAdd(driver->domains, &def, driver->xmlopt, VIR_DOMAIN_OBJ_LIST_ADD_LIVE | VIR_DOMAIN_OBJ_LIST_ADD_CHECK_LIVE, - NULL))) - { - rc = -1; - VIR_WARN("Could not add Domain Obj to List"); + NULL))) { goto cleanup; } - if (virCHProcessInit(driver, vm) < 0) { - rc = -1; - VIR_WARN("Could not init process"); + if (virCHProcessInit(driver, vm) < 0) goto cleanup; - } - VIR_WARN("Try creating migration thread"); priv = vm->privateData; args->port = port; args->priv = priv; - // VM receiving is blocking which we cannot do here, because it would block - // the Libvirt migration protocol. - // Prepare a thread to receive the migration data - // VIR_FREE(priv->migrationDstReceiveThr); + /* VM receiving is blocking which we cannot do here, because it would block + * the Libvirt migration protocol. Prepare a thread to receive the + * migration data. */ priv->migrationDstReceiveThr = g_new0(virThread, 1); - if (virThreadCreateFull(priv->migrationDstReceiveThr, true, + VIR_DEBUG("Creating migration thread"); + if (virThreadCreateFull(priv->migrationDstReceiveThr, + true, chDoMigrateDstReceive, - threadname, + "mig-ch", false, args) < 0) { virReportError(VIR_ERR_OPERATION_FAILED, "%s", _("Failed to create thread for receiving migration data")); goto cleanup; } + VIR_DEBUG("Finished creating migration thread"); - VIR_WARN("Finished creating migration thread"); - - - VIR_WARN("Fin migrationPrepare"); - + /* The thread owns the @args now. */ + args = NULL; + ret = 0; cleanup: virDomainObjEndAPI(&vm); - return rc; + return ret; } static int diff --git c/src/ch/ch_monitor.c w/src/ch/ch_monitor.c index 595fa30be0..a6fa37e9f8 100644 --- c/src/ch/ch_monitor.c +++ w/src/ch/ch_monitor.c @@ -1286,7 +1286,7 @@ int virCHMonitorMigrationReceive(virCHMonitor *mon, if (virCHMonitorBuildKeyValueStringJson(&payload, "receiver_url", rcv_uri) != 0) return -1; - VIR_WARN("Receive VM from url %s json: %s", rcv_uri, payload); + VIR_DEBUG("Receive VM from url %s json: %s", rcv_uri, payload); VIR_WITH_OBJECT_LOCK_GUARD(mon) { /* reset all options of a libcurl session handle at first */ diff --git c/src/hypervisor/domain_interface.c w/src/hypervisor/domain_interface.c index a13fcfb7d2..5bc698d272 100644 --- c/src/hypervisor/domain_interface.c +++ w/src/hypervisor/domain_interface.c @@ -82,7 +82,6 @@ virDomainInterfaceEthernetConnect(virDomainDef *def, bool template_ifname = false; const char *tunpath = "/dev/net/tun"; const char *auditdev = tunpath; - VIR_WARN("virDomainInterfaceEthernetConnect %s", net->ifname); if (net->backend.tap) { tunpath = net->backend.tap; diff --git c/src/libvirt-domain.c w/src/libvirt-domain.c index 28347947d1..86ede6b038 100644 --- c/src/libvirt-domain.c +++ w/src/libvirt-domain.c @@ -3307,11 +3307,6 @@ virDomainMigrateVersion3Full(virDomainPtr domain, "params=%p, nparams=%d, useParams=%d, flags=0x%x", dconn, NULLSTR(xmlin), NULLSTR(dname), NULLSTR(uri), bandwidth, params, nparams, useParams, flags); - VIR_WARN( - "dconn=%p, xmlin=%s, dname=%s, uri=%s, bandwidth=%llu, " - "params=%p, nparams=%d, useParams=%d, flags=0x%x", - dconn, NULLSTR(xmlin), NULLSTR(dname), NULLSTR(uri), - bandwidth, params, nparams, useParams, flags); VIR_TYPED_PARAMS_DEBUG(params, nparams); virCheckNonEmptyOptStringArgReturn(dname, NULL); Michal