[PATCH v5 2/8] qemu: Manage a pre-shared key's lifecycle

Libvirt by default attempt to use the TLS-PSK-enabled VM migration. The source host adds the pre-shared key in the migration cookie. Upon parsing the migration cookie, the destination host informs the source host whether to use TLS X.509 or TLS PSK during VM migration via the migration cookie. For a migration session, Libvirt generates a random key of the specified length, and then stores the content, "qemu:<random key>", at <tls_psk_state_dir>/$ID-$VMNAME/keys.psk on the source host. This is because QEMU's tls-creds-psk object does not accept a raw key string as a parameter, it only accepts a dir argument pointing to a directory from which it can read the key file. Subsequently, it sends the key to destination by embedding it within the migration cookie. The destination's Libvirt extracts the key from the migration cookie. Upon migration completion or any failure, both source and destination Libvirt must delete the directory containing the session's keys.psk. Signed-off-by: Abhisek Panda <abhisek.panda1@nutanix.com> --- src/qemu/qemu_conf.c | 4 ++ src/qemu/qemu_conf.h | 1 + src/qemu/qemu_domain.c | 1 + src/qemu/qemu_domain.h | 1 + src/qemu/qemu_driver.c | 6 ++ src/qemu/qemu_migration.c | 104 ++++++++++++++++++++++++++++- src/qemu/qemu_migration.h | 3 + src/qemu/qemu_migration_cookie.c | 67 ++++++++++++++++++- src/qemu/qemu_migration_cookie.h | 5 ++ src/qemu/qemu_process.c | 3 + tests/qemumigrationcookiexmltest.c | 12 +++- 11 files changed, 200 insertions(+), 7 deletions(-) diff --git a/src/qemu/qemu_conf.c b/src/qemu/qemu_conf.c index cca7776b80..f5c5437c89 100644 --- a/src/qemu/qemu_conf.c +++ b/src/qemu/qemu_conf.c @@ -168,6 +168,7 @@ virQEMUDriverConfig *virQEMUDriverConfigNew(bool privileged, cfg->cacheDir = g_strdup_printf("%s/cache/qemu", root); cfg->libDir = g_strdup_printf("%s/lib/qemu", root); cfg->swtpmStorageDir = g_strdup_printf("%s/lib/swtpm", root); + cfg->tlsPSKStateDir = g_strdup_printf("%s/run/psk", root); cfg->saveDir = g_strdup_printf("%s/save", cfg->libDir); cfg->snapshotDir = g_strdup_printf("%s/snapshot", cfg->libDir); @@ -187,6 +188,7 @@ virQEMUDriverConfig *virQEMUDriverConfigNew(bool privileged, cfg->stateDir = g_strdup_printf("%s/libvirt/qemu", RUNSTATEDIR); cfg->swtpmStateDir = g_strdup_printf("%s/swtpm", cfg->stateDir); cfg->channelTargetDir = g_strdup_printf("%s/channel", cfg->stateDir); + cfg->tlsPSKStateDir = g_strdup_printf("%s/psk", cfg->stateDir); cfg->cacheDir = g_strdup_printf("%s/cache/libvirt/qemu", LOCALSTATEDIR); @@ -214,6 +216,7 @@ virQEMUDriverConfig *virQEMUDriverConfigNew(bool privileged, cfg->stateDir = g_strdup_printf("%s/qemu/run", rundir); cfg->swtpmStateDir = g_strdup_printf("%s/swtpm", cfg->stateDir); cfg->channelTargetDir = g_strdup_printf("%s/channel", cfg->stateDir); + cfg->tlsPSKStateDir = g_strdup_printf("%s/psk", cfg->stateDir); cfg->configBaseDir = virGetUserConfigDirectory(); @@ -375,6 +378,7 @@ static void virQEMUDriverConfigDispose(void *obj) g_free(cfg->dbusStateDir); g_free(cfg->rdpStateDir); g_free(cfg->vncStateDir); + g_free(cfg->tlsPSKStateDir); g_free(cfg->libDir); g_free(cfg->cacheDir); diff --git a/src/qemu/qemu_conf.h b/src/qemu/qemu_conf.h index b700829342..2db3440514 100644 --- a/src/qemu/qemu_conf.h +++ b/src/qemu/qemu_conf.h @@ -113,6 +113,7 @@ struct _virQEMUDriverConfig { char *dbusStateDir; char *rdpStateDir; char *vncStateDir; + char *tlsPSKStateDir; /* These two directories are ones QEMU processes use (so must match * the QEMU user/group */ char *libDir; diff --git a/src/qemu/qemu_domain.c b/src/qemu/qemu_domain.c index a4e5f92840..f4d470553d 100644 --- a/src/qemu/qemu_domain.c +++ b/src/qemu/qemu_domain.c @@ -1990,6 +1990,7 @@ qemuDomainObjPrivateFree(void *data) virObjectUnref(priv->monConfig); g_free(priv->lockState); g_free(priv->origname); + g_free(priv->migTLSPSK); virChrdevFree(priv->devs); diff --git a/src/qemu/qemu_domain.h b/src/qemu/qemu_domain.h index 23e99dc68c..053140f7ec 100644 --- a/src/qemu/qemu_domain.h +++ b/src/qemu/qemu_domain.h @@ -139,6 +139,7 @@ struct _qemuDomainObjPrivate { char *origname; int nbdPort; /* Port used for migration with NBD */ unsigned short migrationPort; + char *migTLSPSK; /* Hex-encoded pre-shared key for TLS-PSK-enabled VM migration session */ unsigned short backupNBDPort; int preMigrationState; unsigned long long preMigrationMemlock; /* Original RLIMIT_MEMLOCK in case diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c index 5922065897..c95a1978cf 100644 --- a/src/qemu/qemu_driver.c +++ b/src/qemu/qemu_driver.c @@ -664,6 +664,12 @@ qemuStateInitialize(bool privileged, cfg->vncStateDir); goto error; } + if (virDirCreate(cfg->tlsPSKStateDir, 0711, 0, 0, + VIR_DIR_CREATE_ALLOW_EXIST) < 0) { + virReportSystemError(errno, _("Failed to create TLS PSK state dir %1$s"), + cfg->tlsPSKStateDir); + goto error; + } qemu_driver->inhibitor = virInhibitorNew( VIR_INHIBITOR_WHAT_SHUTDOWN, diff --git a/src/qemu/qemu_migration.c b/src/qemu/qemu_migration.c index 76661de09a..61d0cc5e00 100644 --- a/src/qemu/qemu_migration.c +++ b/src/qemu/qemu_migration.c @@ -59,6 +59,7 @@ #include "virprocess.h" #include "virdomainsnapshotobjlist.h" #include "virutil.h" +#include "virsecureerase.h" #define VIR_FROM_THIS VIR_FROM_QEMU @@ -1503,6 +1504,92 @@ qemuMigrationSrcIsAllowedHostdev(const virDomainDef *def) } +void +qemuMigrationDeletePSKDir(virQEMUDriver *driver, virDomainObj *vm) +{ + g_autoptr(virQEMUDriverConfig) cfg = virQEMUDriverGetConfig(driver); + qemuDomainObjPrivate *priv = vm->privateData; + g_autofree char *dir_path = NULL; + g_autofree char *shortName = NULL; + + if (priv->migTLSPSK) { + virSecureEraseString(priv->migTLSPSK); + g_clear_pointer(&priv->migTLSPSK, g_free); + } + + if (!(shortName = virDomainDefGetShortName(vm->def))) { + VIR_WARN("Could not get the shortened domain name for domain: %1$s", + vm->def->name); + return; + } + + dir_path = g_strdup_printf("%s/%s", cfg->tlsPSKStateDir, shortName); + + if (virFileIsDir(dir_path) && + virFileDeleteTree(dir_path) < 0) + VIR_WARN("Failed to delete the directory %s containing the pre-shared keys for migration of domain %s", + dir_path, vm->def->name); +} + + +static int +qemuPersistTLSPSKHelper(int pskFD, + const char *pskPath, + const void *opaque) +{ + const char *key = opaque; + + if (safewrite(pskFD, "qemu:", 5) < 0 || + safewrite(pskFD, key, strlen(key)) < 0) { + virReportSystemError(errno, + _("Unable to write the pre-shared key to file '%1$s'"), + pskPath); + return -1; + } + + return 0; +} + + +static int +qemuMigrationPersistPSK(virQEMUDriver *driver, virDomainObj *vm, const char *tlsPSK) +{ + g_autoptr(virQEMUDriverConfig) cfg = virQEMUDriverGetConfig(driver); + g_autofree char *dir_path = NULL; + g_autofree char *key_path = NULL; + g_autofree char *shortName = NULL; + + if (!(shortName = virDomainDefGetShortName(vm->def))) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("Could not get the shortened domain name for domain: %1$s"), + vm->def->name); + return -1; + } + + dir_path = g_strdup_printf("%s/%s", cfg->tlsPSKStateDir, shortName); + key_path = g_strdup_printf("%s/keys.psk", dir_path); + + if (virDirCreate(dir_path, 0711, 0, 0, + VIR_DIR_CREATE_ALLOW_EXIST) < 0) { + virReportSystemError(errno, + _("Could not create the directory %1$s for storing PSKs"), + dir_path); + goto error; + } + + if (virFileRewrite(key_path, S_IRUSR, cfg->user, + cfg->group, qemuPersistTLSPSKHelper, + tlsPSK) < 0) + goto error; + + return 0; + + error: + qemuMigrationDeletePSKDir(driver, vm); + return -1; +} + + static int qemuDomainGetMigrationBlockers(virDomainObj *vm, int asyncJob, @@ -2739,6 +2826,9 @@ qemuMigrationSrcBeginXML(virDomainObj *vm, cookieFlags) < 0) return NULL; + if (mig->tlsPSK && qemuMigrationPersistPSK(driver, vm, mig->tlsPSK) < 0) + return NULL; + if (xmlin) { g_autoptr(virDomainDef) def = NULL; @@ -2802,7 +2892,7 @@ qemuMigrationSrcBeginPhase(virQEMUDriver *driver, unsigned int flags) { qemuDomainObjPrivate *priv = vm->privateData; - unsigned int cookieFlags = QEMU_MIGRATION_COOKIE_LOCKSTATE; + unsigned int cookieFlags = QEMU_MIGRATION_COOKIE_LOCKSTATE | QEMU_MIGRATION_COOKIE_TLS_PSK; VIR_DEBUG("driver=%p, vm=%p, xmlin=%s, dname=%s," " cookieout=%p, cookieoutlen=%p," @@ -4233,6 +4323,8 @@ qemuMigrationSrcConfirmPhase(virQEMUDriver *driver, privJob->stats.mig.downtime = privMigJob->stats.mig.downtime; } + qemuMigrationDeletePSKDir(driver, vm); + if (flags & VIR_MIGRATE_OFFLINE) return 0; @@ -5281,6 +5373,7 @@ qemuMigrationSrcRun(virQEMUDriver *driver, error: virErrorPreserveLast(&orig_err); + qemuMigrationDeletePSKDir(driver, vm); if (qemuDomainObjIsActive(vm)) { int reason; @@ -6402,8 +6495,10 @@ qemuMigrationSrcPerformJob(virQEMUDriver *driver, } endjob: - if (ret < 0) + if (ret < 0) { virErrorPreserveLast(&orig_err); + qemuMigrationDeletePSKDir(driver, vm); + } if (virDomainObjIsFailedPostcopy(vm, vm->job)) { ignore_value(qemuMigrationJobSetPhase(vm, QEMU_MIGRATION_PHASE_POSTCOPY_FAILED)); @@ -7041,6 +7136,8 @@ qemuMigrationDstFinishActive(virQEMUDriver *driver, QEMU_MIGRATION_COOKIE_STATS) < 0) VIR_WARN("Unable to encode migration cookie"); + qemuMigrationDeletePSKDir(driver, vm); + qemuMigrationDstComplete(driver, vm, inPostCopy, VIR_ASYNC_JOB_MIGRATION_IN, vm->job); @@ -7051,6 +7148,8 @@ qemuMigrationDstFinishActive(virQEMUDriver *driver, * overwrites it. */ virErrorPreserveLast(&orig_err); + qemuMigrationDeletePSKDir(driver, vm); + if (qemuDomainObjIsActive(vm)) { if (doKill) { qemuProcessStop(vm, VIR_DOMAIN_SHUTOFF_FAILED, @@ -7209,6 +7308,7 @@ qemuMigrationProcessUnattended(virQEMUDriver *driver, else qemuMigrationSrcComplete(driver, vm, job); + qemuMigrationDeletePSKDir(driver, vm); qemuMigrationJobFinish(vm); if (!virDomainObjIsActive(vm)) diff --git a/src/qemu/qemu_migration.h b/src/qemu/qemu_migration.h index 7e9410e1f7..ce15f8024b 100644 --- a/src/qemu/qemu_migration.h +++ b/src/qemu/qemu_migration.h @@ -279,3 +279,6 @@ int qemuMigrationAnyRefreshStatus(virDomainObj *vm, virDomainAsyncJob asyncJob, virDomainJobStatus *status); + +void +qemuMigrationDeletePSKDir(virQEMUDriver *driver, virDomainObj *vm); diff --git a/src/qemu/qemu_migration_cookie.c b/src/qemu/qemu_migration_cookie.c index 7311a8294b..c6c912e3dc 100644 --- a/src/qemu/qemu_migration_cookie.c +++ b/src/qemu/qemu_migration_cookie.c @@ -20,6 +20,7 @@ #include <gnutls/gnutls.h> #include <gnutls/x509.h> +#include <inttypes.h> #include "locking/domain_lock.h" #include "virerror.h" @@ -27,6 +28,7 @@ #include "virnetdevopenvswitch.h" #include "virstring.h" #include "virutil.h" +#include "virsecureerase.h" #include "qemu_domain.h" #include "qemu_migration_cookie.h" @@ -52,6 +54,7 @@ VIR_ENUM_IMPL(qemuMigrationCookieFlag, "allowReboot", "capabilities", "block-dirty-bitmaps", + "psk", ); @@ -165,6 +168,9 @@ qemuMigrationCookieFree(qemuMigrationCookie *mig) g_free(mig->name); g_free(mig->lockState); g_free(mig->lockDriver); + if (mig->tlsPSK) + virSecureEraseString(mig->tlsPSK); + g_free(mig->tlsPSK); g_clear_pointer(&mig->jobData, virDomainJobDataFree); virCPUDefFree(mig->cpu); qemuMigrationCookieCapsFree(mig->caps); @@ -575,6 +581,51 @@ qemuMigrationCookieAddCaps(qemuMigrationCookie *mig, } +static int +qemuMigrationCookieAddTLSPSK(qemuMigrationCookie *mig, + virQEMUDriver *driver, + virDomainObj *vm) +{ + g_autoptr(virQEMUDriverConfig) cfg = virQEMUDriverGetConfig(driver); + qemuDomainObjPrivate *priv = vm->privateData; + gnutls_datum_t psk_key = {NULL, 0}; + g_autofree char *key = NULL; + size_t key_len; + int ret; + + /* Generate the pre-shared key exactly once for a migration session*/ + if (priv->migTLSPSK) { + mig->tlsPSK = g_strdup(priv->migTLSPSK); + mig->flags |= QEMU_MIGRATION_COOKIE_TLS_PSK; + return 0; + } + + ret = gnutls_key_generate(&psk_key, cfg->migrateTLSPSKLength); + if (ret < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("Generation of a pre-shared key failed")); + return -1; + } + key_len = (psk_key.size*2) + 1; + key = g_new0(char, key_len); + + ret = gnutls_hex_encode(&psk_key, key, &key_len); + if (ret < 0) { + gnutls_free(psk_key.data); + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("Hex encoding of a PSK key failed")); + return -1; + } + + priv->migTLSPSK = g_strdup(key); + mig->tlsPSK = g_steal_pointer(&key); + mig->flags |= QEMU_MIGRATION_COOKIE_TLS_PSK; + + gnutls_free(psk_key.data); + return 0; +} + + static void qemuMigrationCookieGraphicsXMLFormat(virBuffer *buf, qemuMigrationCookieGraphics *grap) @@ -890,6 +941,9 @@ qemuMigrationCookieXMLFormat(virQEMUDriver *driver, if (mig->flags & QEMU_MIGRATION_COOKIE_BLOCK_DIRTY_BITMAPS) qemuMigrationCookieBlockDirtyBitmapsFormat(buf, mig->blockDirtyBitmaps); + if ((mig->flags & QEMU_MIGRATION_COOKIE_TLS_PSK) && mig->tlsPSK) + virBufferAsprintf(buf, "<migration-key>%s</migration-key>\n", mig->tlsPSK); + virBufferAdjustIndent(buf, -2); virBufferAddLit(buf, "</qemu-migration>\n"); return 0; @@ -1396,6 +1450,12 @@ qemuMigrationCookieXMLParse(qemuMigrationCookie *mig, qemuMigrationCookieBlockDirtyBitmapsParse(ctxt, mig) < 0) return -1; + if (flags & QEMU_MIGRATION_COOKIE_TLS_PSK) { + mig->tlsPSK = virXPathString("string(./migration-key[1])", ctxt); + if (mig->tlsPSK) + mig->flags |= QEMU_MIGRATION_COOKIE_TLS_PSK; + } + return 0; } @@ -1471,14 +1531,17 @@ qemuMigrationCookieFormat(qemuMigrationCookie *mig, qemuMigrationCookieAddCaps(mig, dom, party) < 0) return -1; + if (flags & QEMU_MIGRATION_COOKIE_TLS_PSK && + party == QEMU_MIGRATION_SOURCE && + qemuMigrationCookieAddTLSPSK(mig, driver, dom) < 0) + return -1; + if (qemuMigrationCookieXMLFormat(driver, priv->qemuCaps, &buf, mig) < 0) return -1; *cookieoutlen = virBufferUse(&buf) + 1; *cookieout = virBufferContentAndReset(&buf); - VIR_DEBUG("cookielen=%d cookie=%s", *cookieoutlen, *cookieout); - return 0; } diff --git a/src/qemu/qemu_migration_cookie.h b/src/qemu/qemu_migration_cookie.h index 254372234d..fd3b4c5a56 100644 --- a/src/qemu/qemu_migration_cookie.h +++ b/src/qemu/qemu_migration_cookie.h @@ -35,6 +35,7 @@ typedef enum { QEMU_MIGRATION_COOKIE_FLAG_ALLOW_REBOOT, QEMU_MIGRATION_COOKIE_FLAG_CAPS, QEMU_MIGRATION_COOKIE_FLAG_BLOCK_DIRTY_BITMAPS, + QEMU_MIGRATION_COOKIE_FLAG_TLS_PSK, QEMU_MIGRATION_COOKIE_FLAG_LAST } qemuMigrationCookieFlags; @@ -53,6 +54,7 @@ typedef enum { QEMU_MIGRATION_COOKIE_CPU = (1 << QEMU_MIGRATION_COOKIE_FLAG_CPU), QEMU_MIGRATION_COOKIE_CAPS = (1 << QEMU_MIGRATION_COOKIE_FLAG_CAPS), QEMU_MIGRATION_COOKIE_BLOCK_DIRTY_BITMAPS = (1 << QEMU_MIGRATION_COOKIE_FLAG_BLOCK_DIRTY_BITMAPS), + QEMU_MIGRATION_COOKIE_TLS_PSK = (1 << QEMU_MIGRATION_COOKIE_FLAG_TLS_PSK), } qemuMigrationCookieFeatures; typedef struct _qemuMigrationCookieGraphics qemuMigrationCookieGraphics; @@ -171,6 +173,9 @@ struct _qemuMigrationCookie { /* If flags & QEMU_MIGRATION_COOKIE_BLOCK_DIRTY_BITMAPS */ GSList *blockDirtyBitmaps; + + /* If flags & QEMU_MIGRATION_COOKIE_TLS_PSK */ + char *tlsPSK; }; diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c index 3f1960f314..a33cb874f2 100644 --- a/src/qemu/qemu_process.c +++ b/src/qemu/qemu_process.c @@ -9453,6 +9453,9 @@ qemuProcessStop(virDomainObj *vm, /* Do this before we delete the tree and remove pidfile. */ qemuProcessKillManagedPRDaemon(vm); + /* PSK files might be left at this point, clean them up */ + qemuMigrationDeletePSKDir(driver, vm); + qemuDomainCleanupRun(driver, vm); qemuExtDevicesStop(driver, vm, !!(flags & VIR_QEMU_PROCESS_STOP_MIGRATED)); diff --git a/tests/qemumigrationcookiexmltest.c b/tests/qemumigrationcookiexmltest.c index bc0f68b8c5..41846b8631 100644 --- a/tests/qemumigrationcookiexmltest.c +++ b/tests/qemumigrationcookiexmltest.c @@ -160,8 +160,12 @@ testQemuMigrationCookieParse(const void *opaque) return -1; } - /* set all flags so that formatter attempts to format everything */ - data->cookie->flags = ~0; + /* Set all flags except QEMU_MIGRATION_COOKIE_TLS_PSK so that formatter + * attempts to format everything except the migration-key element. This is + * because the value of the migration-key element is randomly generated every time + * the migration cookie is constructed. + */ + data->cookie->flags = ~QEMU_MIGRATION_COOKIE_TLS_PSK; if (qemuMigrationCookieXMLFormat(&driver, priv->qemuCaps, @@ -225,10 +229,12 @@ testQemuMigrationCookieDom2XML(const char *namesuffix, * - lockstate: internals are NULL in tests, causes crash * - nbd: monitor not present * - dirty bitmaps: monitor not present + * - tls-psk: a new key is generated every time the migration cookie is constructed, so we can't test it */ unsigned int cookiePopulateFlagMask = QEMU_MIGRATION_COOKIE_LOCKSTATE | QEMU_MIGRATION_COOKIE_NBD | - QEMU_MIGRATION_COOKIE_BLOCK_DIRTY_BITMAPS; + QEMU_MIGRATION_COOKIE_BLOCK_DIRTY_BITMAPS | + QEMU_MIGRATION_COOKIE_TLS_PSK; data->cookiePopulateFlags = ~cookiePopulateFlagMask; } -- 2.43.7