10:54 a.m.
Now that we have virNVMeDevice module (introduced in previous
commit), let's use it int virHostdev to track which NVMe devices
are free to be used by a domain and which are taken.
Signed-off-by: Michal Privoznik <mprivozn(a)redhat.com>
---
src/libvirt_private.syms | 3 +
src/util/virhostdev.c | 244 +++++++++++++++++++++++++++++++++++++++
src/util/virhostdev.h | 25 ++++
3 files changed, 272 insertions(+)
diff --git a/src/libvirt_private.syms b/src/libvirt_private.syms
index 856b770e57..bc6583562a 100644
--- a/src/libvirt_private.syms
+++ b/src/libvirt_private.syms
@@ -2077,18 +2077,21 @@ virHostdevPCINodeDeviceReAttach;
virHostdevPCINodeDeviceReset;
virHostdevPrepareDomainDevices;
virHostdevPrepareMediatedDevices;
+virHostdevPrepareNVMeDevices;
virHostdevPreparePCIDevices;
virHostdevPrepareSCSIDevices;
virHostdevPrepareSCSIVHostDevices;
virHostdevPrepareUSBDevices;
virHostdevReAttachDomainDevices;
virHostdevReAttachMediatedDevices;
+virHostdevReAttachNVMeDevices;
virHostdevReAttachPCIDevices;
virHostdevReAttachSCSIDevices;
virHostdevReAttachSCSIVHostDevices;
virHostdevReAttachUSBDevices;
virHostdevUpdateActiveDomainDevices;
virHostdevUpdateActiveMediatedDevices;
+virHostdevUpdateActiveNVMeDevices;
virHostdevUpdateActivePCIDevices;
virHostdevUpdateActiveSCSIDevices;
virHostdevUpdateActiveUSBDevices;
diff --git a/src/util/virhostdev.c b/src/util/virhostdev.c
index 07397b9682..90d94b0a92 100644
--- a/src/util/virhostdev.c
+++ b/src/util/virhostdev.c
@@ -137,6 +137,7 @@ virHostdevManagerDispose(void *obj)
virObjectUnref(hostdevMgr->activeSCSIHostdevs);
virObjectUnref(hostdevMgr->activeSCSIVHostHostdevs);
virObjectUnref(hostdevMgr->activeMediatedHostdevs);
+ virObjectUnref(hostdevMgr->activeNVMeHostdevs);
VIR_FREE(hostdevMgr->stateDir);
}
@@ -167,6 +168,9 @@ virHostdevManagerNew(void)
if (!(hostdevMgr->activeMediatedHostdevs = virMediatedDeviceListNew()))
return NULL;
+ if (!(hostdevMgr->activeNVMeHostdevs = virNVMeDeviceListNew()))
+ return NULL;
+
if (privileged) {
if (VIR_STRDUP(hostdevMgr->stateDir, HOSTDEV_STATE_DIR) < 0)
return NULL;
@@ -2229,3 +2233,243 @@ virHostdevUpdateActiveDomainDevices(virHostdevManagerPtr mgr,
return 0;
}
+
+
+static virNVMeDeviceListPtr
+virHostdevGetNVMeDeviceList(virDomainDiskDefPtr *disks,
+ size_t ndisks,
+ const char *drv_name,
+ const char *dom_name)
+{
+ VIR_AUTOUNREF(virNVMeDeviceListPtr) nvmeDevices = NULL;
+ size_t i;
+
+ if (!(nvmeDevices = virNVMeDeviceListNew()))
+ return NULL;
+
+ for (i = 0; i < ndisks; i++) {
+ virDomainDiskDefPtr disk = disks[i];
+ virStorageSourcePtr n;
+
+ for (n = disk->src; virStorageSourceIsBacking(n); n = n->backingStore) {
+ VIR_AUTOPTR(virNVMeDevice) dev = NULL;
+ const virStorageSourceNVMeDef *srcNVMe = n->nvme;
+
+ if (n->type != VIR_STORAGE_TYPE_NVME)
+ continue;
+
+ if (!(dev = virNVMeDeviceNew(&srcNVMe->pciAddr,
+ srcNVMe->namespace,
+ srcNVMe->managed)))
+ return NULL;
+
+ if (virNVMeDeviceUsedBySet(dev, drv_name, dom_name) < 0)
+ return NULL;
+
+ if (virNVMeDeviceListAdd(nvmeDevices, dev) < 0)
+ return NULL;
+ }
+ }
+
+ VIR_RETURN_PTR(nvmeDevices);
+}
+
+
+int
+virHostdevPrepareNVMeDevices(virHostdevManagerPtr hostdev_mgr,
+ const char *drv_name,
+ const char *dom_name,
+ virDomainDiskDefPtr *disks,
+ size_t ndisks)
+{
+ VIR_AUTOUNREF(virNVMeDeviceListPtr) nvmeDevices = NULL;
+ VIR_AUTOUNREF(virPCIDeviceListPtr) pciDevices = NULL;
+ const unsigned int pciFlags = 0;
+ virNVMeDevicePtr temp = NULL;
+ size_t i;
+ ssize_t lastGoodNVMeIdx = -1;
+ int ret = -1;
+
+ if (!(nvmeDevices = virHostdevGetNVMeDeviceList(disks, ndisks, drv_name, dom_name)))
+ return -1;
+
+ if (virNVMeDeviceListCount(nvmeDevices) == 0)
+ return 0;
+
+ virObjectLock(hostdev_mgr->activeNVMeHostdevs);
+
+ /* Firstly, let's check if all devices are free */
+ for (i = 0; i < virNVMeDeviceListCount(nvmeDevices); i++) {
+ const virNVMeDevice *dev = virNVMeDeviceListGet(nvmeDevices, i);
+ const virPCIDeviceAddress *addr = NULL;
+ VIR_AUTOFREE(char *) addrStr = NULL;
+ const char *actual_drvname = NULL;
+ const char *actual_domname = NULL;
+
+ temp = virNVMeDeviceListLookup(hostdev_mgr->activeNVMeHostdevs, dev);
+
+ /* Not on the list means not used */
+ if (!temp)
+ continue;
+
+ virNVMeDeviceUsedByGet(temp, &actual_drvname, &actual_domname);
+ addr = virNVMeDeviceAddressGet(dev);
+ addrStr = virPCIDeviceAddressAsString(addr);
+
+ virReportError(VIR_ERR_OPERATION_INVALID,
+ _("NVMe device %s already in use by driver %s domain
%s"),
+ NULLSTR(addrStr), actual_drvname, actual_domname);
+ goto cleanup;
+ }
+
+ if (!(pciDevices =
virNVMeDeviceListCreateDetachList(hostdev_mgr->activeNVMeHostdevs,
+ nvmeDevices)))
+ goto cleanup;
+
+ /* This looks like a good opportunity to merge inactive NVMe devices onto
+ * the active list. This, however, means that if something goes wrong we
+ * have to perform a rollback before returning.*/
+ for (i = 0; i < virNVMeDeviceListCount(nvmeDevices); i++) {
+ temp = virNVMeDeviceListGet(nvmeDevices, i);
+
+ if (virNVMeDeviceListAdd(hostdev_mgr->activeNVMeHostdevs, temp) < 0)
+ goto rollback;
+
+ lastGoodNVMeIdx = i;
+ }
+
+ if (virHostdevPreparePCIDevicesImpl(hostdev_mgr,
+ drv_name, dom_name, NULL,
+ pciDevices, NULL, 0, pciFlags) < 0)
+ goto rollback;
+
+ ret = 0;
+ cleanup:
+ virObjectUnlock(hostdev_mgr->activeNVMeHostdevs);
+ return ret;
+
+ rollback:
+ while (lastGoodNVMeIdx >= 0) {
+ temp = virNVMeDeviceListGet(nvmeDevices, lastGoodNVMeIdx);
+
+ virNVMeDeviceListDel(hostdev_mgr->activeNVMeHostdevs, temp);
+
+ lastGoodNVMeIdx--;
+ }
+ goto cleanup;
+}
+
+
+int
+virHostdevReAttachNVMeDevices(virHostdevManagerPtr hostdev_mgr,
+ const char *drv_name,
+ const char *dom_name,
+ virDomainDiskDefPtr *disks,
+ size_t ndisks)
+{
+ VIR_AUTOUNREF(virNVMeDeviceListPtr) nvmeDevices = NULL;
+ VIR_AUTOUNREF(virPCIDeviceListPtr) pciDevices = NULL;
+ size_t i;
+ int ret = -1;
+
+ if (!(nvmeDevices = virHostdevGetNVMeDeviceList(disks, ndisks, drv_name, dom_name)))
+ return -1;
+
+ if (virNVMeDeviceListCount(nvmeDevices) == 0)
+ return 0;
+
+ virObjectLock(hostdev_mgr->activeNVMeHostdevs);
+
+ if (!(pciDevices =
virNVMeDeviceListCreateReAttachList(hostdev_mgr->activeNVMeHostdevs,
+ nvmeDevices)))
+ goto cleanup;
+
+ virHostdevReAttachPCIDevicesImpl(hostdev_mgr,
+ drv_name, dom_name, pciDevices,
+ NULL, 0, NULL);
+
+ for (i = 0; i < virNVMeDeviceListCount(nvmeDevices); i++) {
+ virNVMeDevicePtr temp = virNVMeDeviceListGet(nvmeDevices, i);
+
+ if (virNVMeDeviceListDel(hostdev_mgr->activeNVMeHostdevs, temp) < 0)
+ goto cleanup;
+ }
+
+ ret = 0;
+ cleanup:
+ virObjectUnlock(hostdev_mgr->activeNVMeHostdevs);
+ return ret;
+}
+
+
+int
+virHostdevUpdateActiveNVMeDevices(virHostdevManagerPtr hostdev_mgr,
+ const char *drv_name,
+ const char *dom_name,
+ virDomainDiskDefPtr *disks,
+ size_t ndisks)
+{
+ VIR_AUTOUNREF(virNVMeDeviceListPtr) nvmeDevices = NULL;
+ VIR_AUTOUNREF(virPCIDeviceListPtr) pciDevices = NULL;
+ virNVMeDevicePtr temp = NULL;
+ size_t i;
+ ssize_t lastGoodNVMeIdx = -1;
+ ssize_t lastGoodPCIIdx = -1;
+ int ret = -1;
+
+ if (!(nvmeDevices = virHostdevGetNVMeDeviceList(disks, ndisks, drv_name, dom_name)))
+ return -1;
+
+ if (virNVMeDeviceListCount(nvmeDevices) == 0)
+ return 0;
+
+ virObjectLock(hostdev_mgr->activeNVMeHostdevs);
+ virObjectLock(hostdev_mgr->activePCIHostdevs);
+ virObjectLock(hostdev_mgr->inactivePCIHostdevs);
+
+ if (!(pciDevices =
virNVMeDeviceListCreateDetachList(hostdev_mgr->activeNVMeHostdevs,
+ nvmeDevices)))
+ goto cleanup;
+
+ for (i = 0; i < virNVMeDeviceListCount(nvmeDevices); i++) {
+ temp = virNVMeDeviceListGet(nvmeDevices, i);
+
+ if (virNVMeDeviceListAdd(hostdev_mgr->activeNVMeHostdevs, temp) < 0)
+ goto rollback;
+
+ lastGoodNVMeIdx = i;
+ }
+
+ for (i = 0; i < virPCIDeviceListCount(pciDevices); i++) {
+ virPCIDevicePtr actual = virPCIDeviceListGet(pciDevices, i);
+
+ if (virPCIDeviceListAddCopy(hostdev_mgr->activePCIHostdevs, actual) < 0)
+ goto rollback;
+
+ lastGoodPCIIdx = i;
+ }
+
+ ret = 0;
+ cleanup:
+ virObjectUnlock(hostdev_mgr->inactivePCIHostdevs);
+ virObjectUnlock(hostdev_mgr->activePCIHostdevs);
+ virObjectUnlock(hostdev_mgr->activeNVMeHostdevs);
+ return ret;
+
+ rollback:
+ while (lastGoodNVMeIdx >= 0) {
+ temp = virNVMeDeviceListGet(nvmeDevices, lastGoodNVMeIdx);
+
+ virNVMeDeviceListDel(hostdev_mgr->activeNVMeHostdevs, temp);
+
+ lastGoodNVMeIdx--;
+ }
+ while (lastGoodPCIIdx >= 0) {
+ virPCIDevicePtr actual = virPCIDeviceListGet(pciDevices, i);
+
+ virPCIDeviceListDel(hostdev_mgr->activePCIHostdevs, actual);
+
+ lastGoodPCIIdx--;
+ }
+ goto cleanup;
+}
diff --git a/src/util/virhostdev.h b/src/util/virhostdev.h
index 88501e2743..98dc226631 100644
--- a/src/util/virhostdev.h
+++ b/src/util/virhostdev.h
@@ -29,6 +29,7 @@
#include "virscsivhost.h"
#include "conf/domain_conf.h"
#include "virmdev.h"
+#include "virnvme.h"
typedef enum {
VIR_HOSTDEV_STRICT_ACS_CHECK = (1 << 0), /* strict acs check */
@@ -53,6 +54,9 @@ struct _virHostdevManager {
virSCSIDeviceListPtr activeSCSIHostdevs;
virSCSIVHostDeviceListPtr activeSCSIVHostHostdevs;
virMediatedDeviceListPtr activeMediatedHostdevs;
+ /* NVMe devices are PCI devices really, but one NVMe disk can
+ * have multiple namespaces. */
+ virNVMeDeviceListPtr activeNVMeHostdevs;
};
virHostdevManagerPtr virHostdevManagerGetDefault(void);
@@ -201,3 +205,24 @@ int virHostdevPCINodeDeviceReAttach(virHostdevManagerPtr mgr,
int virHostdevPCINodeDeviceReset(virHostdevManagerPtr mgr,
virPCIDevicePtr pci)
ATTRIBUTE_NONNULL(1) ATTRIBUTE_NONNULL(2);
+
+int
+virHostdevPrepareNVMeDevices(virHostdevManagerPtr hostdev_mgr,
+ const char *drv_name,
+ const char *dom_name,
+ virDomainDiskDefPtr *disks,
+ size_t ndisks);
+
+int
+virHostdevReAttachNVMeDevices(virHostdevManagerPtr hostdev_mgr,
+ const char *drv_name,
+ const char *dom_name,
+ virDomainDiskDefPtr *disks,
+ size_t ndisks);
+
+int
+virHostdevUpdateActiveNVMeDevices(virHostdevManagerPtr hostdev_mgr,
+ const char *drv_name,
+ const char *dom_name,
+ virDomainDiskDefPtr *disks,
+ size_t ndisks);
--
2.21.0