NVMe devices in bhyve are modeled this way: -s $pciaddr,nvme,devpath[,opts] devpath can be a path to the image or the block device. It also can be "ram=size_in_MiB", but this is not covered by this series. There could be only a single device per PCI address. Optional configuration options (such as max number of queues, concurrent I/O requests, etc) are also not covered by this series. Signed-off-by: Roman Bogorodskiy <bogorodskiy@gmail.com> --- src/bhyve/bhyve_capabilities.c | 14 +++++++ src/bhyve/bhyve_capabilities.h | 1 + src/bhyve/bhyve_command.c | 41 ++++++++++++++++++- src/bhyve/bhyve_device.c | 1 + src/bhyve/bhyve_domain.c | 15 +++++-- .../bhyvexml2argvdata/bhyvexml2argv-nvme.args | 9 ++++ .../bhyvexml2argv-nvme.ldargs | 4 ++ .../bhyvexml2argvdata/bhyvexml2argv-nvme.xml | 16 ++++++++ tests/bhyvexml2argvtest.c | 4 +- .../bhyvexml2xmlout-nvme.xml | 27 ++++++++++++ tests/bhyvexml2xmltest.c | 1 + 11 files changed, 127 insertions(+), 6 deletions(-) create mode 100644 tests/bhyvexml2argvdata/bhyvexml2argv-nvme.args create mode 100644 tests/bhyvexml2argvdata/bhyvexml2argv-nvme.ldargs create mode 100644 tests/bhyvexml2argvdata/bhyvexml2argv-nvme.xml create mode 100644 tests/bhyvexml2xmloutdata/bhyvexml2xmlout-nvme.xml diff --git a/src/bhyve/bhyve_capabilities.c b/src/bhyve/bhyve_capabilities.c index 04a5a4cf29..aad757e801 100644 --- a/src/bhyve/bhyve_capabilities.c +++ b/src/bhyve/bhyve_capabilities.c @@ -355,6 +355,17 @@ bhyveProbeCapsVirtioRnd(unsigned int *caps, char *binary) } +static int +bhyveProbeCapsNvme(unsigned int *caps, char *binary) +{ + return bhyveProbeCapsDeviceHelper(caps, binary, + "-s", + "0,nvme", + "pci slot 0:0: unknown device \"nvme\"", + BHYVE_CAP_NVME); +} + + int virBhyveProbeCaps(unsigned int *caps) { @@ -395,6 +406,9 @@ virBhyveProbeCaps(unsigned int *caps) if ((ret = bhyveProbeCapsVirtioRnd(caps, binary))) goto out; + if ((ret = bhyveProbeCapsNvme(caps, binary))) + goto out; + out: VIR_FREE(binary); return ret; diff --git a/src/bhyve/bhyve_capabilities.h b/src/bhyve/bhyve_capabilities.h index 9b24241dc1..500b235397 100644 --- a/src/bhyve/bhyve_capabilities.h +++ b/src/bhyve/bhyve_capabilities.h @@ -54,6 +54,7 @@ typedef enum { BHYVE_CAP_VNC_PASSWORD = 1 << 8, BHYVE_CAP_VIRTIO_9P = 1 << 9, BHYVE_CAP_VIRTIO_RND = 1 << 10, + BHYVE_CAP_NVME = 1 << 11, } virBhyveCapsFlags; int virBhyveProbeGrubCaps(virBhyveGrubCapsFlags *caps); diff --git a/src/bhyve/bhyve_command.c b/src/bhyve/bhyve_command.c index ab6d6e92e4..9f2d02b484 100644 --- a/src/bhyve/bhyve_command.c +++ b/src/bhyve/bhyve_command.c @@ -328,6 +328,38 @@ bhyveBuildUSBControllerArgStr(const virDomainDef *def, return 0; } +static int +bhyveBuildNVMeControllerArgStr(const virDomainDef *def, + virDomainControllerDef *controller, + struct _bhyveConn *driver G_GNUC_UNUSED, + virCommand *cmd) +{ + g_auto(virBuffer) buf = VIR_BUFFER_INITIALIZER; + const char *disk_source; + size_t i; + + for (i = 0; i < def->ndisks; i++) { + virDomainDiskDef *disk = def->disks[i]; + + if (disk->bus != VIR_DOMAIN_DISK_BUS_NVME) + continue; + + if (disk->info.addr.drive.controller != controller->idx) + continue; + + VIR_DEBUG("disk %zu controller %d", i, controller->idx); + + disk_source = virDomainDiskGetSource(disk); + + virCommandAddArg(cmd, "-s"); + virCommandAddArgFormat(cmd, "%d:0,nvme,%s", + controller->info.addr.pci.slot, + disk_source); + } + + return 0; +} + static int bhyveBuildVirtIODiskArgStr(const virDomainDef *def G_GNUC_UNUSED, virDomainDiskDef *disk, @@ -370,6 +402,9 @@ bhyveBuildDiskArgStr(const virDomainDef *def, case VIR_DOMAIN_DISK_BUS_SATA: /* Handled by bhyveBuildAHCIControllerArgStr() */ break; + case VIR_DOMAIN_DISK_BUS_NVME: + /* Handled by bhyveBuildNVMeControllerArgStr() */ + break; case VIR_DOMAIN_DISK_BUS_VIRTIO: if (bhyveBuildVirtIODiskArgStr(def, disk, cmd) < 0) return -1; @@ -382,7 +417,6 @@ bhyveBuildDiskArgStr(const virDomainDef *def, case VIR_DOMAIN_DISK_BUS_USB: case VIR_DOMAIN_DISK_BUS_UML: case VIR_DOMAIN_DISK_BUS_SD: - case VIR_DOMAIN_DISK_BUS_NVME: case VIR_DOMAIN_DISK_BUS_LAST: default: virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", @@ -432,13 +466,16 @@ bhyveBuildControllerArgStr(const virDomainDef *def, virCommandAddArgFormat(cmd, "%d:0,lpc", controller->info.addr.pci.slot); break; + case VIR_DOMAIN_CONTROLLER_TYPE_NVME: + if (bhyveBuildNVMeControllerArgStr(def, controller, driver, cmd) < 0) + return -1; + break; case VIR_DOMAIN_CONTROLLER_TYPE_IDE: case VIR_DOMAIN_CONTROLLER_TYPE_FDC: case VIR_DOMAIN_CONTROLLER_TYPE_SCSI: case VIR_DOMAIN_CONTROLLER_TYPE_VIRTIO_SERIAL: case VIR_DOMAIN_CONTROLLER_TYPE_CCID: case VIR_DOMAIN_CONTROLLER_TYPE_XENBUS: - case VIR_DOMAIN_CONTROLLER_TYPE_NVME: case VIR_DOMAIN_CONTROLLER_TYPE_LAST: default: virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", diff --git a/src/bhyve/bhyve_device.c b/src/bhyve/bhyve_device.c index 68983d5bc4..49cfccaeba 100644 --- a/src/bhyve/bhyve_device.c +++ b/src/bhyve/bhyve_device.c @@ -114,6 +114,7 @@ bhyveAssignDevicePCISlots(virDomainDef *def, for (i = 0; i < def->ncontrollers; i++) { if ((def->controllers[i]->type == VIR_DOMAIN_CONTROLLER_TYPE_PCI) || (def->controllers[i]->type == VIR_DOMAIN_CONTROLLER_TYPE_SATA) || + (def->controllers[i]->type == VIR_DOMAIN_CONTROLLER_TYPE_NVME) || ((def->controllers[i]->type == VIR_DOMAIN_CONTROLLER_TYPE_USB) && (def->controllers[i]->model == VIR_DOMAIN_CONTROLLER_MODEL_USB_NEC_XHCI)) || def->controllers[i]->type == VIR_DOMAIN_CONTROLLER_TYPE_ISA) { diff --git a/src/bhyve/bhyve_domain.c b/src/bhyve/bhyve_domain.c index 9dec300a99..79ac336430 100644 --- a/src/bhyve/bhyve_domain.c +++ b/src/bhyve/bhyve_domain.c @@ -111,9 +111,10 @@ bhyveDomainDiskDefAssignAddress(struct _bhyveConn *driver, virDomainDiskDef *def, const virDomainDef *vmdef G_GNUC_UNUSED) { - int idx = virDiskNameToIndex(def->dst); + int idx = -1; + int nvme_ctrl = 0; - if (idx < 0) { + if (virDiskNameParse(def->dst, &nvme_ctrl, &idx, NULL) < 0) { virReportError(VIR_ERR_XML_ERROR, _("Unknown disk name '%1$s' and no address specified"), def->dst); @@ -134,6 +135,15 @@ bhyveDomainDiskDefAssignAddress(struct _bhyveConn *driver, def->info.addr.drive.bus = 0; break; + + case VIR_DOMAIN_DISK_BUS_NVME: + def->info.type = VIR_DOMAIN_DEVICE_ADDRESS_TYPE_DRIVE; + + def->info.addr.drive.controller = nvme_ctrl; + def->info.addr.drive.unit = 0; + def->info.addr.drive.bus = idx; + break; + case VIR_DOMAIN_DISK_BUS_SCSI: case VIR_DOMAIN_DISK_BUS_IDE: case VIR_DOMAIN_DISK_BUS_FDC: @@ -143,7 +153,6 @@ bhyveDomainDiskDefAssignAddress(struct _bhyveConn *driver, case VIR_DOMAIN_DISK_BUS_USB: case VIR_DOMAIN_DISK_BUS_UML: case VIR_DOMAIN_DISK_BUS_SD: - case VIR_DOMAIN_DISK_BUS_NVME: case VIR_DOMAIN_DISK_BUS_LAST: default: break; diff --git a/tests/bhyvexml2argvdata/bhyvexml2argv-nvme.args b/tests/bhyvexml2argvdata/bhyvexml2argv-nvme.args new file mode 100644 index 0000000000..bd39db1fe6 --- /dev/null +++ b/tests/bhyvexml2argvdata/bhyvexml2argv-nvme.args @@ -0,0 +1,9 @@ +bhyve \ +-c 1 \ +-m 214 \ +-u \ +-H \ +-P \ +-s 0:0,hostbridge \ +-s 2:0,nvme,/tmp/freebsd.img \ +bhyve diff --git a/tests/bhyvexml2argvdata/bhyvexml2argv-nvme.ldargs b/tests/bhyvexml2argvdata/bhyvexml2argv-nvme.ldargs new file mode 100644 index 0000000000..5905f4b3e6 --- /dev/null +++ b/tests/bhyvexml2argvdata/bhyvexml2argv-nvme.ldargs @@ -0,0 +1,4 @@ +bhyveload \ +-m 214 \ +-d /tmp/freebsd.img \ +bhyve diff --git a/tests/bhyvexml2argvdata/bhyvexml2argv-nvme.xml b/tests/bhyvexml2argvdata/bhyvexml2argv-nvme.xml new file mode 100644 index 0000000000..8daaa11e85 --- /dev/null +++ b/tests/bhyvexml2argvdata/bhyvexml2argv-nvme.xml @@ -0,0 +1,16 @@ +<domain type='bhyve'> + <name>bhyve</name> + <uuid>df3be7e7-a104-11e3-aeb0-50e5492bd3dc</uuid> + <memory>219136</memory> + <vcpu>1</vcpu> + <os> + <type>hvm</type> + </os> + <devices> + <disk type='file'> + <driver name='file' type='raw'/> + <source file='/tmp/freebsd.img'/> + <target dev='nvme0n1' bus='nvme'/> + </disk> + </devices> +</domain> diff --git a/tests/bhyvexml2argvtest.c b/tests/bhyvexml2argvtest.c index cc6b17233d..0a5202e425 100644 --- a/tests/bhyvexml2argvtest.c +++ b/tests/bhyvexml2argvtest.c @@ -195,7 +195,8 @@ mymain(void) BHYVE_CAP_NET_E1000 | BHYVE_CAP_LPC_BOOTROM | \ BHYVE_CAP_FBUF | BHYVE_CAP_XHCI | \ BHYVE_CAP_CPUTOPOLOGY | BHYVE_CAP_SOUND_HDA | \ - BHYVE_CAP_VNC_PASSWORD | BHYVE_CAP_VIRTIO_9P; + BHYVE_CAP_VNC_PASSWORD | BHYVE_CAP_VIRTIO_9P | \ + BHYVE_CAP_NVME; DO_TEST("base"); DO_TEST("wired"); @@ -259,6 +260,7 @@ mymain(void) DO_TEST("serial-tcp"); DO_TEST("4-consoles"); DO_TEST_FAILURE("serial-invalid-port"); + DO_TEST("nvme"); /* Address allocation tests */ DO_TEST("addr-single-sata-disk"); diff --git a/tests/bhyvexml2xmloutdata/bhyvexml2xmlout-nvme.xml b/tests/bhyvexml2xmloutdata/bhyvexml2xmlout-nvme.xml new file mode 100644 index 0000000000..d4ab69b15f --- /dev/null +++ b/tests/bhyvexml2xmloutdata/bhyvexml2xmlout-nvme.xml @@ -0,0 +1,27 @@ +<domain type='bhyve'> + <name>bhyve</name> + <uuid>df3be7e7-a104-11e3-aeb0-50e5492bd3dc</uuid> + <memory unit='KiB'>219136</memory> + <currentMemory unit='KiB'>219136</currentMemory> + <vcpu placement='static'>1</vcpu> + <os> + <type arch='x86_64'>hvm</type> + <boot dev='hd'/> + </os> + <clock offset='utc'/> + <on_poweroff>destroy</on_poweroff> + <on_reboot>restart</on_reboot> + <on_crash>destroy</on_crash> + <devices> + <disk type='file' device='disk'> + <driver name='file' type='raw'/> + <source file='/tmp/freebsd.img'/> + <target dev='nvme0n1' bus='nvme'/> + <address type='drive' controller='0' bus='0' target='0' unit='0'/> + </disk> + <controller type='pci' index='0' model='pci-root'/> + <controller type='nvme' index='0'> + <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0'/> + </controller> + </devices> +</domain> diff --git a/tests/bhyvexml2xmltest.c b/tests/bhyvexml2xmltest.c index df093a5539..be69e21484 100644 --- a/tests/bhyvexml2xmltest.c +++ b/tests/bhyvexml2xmltest.c @@ -117,6 +117,7 @@ mymain(void) DO_TEST_DIFFERENT("virtio-rnd"); DO_TEST_DIFFERENT("serial-tcp"); DO_TEST_DIFFERENT("4-consoles"); + DO_TEST_DIFFERENT("nvme"); /* Address allocation tests */ DO_TEST_DIFFERENT("addr-single-sata-disk"); -- 2.51.0

As bhyve does not have explicit notion of controllers, and for NVMe devices it allows to specify one a single source for for a given PCI address, it effectively means that there could be only one device per controller. Update validation code to check this case. Signed-off-by: Roman Bogorodskiy <bogorodskiy@gmail.com> --- src/bhyve/bhyve_domain.c | 27 +++++++++++++++++++ .../bhyvexml2argv-2-nvme-same-controller.args | 10 +++++++ ...hyvexml2argv-2-nvme-same-controller.ldargs | 4 +++ .../bhyvexml2argv-2-nvme-same-controller.xml | 21 +++++++++++++++ tests/bhyvexml2argvtest.c | 1 + 5 files changed, 63 insertions(+) create mode 100644 tests/bhyvexml2argvdata/bhyvexml2argv-2-nvme-same-controller.args create mode 100644 tests/bhyvexml2argvdata/bhyvexml2argv-2-nvme-same-controller.ldargs create mode 100644 tests/bhyvexml2argvdata/bhyvexml2argv-2-nvme-same-controller.xml diff --git a/src/bhyve/bhyve_domain.c b/src/bhyve/bhyve_domain.c index 79ac336430..6b47890c93 100644 --- a/src/bhyve/bhyve_domain.c +++ b/src/bhyve/bhyve_domain.c @@ -310,7 +310,34 @@ bhyveDomainDefValidate(const virDomainDef *def, void *opaque G_GNUC_UNUSED, void *parseOpaque G_GNUC_UNUSED) { + size_t i; virStorageSource *src = NULL; + g_autoptr(GHashTable) nvme_controllers = g_hash_table_new(g_direct_hash, + g_direct_equal); + + for (i = 0; i < def->ndisks; i++) { + virDomainDiskDef *disk = def->disks[i]; + int nvme_ctrl = 0; + int idx = -1; + + if (disk->bus == VIR_DOMAIN_DISK_BUS_NVME) { + if (virDiskNameParse(disk->dst, &nvme_ctrl, &idx, NULL) < 0) { + virReportError(VIR_ERR_XML_ERROR, + _("Unknown disk name '%1$s' and no address specified"), + disk->dst); + return -1; + } + + if (g_hash_table_contains(nvme_controllers, GINT_TO_POINTER(nvme_ctrl))) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, + "%s", + _("Cannot have more than one disk per NVMe controller")); + return -1; + } + + g_hash_table_add(nvme_controllers, GINT_TO_POINTER(nvme_ctrl)); + } + } if (!def->os.loader) return 0; diff --git a/tests/bhyvexml2argvdata/bhyvexml2argv-2-nvme-same-controller.args b/tests/bhyvexml2argvdata/bhyvexml2argv-2-nvme-same-controller.args new file mode 100644 index 0000000000..664eec99bc --- /dev/null +++ b/tests/bhyvexml2argvdata/bhyvexml2argv-2-nvme-same-controller.args @@ -0,0 +1,10 @@ +bhyve \ +-c 1 \ +-m 214 \ +-u \ +-H \ +-P \ +-s 0:0,hostbridge \ +-s 2:0,nvme,/tmp/freebsd.img \ +-s 3:0,nvme,/tmp/data.img \ +bhyve diff --git a/tests/bhyvexml2argvdata/bhyvexml2argv-2-nvme-same-controller.ldargs b/tests/bhyvexml2argvdata/bhyvexml2argv-2-nvme-same-controller.ldargs new file mode 100644 index 0000000000..5905f4b3e6 --- /dev/null +++ b/tests/bhyvexml2argvdata/bhyvexml2argv-2-nvme-same-controller.ldargs @@ -0,0 +1,4 @@ +bhyveload \ +-m 214 \ +-d /tmp/freebsd.img \ +bhyve diff --git a/tests/bhyvexml2argvdata/bhyvexml2argv-2-nvme-same-controller.xml b/tests/bhyvexml2argvdata/bhyvexml2argv-2-nvme-same-controller.xml new file mode 100644 index 0000000000..dc4e3c621b --- /dev/null +++ b/tests/bhyvexml2argvdata/bhyvexml2argv-2-nvme-same-controller.xml @@ -0,0 +1,21 @@ +<domain type='bhyve'> + <name>bhyve</name> + <uuid>df3be7e7-a104-11e3-aeb0-50e5492bd3dc</uuid> + <memory>219136</memory> + <vcpu>1</vcpu> + <os> + <type>hvm</type> + </os> + <devices> + <disk type='file'> + <driver name='file' type='raw'/> + <source file='/tmp/freebsd.img'/> + <target dev='nvme0n1' bus='nvme'/> + </disk> + <disk type='file'> + <driver name='file' type='raw'/> + <source file='/tmp/data.img'/> + <target dev='nvme0n2' bus='nvme'/> + </disk> + </devices> +</domain> diff --git a/tests/bhyvexml2argvtest.c b/tests/bhyvexml2argvtest.c index 9d20e5669e..3c6d530f66 100644 --- a/tests/bhyvexml2argvtest.c +++ b/tests/bhyvexml2argvtest.c @@ -262,6 +262,7 @@ mymain(void) DO_TEST_FAILURE("serial-invalid-port"); DO_TEST("nvme"); DO_TEST("2-nvme-2-controllers"); + DO_TEST_FAILURE("2-nvme-same-controller"); /* Address allocation tests */ DO_TEST("addr-single-sata-disk"); -- 2.51.0