Re: [libvirt] [PATCHv4 2/4] add pci passthrough to libxl driver

From: Chunyan Liu <cyliu(a)suse.com&gt; Add pci passthrough to libxl driver, support attach-device, detach-device and start a vm with pci hostdev specified.

Signed-off-by: Chunyan Liu <cyliu(a)suse.com&gt; --- po/POTFILES.in | 2 +- src/libxl/libxl_conf.c | 63 +++++++++++ src/libxl/libxl_conf.h | 4 + src/libxl/libxl_driver.c | 262 +++++++++++++++++++++++++++++++++++++++++++++- 4 files changed, 329 insertions(+), 2 deletions(-) diff --git a/po/POTFILES.in b/po/POTFILES.in index 6799d47..50030f8 100644 --- a/po/POTFILES.in +++ b/po/POTFILES.in @@ -157,6 +157,7 @@ src/util/vireventpoll.c src/util/virfile.c src/util/virhash.c src/util/virhook.c +src/util/virhostdev.c src/util/viridentity.c src/util/virinitctl.c src/util/viriptables.c @@ -194,7 +195,6 @@ src/util/viruri.c src/util/virusb.c src/util/virutil.c src/util/virxml.c -src/util/virhostdev.c src/vbox/vbox_MSCOMGlue.c src/vbox/vbox_XPCOMCGlue.c src/vbox/vbox_driver.c diff --git a/src/libxl/libxl_conf.c b/src/libxl/libxl_conf.c index 73e2d49..f76f432 100644 --- a/src/libxl/libxl_conf.c +++ b/src/libxl/libxl_conf.c @@ -869,6 +869,66 @@ error: return -1; } +int +libxlMakePci(virDomainHostdevDefPtr hostdev, libxl_device_pci *pcidev) +{ + if (hostdev->mode != VIR_DOMAIN_HOSTDEV_MODE_SUBSYS) + return -1; + if (hostdev->source.subsys.type != VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_PCI) + return -1; + + pcidev->domain = hostdev->source.subsys.u.pci.addr.domain; + pcidev->bus = hostdev->source.subsys.u.pci.addr.bus; + pcidev->dev = hostdev->source.subsys.u.pci.addr.slot; + pcidev->func = hostdev->source.subsys.u.pci.addr.function; + + return 0; +} + +static int +libxlMakePciList(virDomainDefPtr def, libxl_domain_config *d_config) +{ + virDomainHostdevDefPtr *l_hostdevs = def->hostdevs; + size_t nhostdevs = def->nhostdevs; + size_t npcidevs = 0; + libxl_device_pci *x_pcidevs; + size_t i, j; + + if (nhostdevs == 0) + return 0; + + if (VIR_ALLOC_N(x_pcidevs, nhostdevs) < 0) + return -1; + + for (i = 0, j = 0; i < nhostdevs; i++) { + if (l_hostdevs[i]->mode != VIR_DOMAIN_HOSTDEV_MODE_SUBSYS) + continue; + if (l_hostdevs[i]->source.subsys.type != VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_PCI) + continue; + + libxl_device_pci_init(&x_pcidevs[j]); + + if (libxlMakePci(l_hostdevs[i], &x_pcidevs[j]) < 0) + goto error; + + npcidevs++; + j++; + } + + VIR_SHRINK_N(x_pcidevs, nhostdevs, nhostdevs - npcidevs); + d_config->pcidevs = x_pcidevs; + d_config->num_pcidevs = npcidevs; + + return 0; + +error: + for (i = 0; i < npcidevs; i++) + libxl_device_pci_dispose(&x_pcidevs[i]); + + VIR_FREE(x_pcidevs); + return -1; +} + virCapsPtr libxlMakeCapabilities(libxl_ctx *ctx) { @@ -913,6 +973,9 @@ libxlBuildDomainConfig(libxlDriverPrivatePtr driver, if (libxlMakeVfbList(driver, def, d_config) < 0) return -1; + if (libxlMakePciList(def, d_config) < 0) + return -1; + d_config->on_reboot = def->onReboot; d_config->on_poweroff = def->onPoweroff; d_config->on_crash = def->onCrash; diff --git a/src/libxl/libxl_conf.h b/src/libxl/libxl_conf.h index 9d72b72..39d09fe 100644 --- a/src/libxl/libxl_conf.h +++ b/src/libxl/libxl_conf.h @@ -37,6 +37,7 @@ # include "virchrdev.h" +# define LIBXL_DRIVER_NAME "xenlight" # define LIBXL_VNC_PORT_MIN 5900 # define LIBXL_VNC_PORT_MAX 65535 @@ -132,6 +133,9 @@ libxlMakeVfb(libxlDriverPrivatePtr driver, virDomainGraphicsDefPtr l_vfb, libxl_device_vfb *x_vfb); int +libxlMakePci(virDomainHostdevDefPtr hostdev, libxl_device_pci *pcidev); + +int libxlBuildDomainConfig(libxlDriverPrivatePtr driver, virDomainObjPtr vm, libxl_domain_config *d_config); diff --git a/src/libxl/libxl_driver.c b/src/libxl/libxl_driver.c index f09ad61..029fa0c 100644 --- a/src/libxl/libxl_driver.c +++ b/src/libxl/libxl_driver.c @@ -49,6 +49,7 @@ #include "virstring.h" #include "virsysinfo.h" #include "viraccessapicheck.h" +#include "virhostdev.h" #define VIR_FROM_THIS VIR_FROM_LIBXL @@ -742,6 +743,7 @@ libxlVmReap(libxlDriverPrivatePtr driver, virDomainShutoffReason reason) { libxlDomainObjPrivatePtr priv = vm->privateData; + virHostdevManagerPtr hostdev_mgr; if (libxl_domain_destroy(priv->ctx, vm->def->id, NULL) < 0) { virReportError(VIR_ERR_INTERNAL_ERROR, @@ -749,6 +751,10 @@ libxlVmReap(libxlDriverPrivatePtr driver, return -1; } + hostdev_mgr = virHostdevManagerGetDefault(); + virHostdevReAttachDomainHostdevs(hostdev_mgr, LIBXL_DRIVER_NAME, + vm->def, VIR_SP_PCI_HOSTDEV); + libxlVmCleanup(driver, vm, reason); return 0; } @@ -972,6 +978,7 @@ libxlVmStart(libxlDriverPrivatePtr driver, virDomainObjPtr vm, char *managed_save_path = NULL; int managed_save_fd = -1; libxlDomainObjPrivatePtr priv = vm->privateData; + virHostdevManagerPtr hostdev_mgr; if (libxlDomainObjPrivateInitCtx(vm) < 0) goto error; @@ -1029,6 +1036,12 @@ libxlVmStart(libxlDriverPrivatePtr driver, virDomainObjPtr vm, goto error; } + VIR_DEBUG("Preparing host PCI devices"); + hostdev_mgr = virHostdevManagerGetDefault(); + if (virHostdevPrepareDomainHostdevs(hostdev_mgr, LIBXL_DRIVER_NAME, + vm->def, VIR_SP_PCI_HOSTDEV) < 0) + goto error; + /* use as synchronous operations => ao_how = NULL and no intermediate reports => ao_progress = NULL */ if (restore_fd < 0) @@ -1122,6 +1135,7 @@ libxlReconnectDomain(virDomainObjPtr vm, libxl_dominfo d_info; int len; uint8_t *data = NULL; + virHostdevManagerPtr hostdev_mgr; virObjectLock(vm); @@ -1144,6 +1158,13 @@ libxlReconnectDomain(virDomainObjPtr vm, /* Update domid in case it changed (e.g. reboot) while we were gone? */ vm->def->id = d_info.domid; + + /* Update hostdev state */ + hostdev_mgr = virHostdevManagerGetDefault(); + if (virHostdevPrepareDomainHostdevs(hostdev_mgr, LIBXL_DRIVER_NAME, + vm->def, VIR_SP_PCI_HOSTDEV) < 0) + goto out; + virDomainObjSetState(vm, VIR_DOMAIN_RUNNING, VIR_DOMAIN_RUNNING_UNKNOWN); if (!driver->nactive && driver->inhibitCallback) @@ -3569,6 +3590,95 @@ cleanup: } static int +libxlDomainAttachHostPciDevice(libxlDomainObjPrivatePtr priv, + virDomainObjPtr vm, + virDomainHostdevDefPtr hostdev) +{ + libxl_device_pci pcidev; + virDomainHostdevDefPtr found; + virHostdevManagerPtr hostdev_mgr; + + if (hostdev->source.subsys.type != VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_PCI) + return -1; + + if (virDomainHostdevFind(vm->def, hostdev, &found) >= 0) { + virReportError(VIR_ERR_OPERATION_FAILED, + _("target pci device %.4x:%.2x:%.2x.%.1x already exists"), + hostdev->source.subsys.u.pci.addr.domain, + hostdev->source.subsys.u.pci.addr.bus, + hostdev->source.subsys.u.pci.addr.slot, + hostdev->source.subsys.u.pci.addr.function); + return -1; + } + + if (VIR_REALLOC_N(vm->def->hostdevs, vm->def->nhostdevs + 1) < 0) + return -1; + + hostdev_mgr = virHostdevManagerGetDefault(); + if (virHostdevPreparePciHostdevs(hostdev_mgr, LIBXL_DRIVER_NAME, + vm->def->name, vm->def->uuid, + &hostdev, 1, 0) < 0) { + goto cleanup; + } + + if (libxlMakePci(hostdev, &pcidev) < 0) + goto reattach_hostdev; + + if (libxl_device_pci_add(priv->ctx, vm->def->id, &pcidev, 0) < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("libxenlight failed to attach pci device %.4x:%.2x:%.2x.%.1x"), + hostdev->source.subsys.u.pci.addr.domain, + hostdev->source.subsys.u.pci.addr.bus, + hostdev->source.subsys.u.pci.addr.slot, + hostdev->source.subsys.u.pci.addr.function); + goto reattach_hostdev; + } + + vm->def->hostdevs[vm->def->nhostdevs++] = hostdev; + return 0; + +reattach_hostdev: + virHostdevReAttachPciHostdevs(hostdev_mgr, LIBXL_DRIVER_NAME, + vm->def->name, &hostdev, 1); + +cleanup: + return -1; +} + +static int +libxlDomainAttachHostDevice(libxlDomainObjPrivatePtr priv, + virDomainObjPtr vm, + virDomainDeviceDefPtr dev) +{ + virDomainHostdevDefPtr hostdev = dev->data.hostdev; + + if (hostdev->mode != VIR_DOMAIN_HOSTDEV_MODE_SUBSYS) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, + _("hostdev mode '%s' not supported"), + virDomainHostdevModeTypeToString(hostdev->mode)); + return -1; + } + + switch (hostdev->source.subsys.type) { + case VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_PCI: + if (libxlDomainAttachHostPciDevice(priv, vm, hostdev) < 0) + goto error; + break; + + default: + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, + _("hostdev subsys type '%s' not supported"), + virDomainHostdevSubsysTypeToString(hostdev->source.subsys.type)); + goto error; + } + + return 0; + +error: + return -1; +} + +static int libxlDomainDetachDeviceDiskLive(libxlDomainObjPrivatePtr priv, virDomainObjPtr vm, virDomainDeviceDefPtr dev) { @@ -3635,6 +3745,12 @@ libxlDomainAttachDeviceLive(libxlDomainObjPrivatePtr priv, virDomainObjPtr vm, dev->data.disk = NULL; break; + case VIR_DOMAIN_DEVICE_HOSTDEV: + ret = libxlDomainAttachHostDevice(priv, vm, dev); + if (!ret) + dev->data.hostdev = NULL; + break; + default: virReportError(VIR_ERR_CONFIG_UNSUPPORTED, _("device type '%s' cannot be attached"), @@ -3649,6 +3765,8 @@ static int libxlDomainAttachDeviceConfig(virDomainDefPtr vmdef, virDomainDeviceDefPtr dev) { virDomainDiskDefPtr disk; + virDomainHostdevDefPtr hostdev; + virDomainHostdevDefPtr found; switch (dev->type) { case VIR_DOMAIN_DEVICE_DISK: @@ -3663,6 +3781,25 @@ libxlDomainAttachDeviceConfig(virDomainDefPtr vmdef, virDomainDeviceDefPtr dev) /* vmdef has the pointer. Generic codes for vmdef will do all jobs */ dev->data.disk = NULL; break; + case VIR_DOMAIN_DEVICE_HOSTDEV: + hostdev = dev->data.hostdev; + + if (hostdev->source.subsys.type != VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_PCI) + return -1; + + if (virDomainHostdevFind(vmdef, hostdev, &found) >= 0) { + virReportError(VIR_ERR_OPERATION_FAILED, + _("target pci device %.4x:%.2x:%.2x.%.1x\ + already exists"), + hostdev->source.subsys.u.pci.addr.domain, + hostdev->source.subsys.u.pci.addr.bus, + hostdev->source.subsys.u.pci.addr.slot, + hostdev->source.subsys.u.pci.addr.function); + return -1; + } + + virDomainHostdevInsert(vmdef, hostdev); + break; default: virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", @@ -3673,6 +3810,124 @@ libxlDomainAttachDeviceConfig(virDomainDefPtr vmdef, virDomainDeviceDefPtr dev) } static int +libxlComparePCIDevice(virDomainDefPtr def ATTRIBUTE_UNUSED, + virDomainDeviceDefPtr device ATTRIBUTE_UNUSED, + virDomainDeviceInfoPtr info1, + void *opaque) +{ + virDomainDeviceInfoPtr info2 = opaque; + + if (info1->type != VIR_DOMAIN_DEVICE_ADDRESS_TYPE_PCI || + info2->type != VIR_DOMAIN_DEVICE_ADDRESS_TYPE_PCI) + return 0; + + if (info1->addr.pci.domain == info2->addr.pci.domain && + info1->addr.pci.bus == info2->addr.pci.bus && + info1->addr.pci.slot == info2->addr.pci.slot && + info1->addr.pci.function != info2->addr.pci.function) + return -1; + return 0; +} + +static bool +libxlIsMultiFunctionDevice(virDomainDefPtr def, + virDomainDeviceInfoPtr dev) +{ + if (virDomainDeviceInfoIterate(def, libxlComparePCIDevice, dev) < 0) + return true; + return false; +} + +static int +libxlDomainDetachHostPciDevice(libxlDomainObjPrivatePtr priv, + virDomainObjPtr vm, + virDomainHostdevDefPtr hostdev) +{ + virDomainHostdevSubsysPtr subsys = &hostdev->source.subsys; + libxl_device_pci pcidev; + virDomainHostdevDefPtr detach; + int idx; + virHostdevManagerPtr hostdev_mgr; + + if (subsys->type != VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_PCI) + return -1; + + idx = virDomainHostdevFind(vm->def, hostdev, &detach); + if (idx < 0) { + virReportError(VIR_ERR_OPERATION_FAILED, + _("host pci device %.4x:%.2x:%.2x.%.1x not found"), + subsys->u.pci.addr.domain, subsys->u.pci.addr.bus, + subsys->u.pci.addr.slot, subsys->u.pci.addr.function); + return -1; + } + + if (libxlIsMultiFunctionDevice(vm->def, detach->info)) { + virReportError(VIR_ERR_OPERATION_FAILED, + _("cannot hot unplug multifunction PCI device: %.4x:%.2x:%.2x.%.1x"), + subsys->u.pci.addr.domain, subsys->u.pci.addr.bus, + subsys->u.pci.addr.slot, subsys->u.pci.addr.function); + goto cleanup; + } + + + libxl_device_pci_init(&pcidev); + + if (libxlMakePci(detach, &pcidev) < 0) + goto cleanup; + + if (libxl_device_pci_remove(priv->ctx, vm->def->id, &pcidev, 0) < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("libxenlight failed to detach pci device\ + %.4x:%.2x:%.2x.%.1x"), + subsys->u.pci.addr.domain, subsys->u.pci.addr.bus, + subsys->u.pci.addr.slot, subsys->u.pci.addr.function); + goto cleanup; + } + + libxl_device_pci_dispose(&pcidev); + + virDomainHostdevRemove(vm->def, idx); + + hostdev_mgr = virHostdevManagerGetDefault(); + virHostdevReAttachPciHostdevs(hostdev_mgr, LIBXL_DRIVER_NAME, + vm->def->name, &hostdev, 1); + + return 0; + +cleanup: + virDomainHostdevDefFree(detach); + return -1; +} + +static int +libxlDomainDetachHostDevice(libxlDomainObjPrivatePtr priv, + virDomainObjPtr vm, + virDomainDeviceDefPtr dev) +{ + virDomainHostdevDefPtr hostdev = dev->data.hostdev; + virDomainHostdevSubsysPtr subsys = &hostdev->source.subsys; + + if (hostdev->mode != VIR_DOMAIN_HOSTDEV_MODE_SUBSYS) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, + _("hostdev mode '%s' not supported"), + virDomainHostdevModeTypeToString(hostdev->mode)); + return -1; + } + + switch (subsys->type) { + case VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_PCI: + return libxlDomainDetachHostPciDevice(priv, vm, hostdev); + + default: + virReportError(VIR_ERR_INTERNAL_ERROR, + _("unexpected hostdev type %d"), subsys->type); + break; + } + + return -1; +} + +static int libxlDomainDetachDeviceLive(libxlDomainObjPrivatePtr priv, virDomainObjPtr vm, virDomainDeviceDefPtr dev) { @@ -3683,6 +3938,10 @@ libxlDomainDetachDeviceLive(libxlDomainObjPrivatePtr priv, virDomainObjPtr vm, ret = libxlDomainDetachDeviceDiskLive(priv, vm, dev); break; + case VIR_DOMAIN_DEVICE_HOSTDEV: + ret = libxlDomainDetachHostDevice(priv, vm, dev); + break; + default: virReportError(VIR_ERR_CONFIG_UNSUPPORTED, _("device type '%s' cannot be detached"), @@ -3693,6 +3952,7 @@ libxlDomainDetachDeviceLive(libxlDomainObjPrivatePtr priv, virDomainObjPtr vm, return ret; } + static int libxlDomainDetachDeviceConfig(virDomainDefPtr vmdef, virDomainDeviceDefPtr dev) { @@ -4951,7 +5211,7 @@ libxlConnectSupportsFeature(virConnectPtr conn, int feature) static virDriver libxlDriver = { .no = VIR_DRV_LIBXL, - .name = "xenlight", + .name = LIBXL_DRIVER_NAME, .connectOpen = libxlConnectOpen, /* 0.9.0 */ .connectClose = libxlConnectClose, /* 0.9.0 */ .connectGetType = libxlConnectGetType, /* 0.9.0 */