Re: [libvirt] [PATCH v14 49/49] add pci passthrough to libxl driver

Add pci passthrough to libxl driver, support attach-device, detach-device and start a vm with pci hostdev specified. Signed-off-by: Chunyan Liu <cyliu(a)suse.com&gt; --- src/libxl/libxl_conf.c | 63 +++++++ src/libxl/libxl_conf.h | 7 +- src/libxl/libxl_driver.c | 454 ++++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 518 insertions(+), 6 deletions(-) diff --git a/src/libxl/libxl_conf.c b/src/libxl/libxl_conf.c index ade0a08..8ba3ce3 100644 --- a/src/libxl/libxl_conf.c +++ b/src/libxl/libxl_conf.c @@ -1147,6 +1147,66 @@ libxlDriverConfigGet(libxlDriverPrivatePtr driver) return cfg; } +int +libxlMakePci(virDomainHostdevDefPtr hostdev, libxl_device_pci *pcidev) +{ + if (hostdev->mode != VIR_DOMAIN_HOSTDEV_MODE_SUBSYS) + return -1; + if (hostdev->source.subsys.type != VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_PCI) + return -1;

+ + pcidev->domain = hostdev->source.subsys.u.pci.addr.domain; + pcidev->bus = hostdev->source.subsys.u.pci.addr.bus; + pcidev->dev = hostdev->source.subsys.u.pci.addr.slot; + pcidev->func = hostdev->source.subsys.u.pci.addr.function; + + return 0; +} + +static int +libxlMakePciList(virDomainDefPtr def, libxl_domain_config *d_config) +{ + virDomainHostdevDefPtr *l_hostdevs = def->hostdevs; + size_t nhostdevs = def->nhostdevs; + size_t npcidevs = 0; + libxl_device_pci *x_pcidevs; + size_t i, j; + + if (nhostdevs == 0) + return 0; + + if (VIR_ALLOC_N(x_pcidevs, nhostdevs) < 0) + return -1; + + for (i = 0, j = 0; i < nhostdevs; i++) { + if (l_hostdevs[i]->mode != VIR_DOMAIN_HOSTDEV_MODE_SUBSYS) + continue; + if (l_hostdevs[i]->source.subsys.type != VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_PCI) + continue; + + libxl_device_pci_init(&x_pcidevs[j]); + + if (libxlMakePci(l_hostdevs[i], &x_pcidevs[j]) < 0) + goto error; + + npcidevs++; + j++; + } + + VIR_SHRINK_N(x_pcidevs, nhostdevs, nhostdevs - npcidevs); + d_config->pcidevs = x_pcidevs; + d_config->num_pcidevs = npcidevs; + + return 0; + +error: + for (i = 0; i < npcidevs; i++) + libxl_device_pci_dispose(&x_pcidevs[i]); + + VIR_FREE(x_pcidevs); + return -1; +} + virCapsPtr libxlMakeCapabilities(libxl_ctx *ctx) { @@ -1195,6 +1255,9 @@ libxlBuildDomainConfig(libxlDriverPrivatePtr driver, if (libxlMakeVfbList(driver, def, d_config) < 0) return -1; + if (libxlMakePciList(def, d_config) < 0) + return -1; + d_config->on_reboot = def->onReboot; d_config->on_poweroff = def->onPoweroff; d_config->on_crash = def->onCrash; diff --git a/src/libxl/libxl_conf.h b/src/libxl/libxl_conf.h index f089a92..4f6f7ce 100644 --- a/src/libxl/libxl_conf.h +++ b/src/libxl/libxl_conf.h @@ -37,8 +37,9 @@ # include "virportallocator.h" # include "virobject.h" # include "virchrdev.h" +# include "virhostdev.h" - +# define LIBXL_DRIVER_NAME "xenlight" # define LIBXL_VNC_PORT_MIN 5900 # define LIBXL_VNC_PORT_MAX 65535 @@ -90,6 +91,7 @@ struct _libxlDriverConfig { struct _libxlDriverPrivate { virMutex lock; + virHostdevManagerPtr hostdevMgr; /* Require lock to get reference on 'config', * then lockless thereafter */ libxlDriverConfigPtr config; @@ -150,6 +152,9 @@ libxlMakeVfb(libxlDriverPrivatePtr driver, virDomainGraphicsDefPtr l_vfb, libxl_device_vfb *x_vfb); int +libxlMakePci(virDomainHostdevDefPtr hostdev, libxl_device_pci *pcidev); + +int libxlBuildDomainConfig(libxlDriverPrivatePtr driver, virDomainObjPtr vm, libxl_domain_config *d_config); diff --git a/src/libxl/libxl_driver.c b/src/libxl/libxl_driver.c index a79efcb..88a0a90 100644 --- a/src/libxl/libxl_driver.c +++ b/src/libxl/libxl_driver.c @@ -52,6 +52,7 @@ #include "virsysinfo.h" #include "viraccessapicheck.h" #include "viratomic.h" +#include "virhostdev.h" #define VIR_FROM_THIS VIR_FROM_LIBXL @@ -309,6 +310,10 @@ libxlVmCleanup(libxlDriverPrivatePtr driver, int vnc_port; char *file; size_t i; + virHostdevManagerPtr hostdev_mgr = driver->hostdevMgr; + + virHostdevReAttachDomainHostdevs(hostdev_mgr, LIBXL_DRIVER_NAME, + vm->def, VIR_HOSTDEV_SP_PCI, NULL); vm->def->id = -1; @@ -699,6 +704,7 @@ libxlVmStart(libxlDriverPrivatePtr driver, virDomainObjPtr vm, #ifdef LIBXL_HAVE_DOMAIN_CREATE_RESTORE_PARAMS libxl_domain_restore_params params; #endif + virHostdevManagerPtr hostdev_mgr = driver->hostdevMgr; if (libxlDomainObjPrivateInitCtx(vm) < 0) return ret; @@ -761,6 +767,10 @@ libxlVmStart(libxlDriverPrivatePtr driver, virDomainObjPtr vm, goto endjob; } + if (virHostdevPrepareDomainHostdevs(hostdev_mgr, LIBXL_DRIVER_NAME, + vm->def, VIR_HOSTDEV_SP_PCI) < 0) + goto endjob; + /* Unlock virDomainObj while creating the domain */ virObjectUnlock(vm); if (restore_fd < 0) { @@ -867,6 +877,7 @@ libxlReconnectDomain(virDomainObjPtr vm, libxl_dominfo d_info; int len; uint8_t *data = NULL; + virHostdevManagerPtr hostdev_mgr = driver->hostdevMgr; virObjectLock(vm); @@ -890,6 +901,12 @@ libxlReconnectDomain(virDomainObjPtr vm, /* Update domid in case it changed (e.g. reboot) while we were gone? */ vm->def->id = d_info.domid; + + /* Update hostdev state */ + if (virHostdevUpdateDomainActiveHostdevs(hostdev_mgr, LIBXL_DRIVER_NAME, + vm->def, VIR_HOSTDEV_SP_PCI) < 0) + goto out; + virDomainObjSetState(vm, VIR_DOMAIN_RUNNING, VIR_DOMAIN_RUNNING_UNKNOWN); if (virAtomicIntInc(&driver->nactive) == 1 && driver->inhibitCallback) @@ -922,6 +939,7 @@ libxlStateCleanup(void) if (!libxl_driver) return -1; + virObjectUnref(libxl_driver->hostdevMgr); virObjectUnref(libxl_driver->config); virObjectUnref(libxl_driver->xmlopt); virObjectUnref(libxl_driver->domains); @@ -1001,6 +1019,9 @@ libxlStateInitialize(bool privileged, if (!(libxl_driver->domains = virDomainObjListNew())) goto error; + if (!(libxl_driver->hostdevMgr = virHostdevManagerGetDefault())) + goto error; + if (!(cfg = libxlDriverConfigNew())) goto error; @@ -3274,6 +3295,95 @@ cleanup: } static int +libxlDomainAttachHostPciDevice(libxlDriverPrivatePtr driver, + libxlDomainObjPrivatePtr priv, + virDomainObjPtr vm, + virDomainHostdevDefPtr hostdev) +{ + libxl_device_pci pcidev; + virDomainHostdevDefPtr found; + virHostdevManagerPtr hostdev_mgr = driver->hostdevMgr; + + if (hostdev->source.subsys.type != VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_PCI) + return -1; + + if (virDomainHostdevFind(vm->def, hostdev, &found) >= 0) { + virReportError(VIR_ERR_OPERATION_FAILED, + _("target pci device %.4x:%.2x:%.2x.%.1x already exists"), + hostdev->source.subsys.u.pci.addr.domain, + hostdev->source.subsys.u.pci.addr.bus, + hostdev->source.subsys.u.pci.addr.slot, + hostdev->source.subsys.u.pci.addr.function); + return -1; + } + + if (VIR_REALLOC_N(vm->def->hostdevs, vm->def->nhostdevs + 1) < 0) + return -1; + + if (virHostdevPreparePciHostdevs(hostdev_mgr, LIBXL_DRIVER_NAME, + vm->def->name, vm->def->uuid, + &hostdev, 1, 0) < 0) + goto cleanup; + + if (libxlMakePci(hostdev, &pcidev) < 0) + goto reattach_hostdev; + + if (libxl_device_pci_add(priv->ctx, vm->def->id, &pcidev, 0) < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("libxenlight failed to attach pci device %.4x:%.2x:%.2x.%.1x"), + hostdev->source.subsys.u.pci.addr.domain, + hostdev->source.subsys.u.pci.addr.bus, + hostdev->source.subsys.u.pci.addr.slot, + hostdev->source.subsys.u.pci.addr.function); + goto reattach_hostdev; + } + + vm->def->hostdevs[vm->def->nhostdevs++] = hostdev; + return 0; + +reattach_hostdev: + virHostdevReAttachPciHostdevs(hostdev_mgr, LIBXL_DRIVER_NAME, + vm->def->name, &hostdev, 1, NULL); + +cleanup: + return -1;

+} + +static int +libxlDomainAttachHostDevice(libxlDriverPrivatePtr driver, + libxlDomainObjPrivatePtr priv, + virDomainObjPtr vm, + virDomainDeviceDefPtr dev) +{ + virDomainHostdevDefPtr hostdev = dev->data.hostdev; + + if (hostdev->mode != VIR_DOMAIN_HOSTDEV_MODE_SUBSYS) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, + _("hostdev mode '%s' not supported"), + virDomainHostdevModeTypeToString(hostdev->mode)); + return -1; + } + + switch (hostdev->source.subsys.type) { + case VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_PCI: + if (libxlDomainAttachHostPciDevice(driver, priv, vm, hostdev) < 0) + goto error; + break; + + default: + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, + _("hostdev subsys type '%s' not supported"), + virDomainHostdevSubsysTypeToString(hostdev->source.subsys.type)); + goto error; + } + + return 0; + +error: + return -1;