Re: [libvirt] [PATCH v14 49/49] add pci passthrough to libxl driver

10 Mar 2014

Chunyan Liu wrote:
...
Add pci passthrough to libxl driver, support attach-device, detach-device and
start a vm with pci hostdev specified.
Signed-off-by: Chunyan Liu <cyliu@suse.com>
---
 src/libxl/libxl_conf.c   |  63 +++++++
 src/libxl/libxl_conf.h   |   7 +-
 src/libxl/libxl_driver.c | 454 ++++++++++++++++++++++++++++++++++++++++++++++-
 3 files changed, 518 insertions(+), 6 deletions(-)

diff --git a/src/libxl/libxl_conf.c b/src/libxl/libxl_conf.c
index ade0a08..8ba3ce3 100644
--- a/src/libxl/libxl_conf.c
+++ b/src/libxl/libxl_conf.c
@@ -1147,6 +1147,66 @@ libxlDriverConfigGet(libxlDriverPrivatePtr driver)
     return cfg;
 }
+int
+libxlMakePci(virDomainHostdevDefPtr hostdev, libxl_device_pci *pcidev)
+{
+    if (hostdev->mode != VIR_DOMAIN_HOSTDEV_MODE_SUBSYS)
+        return -1;
+    if (hostdev->source.subsys.type != VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_PCI)
+        return -1;
Sorry, I should have mentioned this in my v13 review, but we should
report an error here, similar to libxlMake{Disk,Nic}.
...
+
+    pcidev->domain = hostdev->source.subsys.u.pci.addr.domain;
+    pcidev->bus = hostdev->source.subsys.u.pci.addr.bus;
+    pcidev->dev = hostdev->source.subsys.u.pci.addr.slot;
+    pcidev->func = hostdev->source.subsys.u.pci.addr.function;
+
+    return 0;
+}
+
+static int
+libxlMakePciList(virDomainDefPtr def, libxl_domain_config *d_config)
+{
+    virDomainHostdevDefPtr *l_hostdevs = def->hostdevs;
+    size_t nhostdevs = def->nhostdevs;
+    size_t npcidevs = 0;
+    libxl_device_pci *x_pcidevs;
+    size_t i, j;
+
+    if (nhostdevs == 0)
+        return 0;
+
+    if (VIR_ALLOC_N(x_pcidevs, nhostdevs) < 0)
+        return -1;
+
+    for (i = 0, j = 0; i < nhostdevs; i++) {
+        if (l_hostdevs[i]->mode != VIR_DOMAIN_HOSTDEV_MODE_SUBSYS)
+            continue;
+        if (l_hostdevs[i]->source.subsys.type != VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_PCI)
+            continue;
+
+        libxl_device_pci_init(&x_pcidevs[j]);
+
+        if (libxlMakePci(l_hostdevs[i], &x_pcidevs[j]) < 0)
+            goto error;
+
+        npcidevs++;
+        j++;
+    }
+
+    VIR_SHRINK_N(x_pcidevs, nhostdevs, nhostdevs - npcidevs);
+    d_config->pcidevs = x_pcidevs;
+    d_config->num_pcidevs = npcidevs;
+
+    return 0;
+
+error:
+    for (i = 0; i < npcidevs; i++)
+        libxl_device_pci_dispose(&x_pcidevs[i]);
+
+    VIR_FREE(x_pcidevs);
+    return -1;
+}
+
 virCapsPtr
 libxlMakeCapabilities(libxl_ctx *ctx)
 {
@@ -1195,6 +1255,9 @@ libxlBuildDomainConfig(libxlDriverPrivatePtr driver,
     if (libxlMakeVfbList(driver, def, d_config) < 0)
         return -1;
+    if (libxlMakePciList(def, d_config) < 0)
+        return -1;
+
     d_config->on_reboot = def->onReboot;
     d_config->on_poweroff = def->onPoweroff;
     d_config->on_crash = def->onCrash;
diff --git a/src/libxl/libxl_conf.h b/src/libxl/libxl_conf.h
index f089a92..4f6f7ce 100644
--- a/src/libxl/libxl_conf.h
+++ b/src/libxl/libxl_conf.h
@@ -37,8 +37,9 @@
 # include "virportallocator.h"
 # include "virobject.h"
 # include "virchrdev.h"
+# include "virhostdev.h"
-
+# define LIBXL_DRIVER_NAME "xenlight"
 # define LIBXL_VNC_PORT_MIN  5900
 # define LIBXL_VNC_PORT_MAX  65535
@@ -90,6 +91,7 @@ struct _libxlDriverConfig {
 struct _libxlDriverPrivate {
     virMutex lock;
+    virHostdevManagerPtr hostdevMgr;
     /* Require lock to get reference on 'config',
      * then lockless thereafter */
     libxlDriverConfigPtr config;
@@ -150,6 +152,9 @@ libxlMakeVfb(libxlDriverPrivatePtr driver,
              virDomainGraphicsDefPtr l_vfb, libxl_device_vfb *x_vfb);
int
+libxlMakePci(virDomainHostdevDefPtr hostdev, libxl_device_pci *pcidev);
+
+int
 libxlBuildDomainConfig(libxlDriverPrivatePtr driver,
                        virDomainObjPtr vm, libxl_domain_config *d_config);
diff --git a/src/libxl/libxl_driver.c b/src/libxl/libxl_driver.c
index a79efcb..88a0a90 100644
--- a/src/libxl/libxl_driver.c
+++ b/src/libxl/libxl_driver.c
@@ -52,6 +52,7 @@
 #include "virsysinfo.h"
 #include "viraccessapicheck.h"
 #include "viratomic.h"
+#include "virhostdev.h"
#define VIR_FROM_THIS VIR_FROM_LIBXL
@@ -309,6 +310,10 @@ libxlVmCleanup(libxlDriverPrivatePtr driver,
     int vnc_port;
     char *file;
     size_t i;
+    virHostdevManagerPtr hostdev_mgr = driver->hostdevMgr;
+
+    virHostdevReAttachDomainHostdevs(hostdev_mgr, LIBXL_DRIVER_NAME,
+                                     vm->def, VIR_HOSTDEV_SP_PCI, NULL);
vm->def->id = -1;
@@ -699,6 +704,7 @@ libxlVmStart(libxlDriverPrivatePtr driver, virDomainObjPtr vm,
 #ifdef LIBXL_HAVE_DOMAIN_CREATE_RESTORE_PARAMS
     libxl_domain_restore_params params;
 #endif
+    virHostdevManagerPtr hostdev_mgr = driver->hostdevMgr;
if (libxlDomainObjPrivateInitCtx(vm) < 0)
         return ret;
@@ -761,6 +767,10 @@ libxlVmStart(libxlDriverPrivatePtr driver, virDomainObjPtr vm,
         goto endjob;
     }
+    if (virHostdevPrepareDomainHostdevs(hostdev_mgr, LIBXL_DRIVER_NAME,
+                                        vm->def, VIR_HOSTDEV_SP_PCI) < 0)
+        goto endjob;
+
     /* Unlock virDomainObj while creating the domain */
     virObjectUnlock(vm);
     if (restore_fd < 0) {
@@ -867,6 +877,7 @@ libxlReconnectDomain(virDomainObjPtr vm,
     libxl_dominfo d_info;
     int len;
     uint8_t *data = NULL;
+    virHostdevManagerPtr hostdev_mgr = driver->hostdevMgr;
virObjectLock(vm);
@@ -890,6 +901,12 @@ libxlReconnectDomain(virDomainObjPtr vm,
/* Update domid in case it changed (e.g. reboot) while we were gone? */
     vm->def->id = d_info.domid;
+
+    /* Update hostdev state */
+    if (virHostdevUpdateDomainActiveHostdevs(hostdev_mgr, LIBXL_DRIVER_NAME,
+                                             vm->def, VIR_HOSTDEV_SP_PCI) < 0)
+        goto out;
+
     virDomainObjSetState(vm, VIR_DOMAIN_RUNNING, VIR_DOMAIN_RUNNING_UNKNOWN);
if (virAtomicIntInc(&driver->nactive) == 1 && driver->inhibitCallback)
@@ -922,6 +939,7 @@ libxlStateCleanup(void)
     if (!libxl_driver)
         return -1;
+    virObjectUnref(libxl_driver->hostdevMgr);
     virObjectUnref(libxl_driver->config);
     virObjectUnref(libxl_driver->xmlopt);
     virObjectUnref(libxl_driver->domains);
@@ -1001,6 +1019,9 @@ libxlStateInitialize(bool privileged,
     if (!(libxl_driver->domains = virDomainObjListNew()))
         goto error;
+    if (!(libxl_driver->hostdevMgr = virHostdevManagerGetDefault()))
+        goto error;
+
     if (!(cfg = libxlDriverConfigNew()))
         goto error;
@@ -3274,6 +3295,95 @@ cleanup:
 }
static int
+libxlDomainAttachHostPciDevice(libxlDriverPrivatePtr driver,
+                               libxlDomainObjPrivatePtr priv,
+                               virDomainObjPtr vm,
+                               virDomainHostdevDefPtr hostdev)
+{
+    libxl_device_pci pcidev;
+    virDomainHostdevDefPtr found;
+    virHostdevManagerPtr hostdev_mgr = driver->hostdevMgr;
+
+    if (hostdev->source.subsys.type != VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_PCI)
+        return -1;
+
+    if (virDomainHostdevFind(vm->def, hostdev, &found) >= 0) {
+        virReportError(VIR_ERR_OPERATION_FAILED,
+                       _("target pci device %.4x:%.2x:%.2x.%.1x already exists"),
+                       hostdev->source.subsys.u.pci.addr.domain,
+                       hostdev->source.subsys.u.pci.addr.bus,
+                       hostdev->source.subsys.u.pci.addr.slot,
+                       hostdev->source.subsys.u.pci.addr.function);
+        return -1;
+    }
+
+    if (VIR_REALLOC_N(vm->def->hostdevs, vm->def->nhostdevs + 1) < 0)
+        return -1;
+
+    if (virHostdevPreparePciHostdevs(hostdev_mgr, LIBXL_DRIVER_NAME,
+                                     vm->def->name, vm->def->uuid,
+                                     &hostdev, 1, 0) < 0)
+        goto cleanup;
+
+    if (libxlMakePci(hostdev, &pcidev) < 0)
+        goto reattach_hostdev;
+
+    if (libxl_device_pci_add(priv->ctx, vm->def->id, &pcidev, 0) < 0) {
+        virReportError(VIR_ERR_INTERNAL_ERROR,
+                       _("libxenlight failed to attach pci device %.4x:%.2x:%.2x.%.1x"),
+                       hostdev->source.subsys.u.pci.addr.domain,
+                       hostdev->source.subsys.u.pci.addr.bus,
+                       hostdev->source.subsys.u.pci.addr.slot,
+                       hostdev->source.subsys.u.pci.addr.function);
+        goto reattach_hostdev;
+    }
+
+    vm->def->hostdevs[vm->def->nhostdevs++] = hostdev;
+    return 0;
+
+reattach_hostdev:
+    virHostdevReAttachPciHostdevs(hostdev_mgr, LIBXL_DRIVER_NAME,
+                                  vm->def->name, &hostdev, 1, NULL);
+
+cleanup:
+    return -1;
Looks like you missed my comments from v13

https://www.redhat.com/archives/libvir-list/2014-March/msg00170.html
...
+}
+
+static int
+libxlDomainAttachHostDevice(libxlDriverPrivatePtr driver,
+                            libxlDomainObjPrivatePtr priv,
+                            virDomainObjPtr vm,
+                            virDomainDeviceDefPtr dev)
+{
+    virDomainHostdevDefPtr hostdev = dev->data.hostdev;
+
+    if (hostdev->mode != VIR_DOMAIN_HOSTDEV_MODE_SUBSYS) {
+        virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
+                       _("hostdev mode '%s' not supported"),
+                       virDomainHostdevModeTypeToString(hostdev->mode));
+        return -1;
+    }
+
+    switch (hostdev->source.subsys.type) {
+    case VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_PCI:
+        if (libxlDomainAttachHostPciDevice(driver, priv, vm, hostdev) < 0)
+            goto error;
+        break;
+
+    default:
+        virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
+                       _("hostdev subsys type '%s' not supported"),
+                       virDomainHostdevSubsysTypeToString(hostdev->source.subsys.type));
+        goto error;
+    }
+
+    return 0;
+
+error:
+    return -1;
Ah, same here, and I probably throughout the rest of the patch -
although I didn't check. I guess you missed my message :).

Regards,
Jim

    

Re: [libvirt] [PATCH v14 49/49] add pci passthrough to libxl driver

Jim Fehlig