Re: [libvirt] [PATCH v4 4/4] gluster: cache glfs connection object per volume

Currently, in case if we have 4 extra attached disks, then for each disk we need to call 'glfs_init' (over network) and friends which could be costly. Additionally snapshot(external) scenario will further complex the situation.

This patch maintain a cache of glfs objects per volume, hence the all disks/snapshots belonging to the volume whose glfs object is cached can avoid calls to 'glfs_init' and friends by simply taking a ref on existing object. The glfs object is shared only if volume name and all the volfile servers match (includes hostname, transport and port number). Thanks to 'Peter Krempa' for all the inputs. Signed-off-by: Prasanna Kumar Kalever <prasanna.kalever(a)redhat.com&gt; --- src/storage/storage_backend_gluster.c | 279 +++++++++++++++++++++++++++++++--- 1 file changed, 254 insertions(+), 25 deletions(-) diff --git a/src/storage/storage_backend_gluster.c b/src/storage/storage_backend_gluster.c index e0841ca..9f633ac 100644 --- a/src/storage/storage_backend_gluster.c +++ b/src/storage/storage_backend_gluster.c @@ -36,6 +36,20 @@ VIR_LOG_INIT("storage.storage_backend_gluster"); + +typedef struct _virStorageBackendGlusterState virStorageBackendGlusterState; +typedef virStorageBackendGlusterState *virStorageBackendGlusterStatePtr; + +typedef struct _virStorageFileBackendGlusterPriv virStorageFileBackendGlusterPriv; +typedef virStorageFileBackendGlusterPriv *virStorageFileBackendGlusterPrivPtr; + +typedef struct _virStorageBackendGlusterCache virStorageBackendGlusterCache; +typedef virStorageBackendGlusterCache *virStorageBackendGlusterCachePtr; + +typedef struct _virStorageBackendGlusterCacheConn virStorageBackendGlusterCacheConn; +typedef virStorageBackendGlusterCacheConn *virStorageBackendGlusterCacheConnPtr; + + struct _virStorageBackendGlusterState { glfs_t *vol; @@ -47,8 +61,205 @@ struct _virStorageBackendGlusterState { char *dir; /* dir from URI, or "/"; always starts and ends in '/' */ }; -typedef struct _virStorageBackendGlusterState virStorageBackendGlusterState; -typedef virStorageBackendGlusterState *virStorageBackendGlusterStatePtr; +struct _virStorageFileBackendGlusterPriv { + glfs_t *vol;

+ char *canonpath; +}; + +struct _virStorageBackendGlusterCacheConn { + virObjectLockable parent; + virStorageFileBackendGlusterPrivPtr priv;

+ size_t nhosts; + virStorageNetHostDefPtr hosts; + char *volname; +}; + +struct _virStorageBackendGlusterCache { + virMutex lock; + size_t nConn; + virStorageBackendGlusterCacheConnPtr *conn; +}; + + +static virStorageBackendGlusterCachePtr virStorageBackendGlusterGlobalCacheObj; +static virClassPtr virStorageBackendGlusterCacheConnClass; +static virOnceControl virStorageBackendGlusterCacheOnce = VIR_ONCE_CONTROL_INITIALIZER; +static bool virStorageBackendGlusterCacheInitGlobalError; +static bool virStorageBackendGlusterCacheConnCleaned;

+ + +static void virStorageBackendGlusterCacheConnDispose(void *obj); + + +static int virStorageBackendGlusterCacheConnOnceInit(void) +{ + if (!(virStorageBackendGlusterCacheConnClass = + virClassNew(virClassForObjectLockable(), + "virStorageBackendGlusterCacheConnClass", + sizeof(virStorageBackendGlusterCacheConn), + virStorageBackendGlusterCacheConnDispose)))

+ return -1; + + return 0; +} + +static void virStorageBackendGlusterCacheConnDispose(void *object) +{ + virStorageBackendGlusterCacheConnPtr conn = object; + + glfs_fini(conn->priv->vol); + + VIR_FREE(conn->priv->canonpath); + VIR_FREE(conn->priv); + VIR_FREE(conn->volname); + + virStorageNetHostDefFree(conn->nhosts, conn->hosts); + + /* all set to delete the connection from cache */ + virStorageBackendGlusterCacheConnCleaned = true; +} + +VIR_ONCE_GLOBAL_INIT(virStorageBackendGlusterCacheConn); + +static bool +virStorageBackendGlusterCompareHosts(size_t nSrcHosts, + virStorageNetHostDefPtr srcHosts, + size_t nDstHosts, + virStorageNetHostDefPtr dstHosts) +{ + bool match = false; + size_t i; + size_t j; + + if (nSrcHosts != nDstHosts) + return false; + for (i = 0; i < nSrcHosts; i++) { + for (j = 0; j < nDstHosts; j++) { + if (srcHosts[i].type != dstHosts[j].type) + continue; + switch (srcHosts[i].type) { + case VIR_STORAGE_NET_HOST_TRANS_UNIX: + if (STREQ(srcHosts[i].u.uds.path, dstHosts[j].u.uds.path)) + match = true; + break; + case VIR_STORAGE_NET_HOST_TRANS_TCP: + case VIR_STORAGE_NET_HOST_TRANS_RDMA: + if (STREQ(srcHosts[i].u.inet.addr, dstHosts[j].u.inet.addr) + && + STREQ(srcHosts[i].u.inet.port, dstHosts[j].u.inet.port)) + match = true; + break; + case VIR_STORAGE_NET_HOST_TRANS_LAST: + break; + } + if (match) + break; /* out of inner for loop */ + } + if (!match) + return false; + match = false; /* reset */ + } + return true; +} + +static int +virStorageBackendGlusterCacheAdd(virStorageSourcePtr src, + virStorageFileBackendGlusterPrivPtr priv) +{ + virStorageBackendGlusterCacheConnPtr newConn = NULL; + virStorageBackendGlusterCacheConnPtr conn = NULL; + size_t i; + + for (i = 0; i < virStorageBackendGlusterGlobalCacheObj->nConn; i++) {

+ conn = virStorageBackendGlusterGlobalCacheObj->conn[i]; + if (STRNEQ(conn->volname, src->volume)) + continue; + if (virStorageBackendGlusterCompareHosts(conn->nhosts, conn->hosts, + src->nhosts, src->hosts)) { + return 0; + } + } + + if (virStorageBackendGlusterCacheConnInitialize() < 0) + return -1;

+ + if (!(newConn = virObjectLockableNew(virStorageBackendGlusterCacheConnClass))) + return -1; + + if (VIR_STRDUP(newConn->volname, src->volume) < 0) + goto error; + + newConn->nhosts = src->nhosts; + if (!(newConn->hosts = virStorageNetHostDefCopy(src->nhosts, src->hosts))) + goto error; + newConn->priv = priv; + + if (VIR_APPEND_ELEMENT(virStorageBackendGlusterGlobalCacheObj->conn, + virStorageBackendGlusterGlobalCacheObj->nConn, + newConn) < 0) + goto error; + + return 0; + + error: + virStorageNetHostDefFree(newConn->nhosts, newConn->hosts); + VIR_FREE(newConn->volname);

+ return -1; +} + +static glfs_t * +virStorageBackendGlusterCacheQuery(virStorageSourcePtr src)

+{ + virStorageBackendGlusterCacheConnPtr conn; + size_t i; + + if (virStorageBackendGlusterGlobalCacheObj == NULL) + return NULL; + + for (i = 0; i < virStorageBackendGlusterGlobalCacheObj->nConn; i++) { + conn = virStorageBackendGlusterGlobalCacheObj->conn[i]; + if (STRNEQ(conn->volname, src->volume)) + continue; + if (virStorageBackendGlusterCompareHosts(conn->nhosts, conn->hosts, + src->nhosts, src->hosts)) { + virObjectRef(conn); + return conn->priv->vol; + } + } + + return NULL; +} + +static void +virStorageBackendGlusterCacheRefresh(virStorageSourcePtr src)

+{ + virStorageFileBackendGlusterPrivPtr priv = src->drv->priv; + virStorageBackendGlusterCacheConnPtr conn = NULL; + size_t i; + + virMutexLock(&virStorageBackendGlusterGlobalCacheObj->lock); + for (i = 0; i < virStorageBackendGlusterGlobalCacheObj->nConn; i++) { + conn = virStorageBackendGlusterGlobalCacheObj->conn[i]; + if (STRNEQ(conn->volname, src->volume)) + continue; + if (virStorageBackendGlusterCompareHosts(conn->nhosts, conn->hosts, + src->nhosts, src->hosts)) { + virObjectUnref(conn); + if (!virStorageBackendGlusterCacheConnCleaned) {

+ if (priv && priv->canonpath) + VIR_FREE(priv->canonpath); + goto unlock; + } + virStorageBackendGlusterCacheConnCleaned = false; + VIR_DELETE_ELEMENT(virStorageBackendGlusterGlobalCacheObj->conn, i, + virStorageBackendGlusterGlobalCacheObj->nConn); + VIR_FREE(src->drv);

+ } + } + + unlock: + virMutexUnlock(&virStorageBackendGlusterGlobalCacheObj->lock); +} static void virStorageBackendGlusterClose(virStorageBackendGlusterStatePtr state) @@ -538,32 +749,15 @@ virStorageBackend virStorageBackendGluster = { }; -typedef struct _virStorageFileBackendGlusterPriv virStorageFileBackendGlusterPriv; -typedef virStorageFileBackendGlusterPriv *virStorageFileBackendGlusterPrivPtr; - -struct _virStorageFileBackendGlusterPriv { - glfs_t *vol; - char *canonpath; -}; - - static void virStorageFileBackendGlusterDeinit(virStorageSourcePtr src) { - virStorageFileBackendGlusterPrivPtr priv = src->drv->priv; - VIR_DEBUG("deinitializing gluster storage file %p (gluster://%s:%s/%s%s)", src, src->hosts->u.inet.addr, src->hosts->u.inet.port ? src->hosts->u.inet.port : "0", src->volume, src->path); - if (priv->vol) - glfs_fini(priv->vol); - VIR_FREE(priv->canonpath); - - VIR_FREE(priv); - - VIR_FREE(src->drv); + virStorageBackendGlusterCacheRefresh(src); } static int @@ -610,6 +804,20 @@ virStorageFileBackendGlusterInitServer(virStorageFileBackendGlusterPrivPtr priv, return 0; } +static void +virStorageBackendGlusterCacheInit(void) +{ + if (VIR_ALLOC(virStorageBackendGlusterGlobalCacheObj) < 0) { + virStorageBackendGlusterCacheInitGlobalError = false; + return; + } + + if (virMutexInit(&virStorageBackendGlusterGlobalCacheObj->lock) < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("Unable to initialize mutex")); + virStorageBackendGlusterCacheInitGlobalError = false; + } +} static int virStorageFileBackendGlusterInit(virStorageSourcePtr src) @@ -632,11 +840,32 @@ virStorageFileBackendGlusterInit(virStorageSourcePtr src) src, priv, src->volume, src->path, (unsigned int)src->drv->uid, (unsigned int)src->drv->gid); + if (virOnce(&virStorageBackendGlusterCacheOnce, + virStorageBackendGlusterCacheInit) < 0) + return -1; + + if (virStorageBackendGlusterCacheInitGlobalError) + return -1; + + virMutexLock(&virStorageBackendGlusterGlobalCacheObj->lock); + + priv->vol = virStorageBackendGlusterCacheQuery(src); + if (priv->vol) { + src->drv->priv = priv; + virMutexUnlock(&virStorageBackendGlusterGlobalCacheObj->lock); + return 0; + } + if (!(priv->vol = glfs_new(src->volume))) { virReportOOMError(); - goto error; + goto unlock; } + if (virStorageBackendGlusterCacheAdd(src, priv) < 0) + goto unlock; + + virMutexUnlock(&virStorageBackendGlusterGlobalCacheObj->lock); +

for (i = 0; i < src->nhosts; i++) { if (virStorageFileBackendGlusterInitServer(priv, src->hosts + i) < 0) goto error;