Implement a LINSTOR backend storage driver. The Linstor client needs to be installed and it needs to be configured on the nodes used by the controller. It supports most pool/vol commands, except for pool-build/pool-delete and provides a block device in RAW file mode. Linstor supports more than just DRBD so it would also be possible to have it provide LVM, ZFS or NVME volumes, but the common case will be to provide DRBD volumes in a cluster. Sample pool XML: <pool type='linstor'> <name>linstor</name> <source> <host name='ubuntu-focal-60'/> <name>libvirtgrp</name> </source> </pool> <pool/source/name> element must point to an already created LINSTOR resource-group, which is used to spawn resources/volumes. <pool/source/host@name> attribute should be the local linstor node name, if missing it will try to get the hosts uname and use that instead. Result volume XML sample: <volume type='block'> <name>alpine12</name> <key>libvirtgrp/alpine12</key> <capacity unit='bytes'>5368709120</capacity> <allocation unit='bytes'>5540028416</allocation> <target> <path>/dev/drbd1000</path> <format type='raw'/> </target> </volume> Signed-off-by: Rene Peinthor <rene.peinthor(a)linbit.com&gt; --- docs/storage.html.in | 39 + meson.build | 6 + meson_options.txt | 1 + po/POTFILES.in | 1 + src/storage/meson.build | 25 + src/storage/storage_backend.c | 6 + src/storage/storage_backend_linstor.c | 783 +++++++++++++++++++++ src/storage/storage_backend_linstor.h | 24 + src/storage/storage_backend_linstor_priv.h | 53 ++ 9 files changed, 938 insertions(+) create mode 100644 src/storage/storage_backend_linstor.c create mode 100644 src/storage/storage_backend_linstor.h create mode 100644 src/storage/storage_backend_linstor_priv.h diff --git a/docs/storage.html.in b/docs/storage.html.in index b2cf343933..9130fbd180 100644 --- a/docs/storage.html.in +++ b/docs/storage.html.in @@ -829,5 +829,44 @@ <h3>Valid volume format types</h3> <p>The valid volume types are the same as for the directory pool.</p> + + + <h2><a id="StorageBackendLINSTOR">LINSTOR pool</a></h2> + <p> + This provides a pool using the LINSTOR software-defined-storage. + LINSTOR can provide block storage devices based on DRBD or basic + LVM/ZFS volumes. + </p> + + <p> + To use LINSTOR in libvirt, setup a working LINSTOR cluster, documentation + for that is in the LINSTOR Users-guide. + And create a resource-group that will be used by libvirt, also make sure + the resource-group is setup in a way so that all nodes you want to use with libvirt + will create a resource. So either use diskless-on-remaining or make sure + replica-count is the same as you have nodes in your cluster. + </p> + + <p><span class="since">Since 7.1.0</span></p>. + + <h3>Example pool input</h3> + <pre> + &lt;pool type="linstor"&gt; + &lt;name&gt;linstorpool&lt;/name&gt; + &lt;source&gt; + &lt;name&gt;libvirtrscgrp&lt;/name&gt; + &lt;host name="linstornode"&gt;/&gt; + &lt;/source&gt; + &lt;/pool&gt;</pre> + + <h3>Valid pool format types</h3> + <p> + The LINSTOR volume pool does not use the pool format type element. + </p> + + <h3>Valid volume format types</h3> + <p> + The LINSTOR volume pool does not use the volume format type element. + </p> </body> </html> diff --git a/meson.build b/meson.build index 095d6ca664..3406d964f7 100644 --- a/meson.build +++ b/meson.build @@ -1901,6 +1901,11 @@ if conf.has('WITH_LIBVIRTD') error('Need libiscsi for iscsi-direct storage driver') endif + if not get_option('storage_linstor').disabled() + use_storage = true + conf.set('WITH_STORAGE_LINSTOR', 1) + endif + if not get_option('storage_lvm').disabled() lvm_enable = true lvm_progs = [ @@ -2311,6 +2316,7 @@ storagedriver_summary = { 'Dir': conf.has('WITH_STORAGE_DIR'), 'FS': conf.has('WITH_STORAGE_FS'), 'NetFS': conf.has('WITH_STORAGE_FS'), + 'Linstor': conf.has('WITH_STORAGE_LINSTOR'), 'LVM': conf.has('WITH_STORAGE_LVM'), 'iSCSI': conf.has('WITH_STORAGE_ISCSI'), 'iscsi-direct': conf.has('WITH_STORAGE_ISCSI_DIRECT'), diff --git a/meson_options.txt b/meson_options.txt index e5d79c2b6b..247d88e0ee 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -79,6 +79,7 @@ option('storage_fs', type: 'feature', value: 'auto', description: 'FileSystem ba option('storage_gluster', type: 'feature', value: 'auto', description: 'Gluster backend for the storage driver') option('storage_iscsi', type: 'feature', value: 'auto', description: 'iscsi backend for the storage driver') option('storage_iscsi_direct', type: 'feature', value: 'auto', description: 'iscsi-direct backend for the storage driver') +option('storage_linstor', type: 'feature', value: 'auto', description: 'Linstor backend for the storage driver') option('storage_lvm', type: 'feature', value: 'auto', description: 'LVM backend for the storage driver') option('storage_mpath', type: 'feature', value: 'auto', description: 'mpath backend for the storage driver') option('storage_rbd', type: 'feature', value: 'auto', description: 'RADOS Block Device backend for the storage driver') diff --git a/po/POTFILES.in b/po/POTFILES.in index 80c5f145be..3acdb40369 100644 --- a/po/POTFILES.in +++ b/po/POTFILES.in @@ -217,6 +217,7 @@ @SRCDIR(a)src/storage/storage_backend_gluster.c @SRCDIR(a)src/storage/storage_backend_iscsi.c @SRCDIR(a)src/storage/storage_backend_iscsi_direct.c +@SRCDIR(a)src/storage/storage_backend_linstor.c @SRCDIR(a)src/storage/storage_backend_logical.c @SRCDIR(a)src/storage/storage_backend_mpath.c @SRCDIR(a)src/storage/storage_backend_rbd.c diff --git a/src/storage/meson.build b/src/storage/meson.build index 153ff6f846..6e0f472f1f 100644 --- a/src/storage/meson.build +++ b/src/storage/meson.build @@ -35,6 +35,10 @@ storage_backend_iscsi_direct_sources = [ 'storage_backend_iscsi_direct.c', ] +storage_backend_linstor_sources = [ + 'storage_backend_linstor.c', +] + storage_lvm_backend_sources = [ 'storage_backend_logical.c', ] @@ -193,6 +197,27 @@ if conf.has('WITH_STORAGE_ISCSI_DIRECT') } endif +if conf.has('WITH_STORAGE_LINSTOR') + storage_backend_linstor_priv_lib = static_library( + 'virt_storage_backend_linstor_priv', + storage_backend_linstor_sources, + dependencies: [ + src_dep, + ], + include_directories: [ + conf_inc_dir, + ], + ) + + virt_modules += { + 'name': 'virt_storage_backend_linstor', + 'link_whole': [ + storage_backend_linstor_priv_lib, + ], + 'install_dir': storage_backend_install_dir, + } +endif + if conf.has('WITH_STORAGE_LVM') virt_modules += { 'name': 'virt_storage_backend_logical', diff --git a/src/storage/storage_backend.c b/src/storage/storage_backend.c index 83a4b8602a..0cb45d4d53 100644 --- a/src/storage/storage_backend.c +++ b/src/storage/storage_backend.c @@ -70,6 +70,9 @@ #if WITH_STORAGE_VSTORAGE # include "storage_backend_vstorage.h" #endif +#if WITH_STORAGE_LINSTOR +# include "storage_backend_linstor.h" +#endif #define VIR_FROM_THIS VIR_FROM_STORAGE @@ -144,6 +147,9 @@ virStorageBackendDriversRegister(bool allbackends G_GNUC_UNUSED) #if WITH_STORAGE_VSTORAGE VIR_STORAGE_BACKEND_REGISTER(virStorageBackendVstorageRegister, "vstorage"); #endif +#if WITH_STORAGE_LINSTOR + VIR_STORAGE_BACKEND_REGISTER(virStorageBackendLinstorRegister, "linstor"); +#endif return 0; } diff --git a/src/storage/storage_backend_linstor.c b/src/storage/storage_backend_linstor.c new file mode 100644 index 0000000000..e94b930904 --- /dev/null +++ b/src/storage/storage_backend_linstor.c @@ -0,0 +1,783 @@ +/* + * storage_backend_linstor.c: storage backend for linstor volume handling + * + * Copyright (C) 2020-2021 LINBIT HA-Solutions GmbH + * implemented by Rene Peinthor + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library. If not, see + * <http://www.gnu.org/licenses/>;. + */ + +#include <config.h> + +#include "storage_backend_linstor.h" +#define LIBVIRT_STORAGE_BACKEND_LINSTOR_PRIV_H_ALLOW +#include "storage_backend_linstor_priv.h" +#include "virerror.h" +#include "virjson.h" +#include "virstring.h" +#include "virlog.h" +#include "viralloc.h" +#include "virutil.h" +#include "storage_conf.h" +#include "storage_util.h" + +#include <sys/utsname.h> + +#define VIR_FROM_THIS VIR_FROM_STORAGE + +VIR_LOG_INIT("storage.storage_backend_linstor"); + + +#define LINSTORCLI "linstor" + + +/** + * @brief virStorageBackendLinstorGetNodeName + * Get the configured linstor node name, checks pool host[0] + * if node isn't set there, it will try to get hostname and use that. + * @param pool Pool configuration + * @param nodenameOut Retrieved nodename will be copied here, caller is responsible to free. + * @return -1 on error, otherwise 0 + */ +static int +virStorageBackendLinstorGetNodeName(virStoragePoolObjPtr pool, char **nodenameOut) +{ + int ret = 0; + virStoragePoolDefPtr def = virStoragePoolObjGetDef(pool); + if (def->source.nhost > 0 && def->source.hosts[0].name != NULL) { + *nodenameOut = g_strdup(def->source.hosts[0].name); + } else { + *nodenameOut = virGetHostname(); + if (*nodenameOut == NULL) + ret = -1; + } + + return ret; +} + + +static virCommandPtr +virStorageBackendLinstorPrepLinstorCmd(bool machineout) +{ + if (machineout) + return virCommandNewArgList(LINSTORCLI, "-m", "--output-version", "v1", NULL); + else + return virCommandNewArgList(LINSTORCLI, NULL); +} + + +/** + * @brief virStorageBackendLinstorUnpackLinstorJSON + * Linstor client results are packed into an array, as results usually contain + * a list of apicallrcs. But lists usually only have 1 entry. + * @param replyArr linstor reply array json + * @return Pointer to the first array element or NULL if no array or empty + */ +static virJSONValuePtr +virStorageBackendLinstorUnpackLinstorJSON(virJSONValuePtr replyArr) +{ + if (replyArr == NULL) { + return NULL; + } + + if (!virJSONValueIsArray(replyArr)) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("Root Linstor list result is expected to be an array")); + return NULL; + } + + if (virJSONValueArraySize(replyArr) == 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("Empty reply from Linstor client")); + return NULL; + } + + return virJSONValueArrayGet(replyArr, 0); +} + + +int +virStorageBackendLinstorFilterRscDefsForRscGroup(const char *resourceGroup, + const char *output, + virJSONValuePtr rscDefArrayOut) +{ + g_autoptr(virJSONValue) replyArr = NULL; + virJSONValuePtr rscDefArr = NULL; + size_t i; + + replyArr = virJSONValueFromString(output); + + rscDefArr = virStorageBackendLinstorUnpackLinstorJSON(replyArr); + if (rscDefArr == NULL) { + return -1; + } + + for (i = 0; i < virJSONValueArraySize(rscDefArr); i++) { + virJSONValuePtr rscDefObj = virJSONValueArrayGet(rscDefArr, i); + + if (g_ascii_strcasecmp(virJSONValueObjectGetString(rscDefObj, "resource_group_name"), + resourceGroup) == 0) { + + if (virJSONValueArrayAppendString( + rscDefArrayOut, + g_strdup(virJSONValueObjectGetString(rscDefObj, "name")))) + return -1; + } + } + + return 0; +} + + +int +virStorageBackendLinstorParseResourceGroupList(const char *resourceGroup, + const char *output, + virJSONValuePtr *storPoolArrayOut) +{ + bool rscGrpFound = false; + g_autoptr(virJSONValue) replyArr = NULL; + virJSONValuePtr rscGrpArr = NULL; + virJSONValuePtr rscGrpSelFilterObj = NULL; + virJSONValuePtr storPoolsArr = NULL; + size_t i; + + replyArr = virJSONValueFromString(output); + + rscGrpArr = virStorageBackendLinstorUnpackLinstorJSON(replyArr); + if (rscGrpArr == NULL) { + return -1; + } + + for (i = 0; i < virJSONValueArraySize(rscGrpArr); i++) { + virJSONValuePtr rscGrpObj = virJSONValueArrayGet(rscGrpArr, i); + + if (g_ascii_strcasecmp(virJSONValueObjectGetString(rscGrpObj, "name"), + resourceGroup) == 0) { + rscGrpFound = true; + + rscGrpSelFilterObj = virJSONValueObjectGetObject(rscGrpObj, "select_filter"); + if (rscGrpSelFilterObj != NULL) { + storPoolsArr = virJSONValueObjectGetArray(rscGrpSelFilterObj, "storage_pool_list"); + + *storPoolArrayOut = virJSONValueCopy(storPoolsArr); + } + break; + } + } + + if (!rscGrpFound) { + virReportError(VIR_ERR_INVALID_STORAGE_POOL, + _("Specified resource group '%s' not found in linstor"), resourceGroup); + return -1; + } + + return 0; +} + + +/** + * @brief virStorageBackendLinstorParseStoragePoolList + * Parses a storage pool list result and updates the pools capacity, allocation numbers, + * for the given node. + * @param pool Pool object to update + * @param nodename Node name of which storage pools are taken for the update. + * @param output JSON output content from the `linstor storage-pool list` command + * @return -1 on error, 0 on success + */ +int +virStorageBackendLinstorParseStoragePoolList(virStoragePoolDefPtr pool, + const char* nodename, + const char *output) +{ + g_autoptr(virJSONValue) replyArr = NULL; + virJSONValuePtr storpoolArr = NULL; + unsigned long long capacity = 0; + unsigned long long freeCapacity = 0; + size_t i; + + replyArr = virJSONValueFromString(output); + + storpoolArr = virStorageBackendLinstorUnpackLinstorJSON(replyArr); + if (storpoolArr == NULL) { + return -1; + } + + if (!virJSONValueIsArray(storpoolArr)) { + /* probably an ApiCallRc then, with an error */ + virReportError(VIR_ERR_INTERNAL_ERROR, + _("Linstor storage pool list for node '%s' not recieved"), + nodename); + return -1; + } + + for (i = 0; i < virJSONValueArraySize(storpoolArr); i++) { + unsigned long long storCapacity = 0; + unsigned long long storFree = 0; + virJSONValuePtr storPoolObj = virJSONValueArrayGet(storpoolArr, i); + + if (!virJSONValueIsObject(storPoolObj)) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("Unable to parse linstor storage pool object for pool '%s'"), + pool->name); + return -1; + } + + if (g_ascii_strcasecmp(virJSONValueObjectGetString(storPoolObj, "node_name"), nodename) == 0) { + if (g_str_equal(virJSONValueObjectGetString(storPoolObj, "provider_kind"), "DISKLESS")) { + /* ignore diskless pools, as they have no capacity */ + continue; + } + + if (virJSONValueObjectGetNumberUlong(storPoolObj, "total_capacity", &storCapacity)) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("Unable to parse linstor storage pool '%s' capacity"), + virJSONValueObjectGetString(storPoolObj, "storage_pool_name")); + return -1; + } + if (virJSONValueObjectGetNumberUlong(storPoolObj, "free_capacity", &storFree)) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("Unable to parse linstor storage pool '%s' free capacity"), + virJSONValueObjectGetString(storPoolObj, "storage_pool_name")); + return -1; + } + capacity += storCapacity * 1024; /* linstor reports in KiB */ + freeCapacity += storFree * 1024; /* linstor reports in KiB */ + } + } + + pool->capacity = capacity; + pool->available = freeCapacity; + pool->allocation = capacity - freeCapacity; + + return 0; +} + + +/** + * @brief virStorageBackendLinstorParseVolumeDefinition + * Parses the machine output of `linstor volume-definition list` and updates + * the virStorageVolDef capacity. + * @param vol Volume to update the capacity + * @param output JSON output of `linstor volume-definition list -r ...` + * @return -1 on error, 0 on success + */ +int +virStorageBackendLinstorParseVolumeDefinition(virStorageVolDefPtr vol, + const char *output) +{ + g_autoptr(virJSONValue) replyArr = NULL; + virJSONValuePtr resourceDefArr = NULL; + size_t i; + + replyArr = virJSONValueFromString(output); + + resourceDefArr = virStorageBackendLinstorUnpackLinstorJSON(replyArr); + if (resourceDefArr == NULL) { + return -1; + } + + if (!virJSONValueIsArray(resourceDefArr)) { + /* probably an ApiCallRc then, with an error */ + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("Volume definition list not recieved")); + return -1; + } + + for (i = 0; i < virJSONValueArraySize(resourceDefArr); i++) { + unsigned long long volDefCapacityKiB = 0; + virJSONValuePtr resourceDefObj = virJSONValueArrayGet(resourceDefArr, i); + + if (!virJSONValueIsObject(resourceDefObj)) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("Unable to parse resource definition object")); + return -1; + } + + if (g_ascii_strcasecmp(virJSONValueObjectGetString(resourceDefObj, "name"), vol->name) == 0) { + virJSONValuePtr volumeDefArr = virJSONValueObjectGet(resourceDefObj, "volume_definitions"); + virJSONValuePtr volumeDefObj = NULL; + + if (volumeDefArr == NULL || + !virJSONValueIsArray(volumeDefArr) || + virJSONValueArraySize(volumeDefArr) == 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("Volume definition list incorrect for resource definition '%s'"), + vol->name); + return -1; + } + + volumeDefObj = virJSONValueArrayGet(volumeDefArr, 0); + if (virJSONValueObjectGetNumberUlong(volumeDefObj, "size_kib", &volDefCapacityKiB)) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("Unable to parse volume definition size for resource '%s'"), + vol->name); + return -1; + } + + /* linstor reports in KiB */ + vol->target.capacity = volDefCapacityKiB * 1024; + break; + } + } + + return 0; +} + + +static int +virStorageBackendLinstorRefreshVolFromJSON(const char *sourceName, + virStorageVolDefPtr vol, + virJSONValuePtr linstorResObj, + const char *volumeDefListOutput) +{ + virJSONValuePtr volumesArr = NULL; + virJSONValuePtr volumeObj = NULL; + long long alloc_kib = 0; + + volumesArr = virJSONValueObjectGet(linstorResObj, "volumes"); + + if (volumesArr != NULL && !virJSONValueIsArray(volumesArr)) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("'volumes' not found in resource object JSON")); + return -1; + } + + volumeObj = virJSONValueArrayGet(volumesArr, 0); + + if (volumeObj == NULL) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("'volumes' array is empty in resource object JSON")); + return -1; + } + + vol->type = VIR_STORAGE_VOL_BLOCK; + VIR_FREE(vol->key); + vol->key = g_strdup_printf("%s/%s", sourceName, vol->name); + VIR_FREE(vol->target.path); + vol->target.path = g_strdup(virJSONValueObjectGetString(volumeObj, "device_path")); + vol->target.format = VIR_STORAGE_FILE_RAW; + + if (virJSONValueObjectGetNumberLong(volumeObj, "allocated_size_kib", &alloc_kib)) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("Unable to parse allocated size from Linstor result")); + return -1; + } + + if (alloc_kib >= 0) + vol->target.allocation = alloc_kib * 1024; + else + vol->target.allocation = 0; + + if (volumeDefListOutput != NULL) { + return virStorageBackendLinstorParseVolumeDefinition(vol, volumeDefListOutput); + } + + return 0; +} + + +static int +virStorageBackendLinstorRefreshVol(virStoragePoolObjPtr pool, + virStorageVolDefPtr vol) +{ + g_autofree char *output = NULL; + g_autofree char *outputVolDef = NULL; + g_autofree char *nodename = NULL; + g_autoptr(virCommand) cmdResList = NULL; + g_autoptr(virCommand) cmdVolDefList = NULL; + g_autoptr(virJSONValue) replyArr = NULL; + virJSONValuePtr rscArr = NULL; + virStoragePoolDefPtr def = virStoragePoolObjGetDef(pool); + + if (virStorageBackendLinstorGetNodeName(pool, &nodename)) + return -1; + + cmdResList = virStorageBackendLinstorPrepLinstorCmd(true); + virCommandAddArgList(cmdResList, "resource", "list", "-n", nodename, "-r", vol->name, NULL); + virCommandSetOutputBuffer(cmdResList, &output); + if (virCommandRun(cmdResList, NULL) < 0) + return -1; + + cmdVolDefList = virStorageBackendLinstorPrepLinstorCmd(true); + virCommandAddArgList(cmdVolDefList, "volume-definition", "list", "-r", vol->name, NULL); + virCommandSetOutputBuffer(cmdVolDefList, &outputVolDef); + if (virCommandRun(cmdVolDefList, NULL) < 0) + return -1; + + replyArr = virJSONValueFromString(output); + + rscArr = virStorageBackendLinstorUnpackLinstorJSON(replyArr); + if (rscArr == NULL) { + return -1; + } + + if (!virJSONValueIsArray(rscArr)) { + /* probably an ApiCallRc then, with an error */ + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("Resource list not recieved")); + return -1; + } + + if (virJSONValueArraySize(rscArr) != 1) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("Couldn't find resource '%s' in Linstor resource list JSON"), vol->name); + return -1; + } + + return virStorageBackendLinstorRefreshVolFromJSON( + def->source.name, vol, virJSONValueArrayGet(rscArr, 0), outputVolDef); +} + + +static int +virStorageBackendLinstorAddVolume(virStoragePoolObjPtr pool, + virJSONValuePtr resourceObj, + const char *outputVolDef) +{ + g_autoptr(virStorageVolDef) vol = NULL; + virStoragePoolDefPtr def = virStoragePoolObjGetDef(pool); + + if (resourceObj == NULL) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("Missing disk info when adding volume")); + return -1; + } + + vol = g_new0(virStorageVolDef, 1); + + vol->name = g_strdup(virJSONValueObjectGetString(resourceObj, "name")); + + if (virStorageBackendLinstorRefreshVolFromJSON(def->source.name, + vol, resourceObj, outputVolDef) < 0) { + virStorageVolDefFree(vol); + return -1; + } + + if (virStoragePoolObjAddVol(pool, vol) < 0) { + virStorageVolDefFree(vol); + return -1; + } + vol = NULL; + + return 0; +} + + +static bool +virStorageBackendLinstorStringInJSONArray(virJSONValuePtr arr, const char *string) +{ + size_t i; + for (i = 0; i < virJSONValueArraySize(arr); i++) { + if (g_ascii_strcasecmp(virJSONValueGetString(virJSONValueArrayGet(arr, i)), string) == 0) { + return true; + } + } + return false; +} + + +int +virStorageBackendLinstorParseResourceList(virStoragePoolObjPtr pool, + const char *nodeName, + virJSONValuePtr rscDefFilterArr, + const char *outputRscList, + const char *outputVolDef) +{ + g_autoptr(virJSONValue) replyArr = NULL; + virJSONValuePtr rscListArr = NULL; + size_t i; + + replyArr = virJSONValueFromString(outputRscList); + + rscListArr = virStorageBackendLinstorUnpackLinstorJSON(replyArr); + if (rscListArr == NULL) { + return -1; + } + + if (!virJSONValueIsArray(rscListArr)) { + /* probably an ApiCallRc then, with an error */ + virReportError(VIR_ERR_INTERNAL_ERROR, + _("Unable to parse Linstor storage pool object for pool '%s'"), + nodeName); + return -1; + } + + for (i = 0; i < virJSONValueArraySize(rscListArr); i++) { + virJSONValuePtr rscObj = virJSONValueArrayGet(rscListArr, i); + + if (g_ascii_strcasecmp(virJSONValueObjectGetString(rscObj, "node_name"), nodeName) == 0 && + virStorageBackendLinstorStringInJSONArray(rscDefFilterArr, + virJSONValueObjectGetString(rscObj, "name"))) { + if (virStorageBackendLinstorAddVolume(pool, rscObj, outputVolDef)) { + return -1; + } + } + } + + return 0; +} + +static int +virStorageBackendLinstorRefreshAllVol(virStoragePoolObjPtr pool) +{ + g_autofree char *output = NULL; + g_autofree char *outputVolDef = NULL; + g_autofree char *nodename = NULL; + g_autoptr(virCommand) cmdRscList = NULL; + g_autoptr(virCommand) cmdVolDefList = NULL; + g_autoptr(virJSONValue) rscDefFilterArr = virJSONValueNewArray(); + virStoragePoolDefPtr def = virStoragePoolObjGetDef(pool); + + /* Get all resources usable on that node */ + if (virStorageBackendLinstorGetNodeName(pool, &nodename)) { + return -1; + } + + cmdRscList = virStorageBackendLinstorPrepLinstorCmd(true); + virCommandAddArgList(cmdRscList, "resource", "list", "-n", nodename, NULL); + virCommandSetOutputBuffer(cmdRscList, &output); + if (virCommandRun(cmdRscList, NULL) < 0) + return -1; + + /* Get a list of resources that belong to the rsc group for filtering */ + cmdVolDefList = virStorageBackendLinstorPrepLinstorCmd(true); + virCommandAddArgList(cmdVolDefList, "volume-definition", "list", NULL); + virCommandSetOutputBuffer(cmdVolDefList, &outputVolDef); + if (virCommandRun(cmdVolDefList, NULL) < 0) { + return -1; + } + + /* resource belonging to the resource group will be stored in rscDefFilterArr */ + if (virStorageBackendLinstorFilterRscDefsForRscGroup(def->source.name, + outputVolDef, + rscDefFilterArr)) { + return -1; + } + + return virStorageBackendLinstorParseResourceList(pool, + nodename, + rscDefFilterArr, + output, + outputVolDef); +} + + +/** + * @brief virStorageBackendLinstorGetRscGrpPools + * Retrieves the set storage pools used in resource group. + * On success caller is responsible to free the virJSONValuePtr. + * @param rscgrpname resource group name to get the storage pools + * @param storagePoolsOut virJSONArray with used storage pools + * @return -1 on error, 0 on success + */ +static int +virStorageBackendLinstorGetRscGrpPools(const char* rscgrpname, virJSONValuePtr *storagePoolsOut) +{ + g_autofree char *outputRscGrp = NULL; + g_autoptr(virCommand) cmdRscGrpList = NULL; + + cmdRscGrpList = virStorageBackendLinstorPrepLinstorCmd(true); + virCommandAddArgList(cmdRscGrpList, "resource-group", "list", "-r", rscgrpname, NULL); + virCommandSetOutputBuffer(cmdRscGrpList, &outputRscGrp); + if (virCommandRun(cmdRscGrpList, NULL) < 0) + return -1; + + if (virStorageBackendLinstorParseResourceGroupList(rscgrpname, + outputRscGrp, + storagePoolsOut)) { + return -1; + } + + return 0; +} + + +static int +virStorageBackendLinstorRefreshPool(virStoragePoolObjPtr pool) +{ + size_t i; + g_autofree char *outputStorPoolList = NULL; + g_autofree char *nodename = NULL; + g_autoptr(virCommand) cmdStorPoolList = NULL; + virJSONValuePtr storagePoolArr = NULL; + virStoragePoolDefPtr def = virStoragePoolObjGetDef(pool); + + if (virStorageBackendLinstorGetNodeName(pool, &nodename)) + return -1; + + if (virStorageBackendLinstorGetRscGrpPools(def->source.name, &storagePoolArr)) + return -1; + + /* Get storage pools used in the used resource group */ + cmdStorPoolList = virStorageBackendLinstorPrepLinstorCmd(true); + virCommandAddArgList(cmdStorPoolList, "storage-pool", "list", "-n", nodename, NULL); + + if (storagePoolArr != NULL && virJSONValueArraySize(storagePoolArr) > 0) { + virCommandAddArgList(cmdStorPoolList, "-s", NULL); + for (i = 0; i < virJSONValueArraySize(storagePoolArr); i++) { + virCommandAddArg(cmdStorPoolList, + virJSONValueGetString(virJSONValueArrayGet(storagePoolArr, i))); + } + + virJSONValueFree(storagePoolArr); + } + + virCommandSetOutputBuffer(cmdStorPoolList, &outputStorPoolList); + if (virCommandRun(cmdStorPoolList, NULL) < 0) + return -1; + + /* update capacity and allocated from used storage pools */ + if (virStorageBackendLinstorParseStoragePoolList(virStoragePoolObjGetDef(pool), + nodename, + outputStorPoolList) < 0) + return -1; + + /* Get volumes used in the resource group and add */ + return virStorageBackendLinstorRefreshAllVol(pool); +} + +static int +virStorageBackendLinstorCreateVol(virStoragePoolObjPtr pool, + virStorageVolDefPtr vol) +{ + virStoragePoolDefPtr def = virStoragePoolObjGetDef(pool); + g_autoptr(virCommand) cmdRscGrp = NULL; + + VIR_DEBUG("Creating Linstor image %s/%s with size %llu", + def->source.name, vol->name, vol->target.capacity); + + if (!vol->target.capacity) { + virReportError(VIR_ERR_NO_SUPPORT, "%s", + _("volume capacity required for this storage pool")); + return -1; + } + + if (vol->target.format != VIR_STORAGE_FILE_RAW) { + virReportError(VIR_ERR_NO_SUPPORT, "%s", + _("only RAW volumes are supported by this storage pool")); + return -1; + } + + if (vol->target.encryption != NULL) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", + _("storage pool does not support encrypted volumes")); + return -1; + } + + /* spawn resource */ + cmdRscGrp = virStorageBackendLinstorPrepLinstorCmd(false); + virCommandAddArgList(cmdRscGrp, "resource-group", "spawn", + "--partial", def->source.name, vol->name, NULL); + virCommandAddArgFormat(cmdRscGrp, "%lluKiB", vol->target.capacity / 1024); + if (virCommandRun(cmdRscGrp, NULL) < 0) + return -1; + + /* set volume path and key */ + /* we could skip getting the capacity as we already know it */ + return virStorageBackendLinstorRefreshVol(pool, vol); +} + + +static int +virStorageBackendLinstorBuildVolFrom(virStoragePoolObjPtr pool, + virStorageVolDefPtr vol, + virStorageVolDefPtr inputvol, + unsigned int flags) +{ + virStorageBackendBuildVolFrom build_func; + + build_func = virStorageBackendGetBuildVolFromFunction(vol, inputvol); + if (!build_func) + return -1; + + return build_func(pool, vol, inputvol, flags); +} + + +static int +virStorageBackendLinstorDeleteVol(virStoragePoolObjPtr pool G_GNUC_UNUSED, + virStorageVolDefPtr vol, + unsigned int flags) +{ + g_autoptr(virCommand) cmd = NULL; + + virCheckFlags(0, -1); + + cmd = virStorageBackendLinstorPrepLinstorCmd(false); + virCommandAddArgList(cmd, "resource-definition", "delete", vol->name, NULL); + return virCommandRun(cmd, NULL); +} + + +static int +virStorageBackendLinstorResizeVol(virStoragePoolObjPtr pool G_GNUC_UNUSED, + virStorageVolDefPtr vol, + unsigned long long capacity, + unsigned int flags) +{ + g_autoptr(virCommand) cmd = NULL; + + virCheckFlags(0, -1); + + cmd = virStorageBackendLinstorPrepLinstorCmd(false); + virCommandAddArgList(cmd, "volume-definition", "set-size", vol->name, "0", NULL); + virCommandAddArgFormat(cmd, "%lluKiB", capacity / 1024); + return virCommandRun(cmd, NULL); +} + + +/** + * @brief virStorageBackendVzCheck + * Check if we can connect to a Linstor-Controller + */ +static int +virStorageBackendLinstorCheck(virStoragePoolObjPtr pool G_GNUC_UNUSED, + bool *isActive) +{ + g_autoptr(virCommand) cmd = NULL; + + /* This command gets the controller version */ + cmd = virStorageBackendLinstorPrepLinstorCmd(false); + virCommandAddArgList(cmd, "controller", "version", NULL); + if (virCommandRun(cmd, NULL)) { + *isActive = false; + } + + *isActive = true; + return 0; +} + +virStorageBackend virStorageBackendLinstor = { + .type = VIR_STORAGE_POOL_LINSTOR, + + .refreshPool = virStorageBackendLinstorRefreshPool, + .checkPool = virStorageBackendLinstorCheck, + .createVol = virStorageBackendLinstorCreateVol, + .buildVol = NULL, + .buildVolFrom = virStorageBackendLinstorBuildVolFrom, + .refreshVol = virStorageBackendLinstorRefreshVol, + .deleteVol = virStorageBackendLinstorDeleteVol, + .resizeVol = virStorageBackendLinstorResizeVol, + .uploadVol = virStorageBackendVolUploadLocal, + .downloadVol = virStorageBackendVolDownloadLocal, + .wipeVol = virStorageBackendVolWipeLocal, +}; + + +int +virStorageBackendLinstorRegister(void) +{ + return virStorageBackendRegister(&virStorageBackendLinstor); +} diff --git a/src/storage/storage_backend_linstor.h b/src/storage/storage_backend_linstor.h new file mode 100644 index 0000000000..4d4d2ff4ff --- /dev/null +++ b/src/storage/storage_backend_linstor.h @@ -0,0 +1,24 @@ +/* + * storage_backend_linstor.h: storage backend for Sheepdog handling + * + * Copyright (C) 2020-2021 LINBIT HA-Solutions GmbH + * implemented by Rene Peinthor + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library. If not, see + * <http://www.gnu.org/licenses/>;. + */ + +#pragma once + +int virStorageBackendLinstorRegister(void); diff --git a/src/storage/storage_backend_linstor_priv.h b/src/storage/storage_backend_linstor_priv.h new file mode 100644 index 0000000000..36503993b8 --- /dev/null +++ b/src/storage/storage_backend_linstor_priv.h @@ -0,0 +1,53 @@ +/* + * storage_backend_linstor_priv.h: header for functions necessary in tests + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library. If not, see + * <http://www.gnu.org/licenses/>;. + */ + +#ifndef LIBVIRT_STORAGE_BACKEND_LINSTOR_PRIV_H_ALLOW +# error "storage_backend_linstor_priv.h may only be included by storage_backend_linstor.c or test suites" +#endif /* LIBVIRT_STORAGE_BACKEND_LINSTOR_PRIV_H_ALLOW */ + +#pragma once + +#include "virjson.h" +#include "virstorageobj.h" +#include "conf/storage_conf.h" + +int +virStorageBackendLinstorFilterRscDefsForRscGroup(const char *resourceGroup, + const char *output, + virJSONValuePtr rscDefArrayOut); + +int +virStorageBackendLinstorParseResourceGroupList(const char *resourceGroup, + const char *output, + virJSONValuePtr *storPoolArrayOut); + +int +virStorageBackendLinstorParseStoragePoolList(virStoragePoolDefPtr pool, + const char* nodeName, + const char *output); + +int +virStorageBackendLinstorParseResourceList(virStoragePoolObjPtr pool, + const char* nodeName, + virJSONValuePtr rscDefFilterArr, + const char *outputRscList, + const char *outputVolDef); + +int +virStorageBackendLinstorParseVolumeDefinition(virStorageVolDefPtr vol, + const char *output); -- 2.30.0