[libvirt] [PATCH RFC] storage: ZFS support

Implement ZFS storage backend driver. Currently supported only on FreeBSD because of ZFS limitations on Linux. Features supported: - pool-start, pool-stop - pool-info - vol-list - vol-create / vol-delete Pool definition looks like that: <pool type='zfs'> <name>myzfspool</name> <source> <name>actualpoolname</name> </source> </pool> The 'actualpoolname' value is a name of the pool on the system, such as shown by 'zpool list' command. Target makes no sense here because volumes path is always /dev/zvol/$poolname/$volname. Users has to create a pool on his own, this driver doesn't support pool creation currently. A volume could be used with Qemu by adding an entry like this: <disk type='volume' device='disk'> <driver name='qemu' type='raw'/> <source pool='myzfspool' volume='vol5'/> <target dev='hdc' bus='ide'/> </disk> --- configure.ac | 43 +++++ include/libvirt/libvirt.h.in | 1 + po/POTFILES.in | 1 + src/Makefile.am | 8 + src/conf/storage_conf.c | 11 +- src/conf/storage_conf.h | 4 +- src/qemu/qemu_conf.c | 1 + src/storage/storage_backend.c | 6 + src/storage/storage_backend_zfs.c | 344 ++++++++++++++++++++++++++++++++++++++ src/storage/storage_backend_zfs.h | 29 ++++ src/storage/storage_driver.c | 1 + tools/virsh-pool.c | 3 + 12 files changed, 449 insertions(+), 3 deletions(-) create mode 100644 src/storage/storage_backend_zfs.c create mode 100644 src/storage/storage_backend_zfs.h diff --git a/configure.ac b/configure.ac index f37c716..7e46460 100644 --- a/configure.ac +++ b/configure.ac @@ -1734,6 +1734,10 @@ AC_ARG_WITH([storage-gluster], [AS_HELP_STRING([--with-storage-gluster], [with Gluster backend for the storage driver @<:@default=check@:>@])], [],[with_storage_gluster=check]) +AC_ARG_WITH([storage-zfs], + [AS_HELP_STRING([--with-storage-zfs], + [with ZFS backend for the storage driver @<:@default=check@:>@])], + [],[with_storage_zfs=check]) if test "$with_libvirtd" = "no"; then with_storage_dir=no @@ -1746,6 +1750,7 @@ if test "$with_libvirtd" = "no"; then with_storage_rbd=no with_storage_sheepdog=no with_storage_gluster=no + with_storage_zfs=no fi if test "$with_storage_dir" = "yes" ; then AC_DEFINE_UNQUOTED([WITH_STORAGE_DIR], 1, [whether directory backend for storage driver is enabled]) @@ -1963,6 +1968,43 @@ if test "$with_storage_gluster" = "yes"; then fi AM_CONDITIONAL([WITH_STORAGE_GLUSTER], [test "$with_storage_gluster" = "yes"]) +if test "$with_storage_zfs" = "check"; then + with_storage_zfs=$with_freebsd +fi + +if test "$with_storage_zfs" = "yes" && test "$with_freebsd" = "no"; then + AC_MSG_ERROR([The ZFS storage driver can be enabled on FreeBSD only.]) +fi + +if test "$with_storage_zfs" = "yes" || + test "$with_storage_zfs" = "check"; then + AC_PATH_PROG([ZFS], [zfs], [], [$PATH:/sbin:/usr/sbin]) + AC_PATH_PROG([ZPOOL], [zpool], [], [$PATH:/sbin:/usr/sbin]) + + if test "$with_storage_zfs" = "yes"; then + if test -z "$ZFS" || test -z "$ZPOOL"; then + AC_MSG_ERROR([We need zfs and zpool for ZFS storage driver]) + fi + else + if test -z "$ZFS" || test -z "$ZPOOL"; then + with_storage_zfs=no + fi + + if test "$with_storage_zfs" = "check"; then + with_storage_zfs=yes + fi + fi + + if test "$with_storage_zfs" = "yes"; then + AC_DEFINE_UNQUOTED([WITH_STORAGE_ZFS], 1, + [whether ZFS backend for storage driver is enabled]) + AC_DEFINE_UNQUOTED([ZFS], ["$ZFS"], [Location of zfs program]) + AC_DEFINE_UNQUOTED([ZPOOL], ["$ZPOOL"], [Location of zpool program]) + fi +fi +AM_CONDITIONAL([WITH_STORAGE_ZFS], + [test "$with_storage_zfs" = "yes"]) + if test "$with_storage_fs" = "yes" || test "$with_storage_gluster" = "yes"; then AC_PATH_PROG([GLUSTER_CLI], [gluster], [], [$PATH:/sbin:/usr/sbin]) @@ -2806,6 +2848,7 @@ AC_MSG_NOTICE([ Disk: $with_storage_disk]) AC_MSG_NOTICE([ RBD: $with_storage_rbd]) AC_MSG_NOTICE([Sheepdog: $with_storage_sheepdog]) AC_MSG_NOTICE([ Gluster: $with_storage_gluster]) +AC_MSG_NOTICE([ ZFS: $with_storage_zfs]) AC_MSG_NOTICE([]) AC_MSG_NOTICE([Security Drivers]) AC_MSG_NOTICE([]) diff --git a/include/libvirt/libvirt.h.in b/include/libvirt/libvirt.h.in index ad6785f..47ea695 100644 --- a/include/libvirt/libvirt.h.in +++ b/include/libvirt/libvirt.h.in @@ -3170,6 +3170,7 @@ typedef enum { VIR_CONNECT_LIST_STORAGE_POOLS_RBD = 1 << 14, VIR_CONNECT_LIST_STORAGE_POOLS_SHEEPDOG = 1 << 15, VIR_CONNECT_LIST_STORAGE_POOLS_GLUSTER = 1 << 16, + VIR_CONNECT_LIST_STORAGE_POOLS_ZFS = 1 << 17, } virConnectListAllStoragePoolsFlags; int virConnectListAllStoragePools(virConnectPtr conn, diff --git a/po/POTFILES.in b/po/POTFILES.in index d10e892..2646bcc 100644 --- a/po/POTFILES.in +++ b/po/POTFILES.in @@ -148,6 +148,7 @@ src/storage/storage_backend_mpath.c src/storage/storage_backend_rbd.c src/storage/storage_backend_scsi.c src/storage/storage_backend_sheepdog.c +src/storage/storage_backend_zfs.c src/storage/storage_driver.c src/test/test_driver.c src/uml/uml_conf.c diff --git a/src/Makefile.am b/src/Makefile.am index 982f63d..d5d06d1 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -862,6 +862,9 @@ STORAGE_DRIVER_SHEEPDOG_SOURCES = \ STORAGE_DRIVER_GLUSTER_SOURCES = \ storage/storage_backend_gluster.h storage/storage_backend_gluster.c +STORAGE_DRIVER_ZFS_SOURCES = \ + storage/storage_backend_zfs.h storage/storage_backend_zfs.c + STORAGE_HELPER_DISK_SOURCES = \ storage/parthelper.c @@ -1514,6 +1517,10 @@ libvirt_driver_storage_impl_la_CFLAGS += $(GLUSTERFS_CFLAGS) libvirt_driver_storage_impl_la_LIBADD += $(GLUSTERFS_LIBS) endif WITH_STORAGE_GLUSTER +if WITH_STORAGE_ZFS +libvirt_driver_storage_impl_la_SOURCES += $(STORAGE_DRIVER_ZFS_SOURCES) +endif WITH_STORAGE_ZFS + if WITH_NODE_DEVICES # Needed to keep automake quiet about conditionals if WITH_DRIVER_MODULES @@ -1723,6 +1730,7 @@ EXTRA_DIST += \ $(STORAGE_DRIVER_RBD_SOURCES) \ $(STORAGE_DRIVER_SHEEPDOG_SOURCES) \ $(STORAGE_DRIVER_GLUSTER_SOURCES) \ + $(STORAGE_DRIVER_ZFS_SOURCES) \ $(NODE_DEVICE_DRIVER_SOURCES) \ $(NODE_DEVICE_DRIVER_HAL_SOURCES) \ $(NODE_DEVICE_DRIVER_UDEV_SOURCES) \ diff --git a/src/conf/storage_conf.c b/src/conf/storage_conf.c index e33eb1b..5ec9c97 100644 --- a/src/conf/storage_conf.c +++ b/src/conf/storage_conf.c @@ -61,7 +61,8 @@ VIR_ENUM_IMPL(virStoragePool, VIR_STORAGE_POOL_LAST, "dir", "fs", "netfs", "logical", "disk", "iscsi", - "scsi", "mpath", "rbd", "sheepdog", "gluster") + "scsi", "mpath", "rbd", + "sheepdog", "gluster", "zfs") VIR_ENUM_IMPL(virStoragePoolFormatFileSystem, VIR_STORAGE_POOL_FS_LAST, @@ -278,7 +279,13 @@ static virStoragePoolTypeInfo poolTypeInfo[] = { .formatFromString = virStorageVolFormatDiskTypeFromString, .formatToString = virStorageVolFormatDiskTypeToString, }, - } + }, + {.poolType = VIR_STORAGE_POOL_ZFS, + .poolOptions = { + .flags = (VIR_STORAGE_POOL_SOURCE_NAME), + .defaultFormat = VIR_STORAGE_FILE_RAW, + }, + }, }; diff --git a/src/conf/storage_conf.h b/src/conf/storage_conf.h index ecd0dfe..1276ac2 100644 --- a/src/conf/storage_conf.h +++ b/src/conf/storage_conf.h @@ -92,6 +92,7 @@ typedef enum { VIR_STORAGE_POOL_RBD, /* RADOS Block Device */ VIR_STORAGE_POOL_SHEEPDOG, /* Sheepdog device */ VIR_STORAGE_POOL_GLUSTER, /* Gluster device */ + VIR_STORAGE_POOL_ZFS, /* ZFS */ VIR_STORAGE_POOL_LAST, } virStoragePoolType; @@ -509,7 +510,8 @@ VIR_ENUM_DECL(virStoragePartedFs) VIR_CONNECT_LIST_STORAGE_POOLS_MPATH | \ VIR_CONNECT_LIST_STORAGE_POOLS_RBD | \ VIR_CONNECT_LIST_STORAGE_POOLS_SHEEPDOG | \ - VIR_CONNECT_LIST_STORAGE_POOLS_GLUSTER) + VIR_CONNECT_LIST_STORAGE_POOLS_GLUSTER | \ + VIR_CONNECT_LIST_STORAGE_POOLS_ZFS) # define VIR_CONNECT_LIST_STORAGE_POOLS_FILTERS_ALL \ (VIR_CONNECT_LIST_STORAGE_POOLS_FILTERS_ACTIVE | \ diff --git a/src/qemu/qemu_conf.c b/src/qemu/qemu_conf.c index e62bec0..1ed47a8 100644 --- a/src/qemu/qemu_conf.c +++ b/src/qemu/qemu_conf.c @@ -1292,6 +1292,7 @@ qemuTranslateDiskSourcePool(virConnectPtr conn, case VIR_STORAGE_POOL_LOGICAL: case VIR_STORAGE_POOL_DISK: case VIR_STORAGE_POOL_SCSI: + case VIR_STORAGE_POOL_ZFS: if (!(def->src->path = virStorageVolGetPath(vol))) goto cleanup; diff --git a/src/storage/storage_backend.c b/src/storage/storage_backend.c index 27b02cb..aad913e 100644 --- a/src/storage/storage_backend.c +++ b/src/storage/storage_backend.c @@ -88,6 +88,9 @@ #if WITH_STORAGE_GLUSTER # include "storage_backend_gluster.h" #endif +#if WITH_STORAGE_ZFS +# include "storage_backend_zfs.h" +#endif #define VIR_FROM_THIS VIR_FROM_STORAGE @@ -125,6 +128,9 @@ static virStorageBackendPtr backends[] = { #if WITH_STORAGE_GLUSTER &virStorageBackendGluster, #endif +#if WITH_STORAGE_ZFS + &virStorageBackendZFS, +#endif NULL }; diff --git a/src/storage/storage_backend_zfs.c b/src/storage/storage_backend_zfs.c new file mode 100644 index 0000000..1079bc7 --- /dev/null +++ b/src/storage/storage_backend_zfs.c @@ -0,0 +1,344 @@ +/* + * storage_backend_zfs.c: storage backend for ZFS handling + * + * Copyright (C) 2014 Roman Bogorodskiy + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library. If not, see + * <http://www.gnu.org/licenses/>;. + * + */ + +#include <config.h> + +#include "viralloc.h" +#include "virerror.h" +#include "virfile.h" +#include "storage_backend_zfs.h" +#include "virlog.h" +#include "virstring.h" + +#define VIR_FROM_THIS VIR_FROM_STORAGE + +VIR_LOG_INIT("storage.storage_backend_zfs"); + +/* + * Some common flags of zfs and zpool commands we use: + * -H -- don't print headers and separate fields by tab + * -p -- show exact numbers instead of human-readable, i.e. + * for size, show just a number instead of 2G etc + */ + + +static int +virStorageBackendZFSCheckPool(virConnectPtr conn ATTRIBUTE_UNUSED, + virStoragePoolObjPtr pool ATTRIBUTE_UNUSED, + bool *isActive) +{ + char *devpath; + + if (virAsprintf(&devpath, "/dev/zvol/%s", + pool->def->source.name) == -1) + return -1; + *isActive = virFileIsDir(devpath); + VIR_FREE(devpath); + + return 0; +} + +static int +virStorageBackendZFSStartPool(virConnectPtr conn ATTRIBUTE_UNUSED, + virStoragePoolObjPtr pool ATTRIBUTE_UNUSED) +{ + return 0; +} + +static int +virStorageBackendZFSParseVol(virStoragePoolObjPtr pool, + const char *volume) +{ + int ret = -1; + char **tokens; + size_t count; + char **name_tokens = NULL; + char *vol_name; + bool is_new_vol = false; + virStorageVolDefPtr vol = NULL; + + if (!(tokens = virStringSplitCount(volume, "\t", 0, &count))) + return -1; + + if (count != 2) + goto cleanup; + + if (!(name_tokens = virStringSplit(tokens[0], "/", 2))) + goto cleanup; + + vol_name = name_tokens[1]; + + vol = virStorageVolDefFindByName(pool, vol_name); + + if (vol == NULL) { + if (VIR_ALLOC(vol) < 0) + goto cleanup; + + is_new_vol = true; + vol->type = VIR_STORAGE_VOL_BLOCK; + + if (VIR_STRDUP(vol->name, vol_name) < 0) + goto cleanup; + } + + if (!vol->key && VIR_STRDUP(vol->key, tokens[0]) < 0) + goto cleanup; + + if (vol->target.path == NULL) { + if (virAsprintf(&vol->target.path, "/dev/zvol/%s/%s", + pool->def->source.name, vol->name) < 0) + goto cleanup; + } + + if (virStrToLong_ull(tokens[1], NULL, 10, &vol->target.capacity) < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, + "%s", _("malformed volsize reported")); + goto cleanup; + } + + if (is_new_vol && + VIR_APPEND_ELEMENT(pool->volumes.objs, pool->volumes.count, vol) < 0) + goto cleanup; + + ret = 0; + cleanup: + virStringFreeList(tokens); + virStringFreeList(name_tokens); + if (is_new_vol && (ret == -1)) + virStorageVolDefFree(vol); + return ret; +} + +static int +virStorageBackendZFSFindVols(virStoragePoolObjPtr pool) +{ + virCommandPtr cmd = NULL; + char *volumes_list = NULL; + char **lines = NULL; + size_t i; + + /** + * $ zfs list -Hp -t volume -o name,volsize -r test + * test/vol1 5368709120 + * test/vol3 1073741824 + * test/vol4 1572864000 + * $ + * + * Arguments description: + * -t volume -- we want to see only volumes + * -o name,volsize -- limit output to name and volume size + * -r -- we want to see all the childer of our pool + */ + cmd = virCommandNewArgList(ZFS, + "list", "-Hp", + "-t", "volume", "-r", + "-o", "name,volsize", + pool->def->source.name, + NULL); + virCommandSetOutputBuffer(cmd, &volumes_list); + if (virCommandRun(cmd, NULL) < 0) + goto cleanup; + + if (!(lines = virStringSplit(volumes_list, "\n", 0))) + goto cleanup; + + for (i = 0; lines[i]; i++) { + if (STREQ(lines[i], "")) + continue; + + if (virStorageBackendZFSParseVol(pool, lines[i]) < 0) + continue; + } + + cleanup: + virCommandFree(cmd); + virStringFreeList(lines); + VIR_FREE(volumes_list); + + return 0; +} + +static int +virStorageBackendZFSRefreshPool(virConnectPtr conn ATTRIBUTE_UNUSED, + virStoragePoolObjPtr pool ATTRIBUTE_UNUSED) +{ + virCommandPtr cmd = NULL; + char *zpool_props = NULL; + char **lines = NULL; + size_t i; + + /** + * $ zpool get -Hp health,size,free,allocated test + * test health ONLINE - + * test size 199715979264 - + * test free 198899976704 - + * test allocated 816002560 - + * $ + * + * Here we just provide a list of properties we want to see + */ + cmd = virCommandNewArgList(ZPOOL, + "get", "-Hp", + "health,size,free,allocated", + pool->def->source.name, + NULL); + virCommandSetOutputBuffer(cmd, &zpool_props); + if (virCommandRun(cmd, NULL) < 0) + goto cleanup; + + if (!(lines = virStringSplit(zpool_props, "\n", 0))) + goto cleanup; + + for (i = 0; lines[i]; i++) { + char **tokens; + size_t count; + char *prop_name; + + if (STREQ(lines[i], "")) + continue; + + if (!(tokens = virStringSplitCount(lines[i], "\t", 0, &count))) + goto cleanup; + + if (count != 4) + continue; + + prop_name = tokens[1]; + + if (STREQ(prop_name, "free") || STREQ(prop_name, "size") || + STREQ(prop_name, "allocated")) { + unsigned long long value; + if (virStrToLong_ull(tokens[2], NULL, 10, &value) < 0) + goto cleanup; + + if (STREQ(prop_name, "free")) + pool->def->available = value; + else if (STREQ(prop_name, "size")) + pool->def->capacity = value; + else if (STREQ(prop_name, "allocated")) + pool->def->allocation = value; + } + } + + /* Obtain a list of volumes */ + if (virStorageBackendZFSFindVols(pool) < 0) + goto cleanup; + + cleanup: + virCommandFree(cmd); + virStringFreeList(lines); + VIR_FREE(zpool_props); + + return 0; +} + +static int +virStorageBackendZFSStopPool(virConnectPtr conn ATTRIBUTE_UNUSED, + virStoragePoolObjPtr pool ATTRIBUTE_UNUSED) +{ + return 0; +} + +static int +virStorageBackendZFSCreateVol(virConnectPtr conn ATTRIBUTE_UNUSED, + virStoragePoolObjPtr pool, + virStorageVolDefPtr vol) +{ + virCommandPtr cmd = NULL; + int ret = -1; + + vol->type = VIR_STORAGE_VOL_BLOCK; + + if (vol->target.path != NULL) { + /* A target path passed to CreateVol has no meaning */ + VIR_FREE(vol->target.path); + } + + if (virAsprintf(&vol->target.path, "/dev/zvol/%s/%s", + pool->def->source.name, vol->name) == -1) + return -1; + + /** + * $ zfs create -o volmode=dev -V 10240K test/volname + * + * -o volmode=dev -- we want to get volumes exposed as cdev + * devices. If we don't specify that zfs + * will lookup vfs.zfs.vol.mode sysctl value + * -V -- tells to create a volume with the specified size + */ + cmd = virCommandNewArgList(ZFS, "create", "-o", "volmode=dev", + "-V", NULL); + virCommandAddArgFormat(cmd, "%lluK", + VIR_DIV_UP(vol->target.capacity, 1024)); + virCommandAddArgFormat(cmd, "%s/%s", + pool->def->source.name, vol->name); + + if (virCommandRun(cmd, NULL) < 0) + goto cleanup; + + if (virAsprintf(&vol->key, "%s/%s", + pool->def->source.name, vol->name) == -1) + goto cleanup; + + if (virStorageBackendZFSFindVols(pool) < 0) + goto cleanup; + + ret = 0; + cleanup: + virCommandFree(cmd); + return ret; + +} + +static int +virStorageBackendZFSDeleteVol(virConnectPtr conn ATTRIBUTE_UNUSED, + virStoragePoolObjPtr pool, + virStorageVolDefPtr vol, + unsigned int flags) +{ + int ret = -1; + virCommandPtr destroy_cmd = virCommandNewArgList(ZFS, "destroy", NULL); + + virCheckFlags(0, -1); + + virCommandAddArgFormat(destroy_cmd, "%s/%s", + pool->def->source.name, vol->name); + + if (virCommandRun(destroy_cmd, NULL) < 0) + goto cleanup; + + ret = 0; + cleanup: + virCommandFree(destroy_cmd); + return ret; +} + + +virStorageBackend virStorageBackendZFS = { + .type = VIR_STORAGE_POOL_ZFS, + + .checkPool = virStorageBackendZFSCheckPool, + .startPool = virStorageBackendZFSStartPool, + .refreshPool = virStorageBackendZFSRefreshPool, + .stopPool = virStorageBackendZFSStopPool, + .createVol = virStorageBackendZFSCreateVol, + .deleteVol = virStorageBackendZFSDeleteVol, +}; diff --git a/src/storage/storage_backend_zfs.h b/src/storage/storage_backend_zfs.h new file mode 100644 index 0000000..4c34b59 --- /dev/null +++ b/src/storage/storage_backend_zfs.h @@ -0,0 +1,29 @@ +/* + * storage_backend_zfs.h: storage backend for ZFS handling + * + * Copyright (C) 2014 Roman Bogorodskiy + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library. If not, see + * <http://www.gnu.org/licenses/>;. + * + */ + +#ifndef __VIR_STORAGE_BACKEND_ZFS_H__ +# define __VIR_STORAGE_BACKEND_ZFS_H__ + +# include "storage_backend.h" + +extern virStorageBackend virStorageBackendZFS; + +#endif /* __VIR_STORAGE_BACKEND_ZFS_H__ */ diff --git a/src/storage/storage_driver.c b/src/storage/storage_driver.c index 3830867..d23d663 100644 --- a/src/storage/storage_driver.c +++ b/src/storage/storage_driver.c @@ -1431,6 +1431,7 @@ storageVolLookupByPath(virConnectPtr conn, case VIR_STORAGE_POOL_GLUSTER: case VIR_STORAGE_POOL_RBD: case VIR_STORAGE_POOL_SHEEPDOG: + case VIR_STORAGE_POOL_ZFS: case VIR_STORAGE_POOL_LAST: if (VIR_STRDUP(stable_path, path) < 0) { virStoragePoolObjUnlock(pool); diff --git a/tools/virsh-pool.c b/tools/virsh-pool.c index 7c40b5b..80313b1 100644 --- a/tools/virsh-pool.c +++ b/tools/virsh-pool.c @@ -1063,6 +1063,9 @@ cmdPoolList(vshControl *ctl, const vshCmd *cmd ATTRIBUTE_UNUSED) case VIR_STORAGE_POOL_GLUSTER: flags |= VIR_CONNECT_LIST_STORAGE_POOLS_GLUSTER; break; + case VIR_STORAGE_POOL_ZFS: + flags |= VIR_CONNECT_LIST_STORAGE_POOLS_ZFS; + break; case VIR_STORAGE_POOL_LAST: break; } -- 1.9.0