Re: [Libvir] PATCH: 13/16: iSCSI backend

This provides a storage pool using the iSCSI protocol. Since there is no API for iSCSI it is implemented by simply shelling out to the iscsiadm command line tool. A pool corresponds to a single target on the iSCSI server. Starting a pool logs into the server and maps the target's LUNs into the local filesystem. The default nodes are under /dev, allocated-on-demand and thus not guarenteed to be stable across reboots. For this reason it is recommended by the pool target path be configured to point to /dev/disk/by-path or /dev/disk/by-id whose entries are guarenteed stable for lifetime of the target+LUN. The 'refresh' operation will rescan the target for new LUNs and purge old LUNs allowing dynamic updates without needing a pool restart. b/src/storage_backend_iscsi.c | 424 ++++++++++++++++++++++++++++++++++++++++++ b/src/storage_backend_iscsi.h | 45 ++++ configure.in | 22 ++ libvirt.spec.in | 4 src/Makefile.am | 6 src/storage_backend.c | 15 + 6 files changed, 516 insertions(+) diff -r 9a3200af0a3d configure.in --- a/configure.in Thu Feb 07 11:14:56 2008 -0500 +++ b/configure.in Thu Feb 07 11:34:46 2008 -0500 @@ -561,6 +561,8 @@ AC_ARG_WITH(storage-fs, [ --with-storage-fs with FileSystem backend for the storage driver (on)],[],[with_storage_fs=check]) AC_ARG_WITH(storage-lvm, [ --with-storage-lvm with LVM backend for the storage driver (on)],[],[with_storage_lvm=check]) +AC_ARG_WITH(storage-iscsi, +[ --with-storage-iscsi with iSCSI backend for the storage driver (on)],[],[with_storage_iscsi=check]) if test "$with_storage_fs" = "yes" -o "$with_storage_fs" = "check"; then AC_PATH_PROG(MOUNT, [mount], [], [$PATH:/sbin:/usr/sbin]) @@ -653,6 +655,25 @@ if test "$with_storage_lvm" = "yes" -o " fi fi AM_CONDITIONAL(WITH_STORAGE_LVM, [test "$with_storage_lvm" = "yes"]) + + + +if test "$with_storage_iscsi" = "yes" -o "$with_storage_iscsi" = "check"; then + AC_PATH_PROG(ISCSIADM, [iscsiadm], [], [$PATH:/sbin:/usr/sbin]) + if test "$with_storage_iscsi" = "yes" ; then + if test -z "$ISCSIADM" ; then AC_MSG_ERROR(We need iscsiadm for iSCSI storage driver) ; fi + else + if test -z "$ISCSIADM" ; then with_storage_iscsi=no ; fi + + if test "$with_storage_iscsi" = "check" ; then with_storage_iscsi=yes ; fi + fi + + if test "$with_storage_iscsi" = "yes" ; then + AC_DEFINE_UNQUOTED(WITH_STORAGE_ISCSI, 1, [whether iSCSI backend for storage driver is enabled]) + AC_DEFINE_UNQUOTED([ISCSIADM],["$ISCSIADM"],[Location of iscsiadm program]) + fi +fi +AM_CONDITIONAL(WITH_STORAGE_ISCSI, [test "$with_storage_iscsi" = "yes"]) dnl @@ -869,6 +890,7 @@ AC_MSG_NOTICE([ FS: $with_storage_f AC_MSG_NOTICE([ FS: $with_storage_fs]) AC_MSG_NOTICE([ NetFS: $with_storage_fs]) AC_MSG_NOTICE([ LVM: $with_storage_lvm]) +AC_MSG_NOTICE([ iSCSI: $with_storage_iscsi]) AC_MSG_NOTICE([]) AC_MSG_NOTICE([Libraries]) AC_MSG_NOTICE([]) diff -r 9a3200af0a3d libvirt.spec.in --- a/libvirt.spec.in Thu Feb 07 11:14:56 2008 -0500 +++ b/libvirt.spec.in Thu Feb 07 11:34:46 2008 -0500 @@ -51,6 +51,8 @@ Requires: /usr/sbin/qcow-create %endif # For LVM drivers Requires: lvm2 +# For ISCSI driver +Requires: iscsi-initiator-utils BuildRequires: xen-devel BuildRequires: libxml2-devel BuildRequires: readline-devel @@ -77,6 +79,8 @@ BuildRequires: /usr/sbin/qcow-create %endif # For LVM drivers BuildRequires: lvm2 +# For ISCSI driver +BuildRequires: iscsi-initiator-utils Obsoletes: libvir ExclusiveArch: i386 x86_64 ia64 diff -r 9a3200af0a3d src/Makefile.am --- a/src/Makefile.am Thu Feb 07 11:14:56 2008 -0500 +++ b/src/Makefile.am Thu Feb 07 11:34:46 2008 -0500 @@ -75,6 +75,12 @@ EXTRA_DIST += storage_backend_logical.h EXTRA_DIST += storage_backend_logical.h storage_backend_logical.c endif +if WITH_STORAGE_ISCSI +CLIENT_SOURCES += storage_backend_iscsi.h storage_backend_iscsi.c +else +EXTRA_DIST += storage_backend_iscsi.h storage_backend_iscsi.c +endif + libvirt_la_SOURCES = $(CLIENT_SOURCES) $(SERVER_SOURCES) diff -r 9a3200af0a3d src/storage_backend.c --- a/src/storage_backend.c Thu Feb 07 11:14:56 2008 -0500 +++ b/src/storage_backend.c Thu Feb 07 11:34:46 2008 -0500 @@ -39,6 +39,10 @@ #if WITH_STORAGE_LVM #include "storage_backend_logical.h" #endif +#if WITH_STORAGE_ISCSI +#include "storage_backend_iscsi.h" +#endif + #include "util.h" @@ -53,6 +57,9 @@ static virStorageBackendPtr backends[] = #endif #if WITH_STORAGE_LVM &virStorageBackendLogical, +#endif +#if WITH_STORAGE_ISCSI + &virStorageBackendISCSI, #endif }; @@ -94,6 +101,10 @@ int virStorageBackendFromString(const ch #if WITH_STORAGE_LVM if (STREQ(type, "logical")) return VIR_STORAGE_POOL_LOGICAL; +#endif +#if WITH_STORAGE_ISCSI + if (STREQ(type, "iscsi")) + return VIR_STORAGE_POOL_ISCSI; #endif virStorageReportError(NULL, VIR_ERR_INTERNAL_ERROR, "unknown storage backend type %s", type); return -1; @@ -112,6 +123,10 @@ const char *virStorageBackendToString(in #if WITH_STORAGE_LVM case VIR_STORAGE_POOL_LOGICAL: return "logical"; +#endif +#if WITH_STORAGE_ISCSI + case VIR_STORAGE_POOL_ISCSI: + return "iscsi"; #endif } diff -r 9a3200af0a3d src/storage_backend_iscsi.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/storage_backend_iscsi.c Thu Feb 07 11:34:46 2008 -0500 @@ -0,0 +1,424 @@ +/* + * storage_backend_iscsi.c: storage backend for iSCSI handling + * + * Copyright (C) 2007-2008 Red Hat, Inc. + * Copyright (C) 2007-2008 Daniel P. Berrange + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + * Author: Daniel P. Berrange <berrange(a)redhat.com&gt; + */ + +#include <config.h> + +#include <sys/socket.h> +#include <netdb.h> +#include <sys/types.h> +#include <sys/wait.h> +#include <string.h> +#include <stdio.h> +#include <regex.h> +#include <fcntl.h> +#include <unistd.h> +#include <dirent.h> + +#include "storage_backend_iscsi.h" +#include "util.h" + +static int virStorageBackendISCSITargetIP(virConnectPtr conn, + const char *hostname, + char *ipaddr, + size_t ipaddrlen) +{ + struct addrinfo hints; + struct addrinfo *result = NULL; + int ret; + + memset(&hints, 0, sizeof hints); + hints.ai_flags = AI_ADDRCONFIG; + hints.ai_family = AF_INET; + hints.ai_socktype = SOCK_STREAM; + hints.ai_protocol = 0; + + ret = getaddrinfo(hostname, NULL, &hints, &result); + if (ret != 0) { + virStorageReportError(conn, VIR_ERR_INTERNAL_ERROR, "getaddrinfo %s", + gai_strerror(ret)); + return -1; + } + + if (result == NULL) { + virStorageReportError(conn, VIR_ERR_INTERNAL_ERROR, + "no IP address for target %s", hostname); + return -1; + } + + if (getnameinfo(result->ai_addr, result->ai_addrlen, + ipaddr, ipaddrlen, NULL, 0, + NI_NUMERICHOST) < 0) { + virStorageReportError(conn, VIR_ERR_INTERNAL_ERROR, + "cannot format ip addr for %s", hostname); + freeaddrinfo(result); + return -1; + } + + freeaddrinfo(result); + return 0; +} + +static int virStorageBackendISCSIExtractSession(virConnectPtr conn, + virStoragePoolObjPtr pool, + char **const groups, + void *data) +{ + char **session = data; + + if (STREQ(groups[1], pool->def->source.devices[0].path)) { + if ((*session = strdup(groups[0])) == NULL) { + virStorageReportError(conn, VIR_ERR_NO_MEMORY, "session"); + return -1; + } + } + + return 0; +} + +static char *virStorageBackendISCSISession(virConnectPtr conn, + virStoragePoolObjPtr pool) +{ + /* + * # iscsiadm --mode session -P 0 + * tcp: [1] 192.168.122.170:3260,1 demo-tgt-b + * tcp: [2] 192.168.122.170:3260,1 demo-tgt-a + * + * Pull out 2nd and 4th fields + */ + const char *regexes[] = { + "^tcp:\\s+\\[(\\S+)\\]\\s+\\S+\\s+(\\S+)\\s*$" + }; + int vars[] = { + 2, + }; + const char *prog[] = { + ISCSIADM, "--mode", "session", "-P", "0", NULL + }; + char *session = NULL; + + if (virStorageBackendRunProgRegex(conn, pool, + prog, + 1, + regexes, + vars, + virStorageBackendISCSIExtractSession, + &session) < 0) + return NULL; + + if (session == NULL) { + virStorageReportError(conn, VIR_ERR_INTERNAL_ERROR, "cannot find session"); + return NULL; + } + + return session; +} + +static int virStorageBackendISCSIConnection(virConnectPtr conn, + virStoragePoolObjPtr pool, + const char *portal, + const char *action) +{ + const char *cmdargv[] = { + ISCSIADM, "--mode", "node", "--portal", portal, + "--targetname", pool->def->source.devices[0].path, action, NULL + }; + + if (virRun(conn, (char **)cmdargv, NULL) < 0) + return -1; + + return 0; +} + + +static int virStorageBackendISCSIMakeLUN(virConnectPtr conn, + virStoragePoolObjPtr pool, + char **const groups, + void *data ATTRIBUTE_UNUSED) +{ + virStorageVolDefPtr vol; + int fd = -1; + char scsiid[100]; + char *dev = groups[4]; + int opentries = 0; + char *devpath = NULL; + + snprintf(scsiid, sizeof(scsiid)-1, "%s:%s:%s:%s", + groups[0], groups[1], groups[2], groups[3]); + + if ((vol = calloc(1, sizeof(virStorageVolDef))) == NULL) { + virStorageReportError(conn, VIR_ERR_NO_MEMORY, "volume"); + return -1; + } + + if ((vol->name = strdup(scsiid)) == NULL) { + virStorageReportError(conn, VIR_ERR_NO_MEMORY, "name"); + goto cleanup; + } + + if ((devpath = malloc(5 + strlen(dev) + 1)) == NULL) { + virStorageReportError(conn, VIR_ERR_NO_MEMORY, "devpath"); + goto cleanup; + } + strcpy(devpath, "/dev/"); + strcat(devpath, dev); + /* It can take a little while between logging into the ISCSI + * server and udev creating the /dev nodes, so if we get ENOENT + * we must retry a few times - they should eventually appear. + * We currently wait for upto 5 seconds. Is this good enough ? + * Perhaps not on a very heavily loaded system Any other + * options... ? + */ + reopen: + if ((fd = open(devpath, O_RDONLY)) < 0) { + opentries++; + if (errno == ENOENT && opentries < 50) { + usleep(100 * 1000); + goto reopen; + } + virStorageReportError(conn, VIR_ERR_INTERNAL_ERROR, + "cannot open %s: %s (%d)", + devpath, strerror(errno), errno); + goto cleanup; + } + + /* Now figure out the stable path + * + * XXX this method is O(N) because it scans the pool target + * dir every time its run. Should figure out a more efficient + * way of doing this... + */ + if ((vol->target.path = virStorageBackendStablePath(conn, pool, devpath)) == NULL) + goto cleanup; + + if (devpath != vol->target.path) + free(devpath); + devpath = NULL; + + if (virStorageBackendUpdateVolInfoFD(conn, vol, fd, 1) < 0) + goto cleanup; + + /* XXX use unique iSCSI id instead */ + vol->key = strdup(vol->target.path); + if (vol->key == NULL) { + virStorageReportError(conn, VIR_ERR_NO_MEMORY, "key"); + goto cleanup; + } + + + pool->def->capacity += vol->capacity; + pool->def->allocation += vol->allocation; + + vol->next = pool->volumes; + pool->volumes = vol; + pool->nvolumes++; + + close(fd); + + return 0; + + cleanup: + if (fd != -1) close(fd); + free(devpath); + virStorageVolDefFree(vol); + return -1; +} + +static int virStorageBackendISCSIFindLUNs(virConnectPtr conn, + virStoragePoolObjPtr pool, + const char *session) +{ + /* + * # iscsiadm --mode session -r $session -P 3 + * + * scsi1 Channel 00 Id 0 Lun: 0 + * scsi1 Channel 00 Id 0 Lun: 1 + * Attached scsi disk sdc State: running + * scsi1 Channel 00 Id 0 Lun: 2 + * Attached scsi disk sdd State: running + * scsi1 Channel 00 Id 0 Lun: 3 + * Attached scsi disk sde State: running + * scsi1 Channel 00 Id 0 Lun: 4 + * Attached scsi disk sdf State: running + * scsi1 Channel 00 Id 0 Lun: 5 + * Attached scsi disk sdg State: running + * + * Need 2 regex to match alternating lines + */ + const char *regexes[] = { + "^\\s*scsi(\\S+)\\s+Channel\\s+(\\S+)\\s+Id\\s+(\\S+)\\s+Lun:\\s+(\\S+)\\s*$", + "^\\s*Attached\\s+scsi\\s+disk\\s+(\\S+)\\s+State:\\s+running\\s*$" + }; + int vars[] = { + 4, 1 + }; + const char *prog[] = { + ISCSIADM, "--mode", "session", "-r", session, "-P", "3", NULL, + }; + + return virStorageBackendRunProgRegex(conn, pool, + prog, + 2, + regexes, + vars, + virStorageBackendISCSIMakeLUN, + NULL); +} + + +static int virStorageBackendISCSILogin(virConnectPtr conn, + virStoragePoolObjPtr pool, + const char *portal) +{ + return virStorageBackendISCSIConnection(conn, pool, portal, "--login"); +} + +static int virStorageBackendISCSILogout(virConnectPtr conn, + virStoragePoolObjPtr pool, + const char *portal) +{ + return virStorageBackendISCSIConnection(conn, pool, portal, "--logout"); +} + +static char *virStorageBackendISCSIPortal(virConnectPtr conn, + virStoragePoolObjPtr pool) +{ + char ipaddr[NI_MAXHOST]; + char *portal; + + if (virStorageBackendISCSITargetIP(conn, + pool->def->source.host.name, + ipaddr, sizeof(ipaddr)) < 0) + return NULL; + + portal = malloc(strlen(ipaddr) + 1 + 4 + 2 + 1); + if (portal == NULL) { + virStorageReportError(conn, VIR_ERR_NO_MEMORY, "portal"); + return NULL; + } + + strcpy(portal, ipaddr); + strcat(portal, ":3260,1"); + + return portal; +} + + +static int virStorageBackendISCSIStartPool(virConnectPtr conn, + virStoragePoolObjPtr pool) +{ + char *portal = NULL; + + if (pool->def->source.host.name == NULL) { + virStorageReportError(conn, VIR_ERR_INTERNAL_ERROR, "missing source host"); + return -1; + } + + if (pool->def->source.ndevice != 1 || + pool->def->source.devices[0].path == NULL) { + virStorageReportError(conn, VIR_ERR_INTERNAL_ERROR, "missing source device"); + return -1; + } + + if ((portal = virStorageBackendISCSIPortal(conn, pool)) == NULL) + return -1; + if (virStorageBackendISCSILogin(conn, pool, portal) < 0) { + free(portal); + return -1; + } + free(portal); + return 0; +} + +static int virStorageBackendISCSIRefreshPool(virConnectPtr conn, + virStoragePoolObjPtr pool) +{ + char *portal = NULL, *session = NULL; + + pool->def->allocation = pool->def->capacity = pool->def->available = 0; + + if ((portal = virStorageBackendISCSIPortal(conn, pool)) == NULL) + return -1; + + if ((session = virStorageBackendISCSISession(conn, pool)) == NULL) { + goto cleanup; + } + if (virStorageBackendISCSIFindLUNs(conn, pool, session) < 0) + goto cleanup; + free(portal); + free(session); + + return 0; + + cleanup: + free(portal); + free(session); + return -1; +} + + +static int virStorageBackendISCSIStopPool(virConnectPtr conn, + virStoragePoolObjPtr pool) +{ + char *portal; + + if ((portal = virStorageBackendISCSIPortal(conn, pool)) == NULL) + return -1; + + if (virStorageBackendISCSILogout(conn, pool, portal) < 0) { + free(portal); + return -1; + } + free(portal); + + return 0; +} + + +virStorageBackend virStorageBackendISCSI = { + .type = VIR_STORAGE_POOL_ISCSI, + + .startPool = virStorageBackendISCSIStartPool, + .refreshPool = virStorageBackendISCSIRefreshPool, + .stopPool = virStorageBackendISCSIStopPool, + + .poolOptions = { + .flags = (VIR_STORAGE_BACKEND_POOL_SOURCE_HOST | + VIR_STORAGE_BACKEND_POOL_SOURCE_DEVICE) + }, + + .volType = VIR_STORAGE_VOL_BLOCK, +}; + +/* + * vim: set tabstop=4: + * vim: set shiftwidth=4: + * vim: set expandtab: + */ +/* + * Local variables: + * indent-tabs-mode: nil + * c-indent-level: 4 + * c-basic-offset: 4 + * tab-width: 4 + * End: + */ diff -r 9a3200af0a3d src/storage_backend_iscsi.h --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/storage_backend_iscsi.h Thu Feb 07 11:34:46 2008 -0500 @@ -0,0 +1,45 @@ +/* + * storage_backend_iscsi.h: storage backend for iSCSI handling + * + * Copyright (C) 2007-2008 Red Hat, Inc. + * Copyright (C) 2007-2008 Daniel P. Berrange + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + * Author: Daniel P. Berrange <berrange(a)redhat.com&gt; + */ + +#ifndef __VIR_STORAGE_BACKEND_ISCSI_H__ +#define __VIR_STORAGE_BACKEND_ISCSI_H__ + +#include "storage_backend.h" + +extern virStorageBackend virStorageBackendISCSI; + +#endif /* __VIR_STORAGE_BACKEND_ISCSI_H__ */ + +/* + * vim: set tabstop=4: + * vim: set shiftwidth=4: + * vim: set expandtab: + */ +/* + * Local variables: + * indent-tabs-mode: nil + * c-indent-level: 4 + * c-basic-offset: 4 + * tab-width: 4 + * End: + */ -- |=- Red Hat, Engineering, Emerging Technologies, Boston. +1 978 392 2496 -=| |=- Perl modules: http://search.cpan.org/~danberr/ -=| |=- Projects: http://freshmeat.net/~danielpb/ -=| |=- GnuPG: 7D3B9505 F3C9 553F A1DA 4AC2 5648 23C1 B3DF F742 7D3B 9505 -=|