Some libvirt functions use streams, this patch add compress stream support. So VolumeDownload/VolumeUpload can greatly speedup by using compressed streams to save network bandtwidth and don't transfer zero bytes (in case of raw disk format) Signed-off-by: Vasiliy Tolstov <v.tolstov(a)selfip.ru&gt; --- configure.ac | 2 ++ daemon/Makefile.am | 3 +++ include/libvirt/libvirt-stream.h | 3 +++ m4/virt-archive.m4 | 29 ++++++++++++++++++++++++++ src/Makefile.am | 17 +++++++++++----- src/datatypes.h | 7 +++++++ src/fdstream.c | 44 ++++++++++++++++++++++++++++++++++++++++ src/libvirt-stream.c | 15 ++++++++++++++ tools/Makefile.am | 17 ++++++++++++++-- 9 files changed, 130 insertions(+), 7 deletions(-) create mode 100644 m4/virt-archive.m4 diff --git a/configure.ac b/configure.ac index 03463b0..58a15bb 100644 --- a/configure.ac +++ b/configure.ac @@ -250,6 +250,7 @@ LIBVIRT_CHECK_SANLOCK LIBVIRT_CHECK_SASL LIBVIRT_CHECK_SELINUX LIBVIRT_CHECK_SSH2 +LIBVIRT_CHECK_LIBARCHIVE LIBVIRT_CHECK_SYSTEMD_DAEMON LIBVIRT_CHECK_UDEV LIBVIRT_CHECK_WIRESHARK @@ -2892,6 +2893,7 @@ LIBVIRT_RESULT_SANLOCK LIBVIRT_RESULT_SASL LIBVIRT_RESULT_SELINUX LIBVIRT_RESULT_SSH2 +LIBVIRT_RESULT_LIBARCHIVE LIBVIRT_RESULT_SYSTEMD_DAEMON LIBVIRT_RESULT_UDEV LIBVIRT_RESULT_WIRESHARK diff --git a/daemon/Makefile.am b/daemon/Makefile.am index be1b5a9..a4d63eb 100644 --- a/daemon/Makefile.am +++ b/daemon/Makefile.am @@ -177,17 +177,20 @@ libvirtd_CFLAGS = \ $(XDR_CFLAGS) $(DBUS_CFLAGS) $(LIBNL_CFLAGS) \ $(WARN_CFLAGS) $(PIE_CFLAGS) \ $(COVERAGE_CFLAGS) \ + $(LIBARCHIVE_CFLAGS) \ -DQEMUD_PID_FILE="\"$(QEMUD_PID_FILE)\"" libvirtd_LDFLAGS = \ $(RELRO_LDFLAGS) \ $(PIE_LDFLAGS) \ $(COVERAGE_LDFLAGS) \ + $(LIBARCHIVE_LDFLAGS) \ $(NO_INDIRECT_LDFLAGS) \ $(NULL) libvirtd_LDADD = \ $(LIBXML_LIBS) \ + $(LIBARCHIVE_LIBS) \ $(GNUTLS_LIBS) \ $(SASL_LIBS) \ $(DBUS_LIBS) \ diff --git a/include/libvirt/libvirt-stream.h b/include/libvirt/libvirt-stream.h index 831640d..c75f03e 100644 --- a/include/libvirt/libvirt-stream.h +++ b/include/libvirt/libvirt-stream.h @@ -31,10 +31,13 @@ typedef enum { VIR_STREAM_NONBLOCK = (1 << 0), + VIR_STREAM_COMPRESS_GZIP = (1 << 1), + VIR_STREAM_COMPRESS_XZ = (1 << 2), } virStreamFlags; virStreamPtr virStreamNew(virConnectPtr conn, unsigned int flags); + int virStreamRef(virStreamPtr st); int virStreamSend(virStreamPtr st, diff --git a/m4/virt-archive.m4 b/m4/virt-archive.m4 new file mode 100644 index 0000000..9770732 --- /dev/null +++ b/m4/virt-archive.m4 @@ -0,0 +1,29 @@ +dnl The libarchive.so library +dnl +dnl Copyright (C) 2012-2013 Red Hat, Inc. +dnl +dnl This library is free software; you can redistribute it and/or +dnl modify it under the terms of the GNU Lesser General Public +dnl License as published by the Free Software Foundation; either +dnl version 2.1 of the License, or (at your option) any later version. +dnl +dnl This library is distributed in the hope that it will be useful, +dnl but WITHOUT ANY WARRANTY; without even the implied warranty of +dnl MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +dnl Lesser General Public License for more details. +dnl +dnl You should have received a copy of the GNU Lesser General Public +dnl License along with this library. If not, see +dnl <http://www.gnu.org/licenses/>;. +dnl + +AC_DEFUN([LIBVIRT_CHECK_LIBARCHIVE],[ + LIBVIRT_CHECK_PKG([LIBARCHIVE], [libarchive], [3.1.2]) + AC_SUBST(LIBARCHIVE_CFLAGS) + AC_SUBST(LIBARCHIVE_LIBS) + AC_SUBST(LIBARCHIVE_LDFLAGS) +]) + +AC_DEFUN([LIBVIRT_RESULT_LIBARCHIVE],[ + LIBVIRT_RESULT_LIB([LIBARCHIVE]) +]) diff --git a/src/Makefile.am b/src/Makefile.am index 060abe8..429c2c7 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -25,11 +25,11 @@ abs_topsrcdir = $(shell cd $(top_srcdir) && pwd) # No libraries with the exception of LIBXML should be listed # here. List them against the individual XXX_la_CFLAGS targets # that actually use them. Also keep GETTEXT_CPPFLAGS at the end. -INCLUDES = -I../gnulib/lib \ +INCLUDES = -I../gnulib/lib \ -I$(top_srcdir)/gnulib/lib \ - -I$(top_srcdir) \ + -I$(top_srcdir) \ -I../include \ - -I$(top_srcdir)/include \ + -I$(top_srcdir)/include \ -I$(srcdir)/util \ -DIN_LIBVIRT \ -Dabs_topbuilddir="\"$(abs_topbuilddir)\"" \ @@ -37,14 +37,16 @@ INCLUDES = -I../gnulib/lib \ $(GETTEXT_CPPFLAGS) AM_CFLAGS = $(LIBXML_CFLAGS) \ + $(LIBARCHIVE_CFLAGS) \ $(WARN_CFLAGS) \ - $(LOCK_CHECKING_CFLAGS) \ + $(LOCK_CHECKING_CFLAGS) \ $(WIN32_EXTRA_CFLAGS) \ $(COVERAGE_CFLAGS) AM_LDFLAGS = $(DRIVER_MODULE_LDFLAGS) \ $(COVERAGE_LDFLAGS) \ $(RELRO_LDFLAGS) \ $(NO_INDIRECT_LDFLAGS) \ + $(LIBARCHIVE_LDFLAGS) \ $(NULL) EXTRA_DIST = $(conf_DATA) util/keymaps.csv @@ -1051,11 +1053,13 @@ libvirt_util_la_CFLAGS = $(CAPNG_CFLAGS) $(YAJL_CFLAGS) $(LIBNL_CFLAGS) \ $(AM_CFLAGS) $(AUDIT_CFLAGS) $(DEVMAPPER_CFLAGS) \ $(DBUS_CFLAGS) $(LDEXP_LIBM) $(NUMACTL_CFLAGS) \ $(SYSTEMD_DAEMON_CFLAGS) $(POLKIT_CFLAGS) \ + $(LIBARCHIVE_CFLAGS) \ -I$(srcdir)/conf libvirt_util_la_LIBADD = $(CAPNG_LIBS) $(YAJL_LIBS) $(LIBNL_LIBS) \ $(THREAD_LIBS) $(AUDIT_LIBS) $(DEVMAPPER_LIBS) \ $(LIB_CLOCK_GETTIME) $(DBUS_LIBS) $(MSCOM_LIBS) $(LIBXML_LIBS) \ $(SECDRIVER_LIBS) $(NUMACTL_LIBS) $(SYSTEMD_DAEMON_LIBS) \ + $(LIBARCHIVE_LIBS) \ $(POLKIT_LIBS) @@ -2214,6 +2218,7 @@ libvirt_lxc_la_LDFLAGS = \ $(AM_LDFLAGS) \ $(CYGWIN_EXTRA_LDFLAGS) \ $(MINGW_EXTRA_LDFLAGS) \ + $(LIBARCHIVE_LDFLAGS) \ $(NULL) libvirt_lxc_la_CFLAGS = $(AM_CFLAGS) libvirt_lxc_la_LIBADD = libvirt.la $(CYGWIN_EXTRA_LIBADD) @@ -2292,13 +2297,15 @@ libvirt_setuid_rpc_client_la_SOURCES = \ libvirt_setuid_rpc_client_la_LDFLAGS = \ $(AM_LDFLAGS) \ $(LIBXML_LIBS) \ + $(LIBARCHIVE_LIBS) \ $(SECDRIVER_LIBS) \ $(NULL) libvirt_setuid_rpc_client_la_CFLAGS = \ -DLIBVIRT_SETUID_RPC_CLIENT \ -I$(srcdir)/conf \ - -I$(srcdir)/rpc \ + -I$(srcdir)/rpc \ $(AM_CFLAGS) \ + $(LIBARCHIVE_CFLAGS) \ $(SECDRIVER_CFLAGS) \ $(XDR_CFLAGS) \ $(NULL) diff --git a/src/datatypes.h b/src/datatypes.h index be108fe..4c0c10d 100644 --- a/src/datatypes.h +++ b/src/datatypes.h @@ -29,6 +29,10 @@ # include "virobject.h" # include "viruuid.h" +#ifdef WITH_LIBARCHIVE +# include <archive.h> +#endif + extern virClassPtr virConnectClass; extern virClassPtr virDomainClass; extern virClassPtr virDomainSnapshotClass; @@ -520,6 +524,9 @@ struct _virStream { virStreamDriverPtr driver; void *privateData; +#ifdef WITH_LIBARCHIVE + struct archive *archive; +#endif }; /** diff --git a/src/fdstream.c b/src/fdstream.c index b8ea86e..9a71a9f 100644 --- a/src/fdstream.c +++ b/src/fdstream.c @@ -33,6 +33,10 @@ #include <netinet/in.h> #include <termios.h> +#ifdef WITH_LIBARCHIVE +# include <archive.h> +#endif + #include "fdstream.h" #include "virerror.h" #include "datatypes.h" @@ -319,6 +323,12 @@ virFDStreamCloseInt(virStreamPtr st, bool streamAbort) if (VIR_CLOSE(fdst->errfd) < 0) VIR_DEBUG("ignoring failed close on fd %d", fdst->errfd); +#ifdef WITH_LIBARCHIVE + if (st->archive != NULL) { + archive_write_free(st->archive); + archive_read_free(st->archive); + } +#endif st->privateData = NULL; /* call the internal stream closing callback */ @@ -385,7 +395,16 @@ static int virFDStreamWrite(virStreamPtr st, const char *bytes, size_t nbytes) } retry: +#ifdef WITH_LIBARCHIVE + if (st->archive != NULL) { + archive_write_open_fd(st->archive, fdst->fd); + ret = archive_write_data(st->archive, bytes, nbytes); + } else { + ret = write(fdst->fd, bytes, nbytes); + } +#else ret = write(fdst->fd, bytes, nbytes); +#endif if (ret < 0) { if (errno == EAGAIN || errno == EWOULDBLOCK) { ret = -2; @@ -397,7 +416,15 @@ static int virFDStreamWrite(virStreamPtr st, const char *bytes, size_t nbytes) _("cannot write to stream")); } } else if (fdst->length) { +#ifdef WITH_LIBARCHIVE + if (st->archive != NULL) { + fdst->offset += nbytes; + } else { + fdst->offset += ret; + } +#else fdst->offset += ret; +#endif } virMutexUnlock(&fdst->lock); @@ -435,7 +462,16 @@ static int virFDStreamRead(virStreamPtr st, char *bytes, size_t nbytes) } retry: +#ifdef WITH_LIBARCHIVE + if (st->archive != NULL) { + archive_read_open_fd(st->archive, fdst->fd, 4096); + ret = archive_read_data(st->archive, bytes, nbytes); + } else { + ret = read(fdst->fd, bytes, nbytes); + } +#else ret = read(fdst->fd, bytes, nbytes); +#endif if (ret < 0) { if (errno == EAGAIN || errno == EWOULDBLOCK) { ret = -2; @@ -447,7 +483,15 @@ static int virFDStreamRead(virStreamPtr st, char *bytes, size_t nbytes) _("cannot read from stream")); } } else if (fdst->length) { +#ifdef WITH_LIBARCHIVE + if (st->archive != NULL) { + fdst->offset += nbytes; + } else { + fdst->offset += ret; + } +#else fdst->offset += ret; +#endif } virMutexUnlock(&fdst->lock); diff --git a/src/libvirt-stream.c b/src/libvirt-stream.c index c16f586..0f09d9d 100644 --- a/src/libvirt-stream.c +++ b/src/libvirt-stream.c @@ -24,6 +24,10 @@ #include "viralloc.h" #include "virlog.h" +#ifdef WITH_LIBARCHIVE +# include <archive.h> +#endif + VIR_LOG_INIT("libvirt.stream"); #define VIR_FROM_THIS VIR_FROM_STREAMS @@ -45,6 +49,7 @@ VIR_LOG_INIT("libvirt.stream"); * * If a non-blocking data stream is required passed * VIR_STREAM_NONBLOCK for flags, otherwise pass 0. + * VIR_STREAM_COMPRESS_GZIP or VIR_STREAM_COMPRESS_XZ * * Returns the new stream, or NULL upon error */ @@ -66,6 +71,16 @@ virStreamNew(virConnectPtr conn, else virDispatchError(conn); + if ((flags & VIR_STREAM_COMPRESS_GZIP) || (flags & VIR_STREAM_COMPRESS_XZ)) { + st->archive = archive_write_new(); + + if (flags & VIR_STREAM_COMPRESS_GZIP) + archive_write_add_filter(st->archive, ARCHIVE_FILTER_GZIP); + + if (flags & VIR_STREAM_COMPRESS_XZ) + archive_write_add_filter(st->archive, ARCHIVE_FILTER_XZ); + } + return st; } diff --git a/tools/Makefile.am b/tools/Makefile.am index 03e9339..fa19a20 100644 --- a/tools/Makefile.am +++ b/tools/Makefile.am @@ -25,8 +25,12 @@ INCLUDES = \ AM_LDFLAGS = \ $(RELRO_LDFLAGS) \ $(NO_INDIRECT_LDFLAGS) \ + $(LIBARCHIVE_LDFLAGS) \ $(NULL) +AM_CFLAGS = \ + $(LIBARCHIVE_CFLAGS) + POD2MAN = pod2man -c "Virtualization Support" -r "$(PACKAGE)-$(VERSION)" ICON_FILES = \ @@ -140,15 +144,18 @@ virt_host_validate_LDFLAGS = \ $(AM_LDFLAGS) \ $(PIE_LDFLAGS) \ $(COVERAGE_LDFLAGS) \ + $(LIBARCHIVE_LDFLAGS) \ $(NULL) virt_host_validate_LDADD = \ ../src/libvirt.la \ - ../gnulib/lib/libgnu.la \ + ../gnulib/lib/libgnu.la \ + $(LIBARCHIVE_LDFLAGS) \ $(NULL) virt_host_validate_CFLAGS = \ $(LIBXML_CFLAGS) \ + $(LIBARCHIVE_CFLAGS) \ $(WARN_CFLAGS) \ $(PIE_CFLAGS) \ $(COVERAGE_CFLAGS) \ @@ -165,15 +172,18 @@ virt_login_shell_SOURCES = \ virt_login_shell_LDFLAGS = \ $(AM_LDFLAGS) \ $(COVERAGE_LDFLAGS) \ + $(LIBARHCIVE_LDFLAGS) \ $(NULL) virt_login_shell_LDADD = \ $(STATIC_BINARIES) \ $(PIE_LDFLAGS) \ ../src/libvirt-setuid-rpc-client.la \ + $(LIBARCHIVE_LDFLAGS) \ ../gnulib/lib/libgnu.la virt_login_shell_CFLAGS = \ -DLIBVIRT_SETUID_RPC_CLIENT \ + $(LIBARCHIVE_CFLAGS) \ $(LIBXML_CFLAGS) \ $(WARN_CFLAGS) \ $(PIE_CFLAGS) \ @@ -202,6 +212,7 @@ virsh_SOURCES = \ virsh_LDFLAGS = \ $(AM_LDFLAGS) \ $(COVERAGE_LDFLAGS) \ + $(LIBARCHIVE_LDFLAGS) \ $(NULL) virsh_LDADD = \ $(STATIC_BINARIES) \ @@ -209,14 +220,16 @@ virsh_LDADD = \ ../src/libvirt.la \ ../src/libvirt-lxc.la \ ../src/libvirt-qemu.la \ - ../gnulib/lib/libgnu.la \ + ../gnulib/lib/libgnu.la \ $(LIBXML_LIBS) \ + $(LIBARCHIVE_LIBS) \ $(VIRSH_LIBS) virsh_CFLAGS = \ $(WARN_CFLAGS) \ $(PIE_CFLAGS) \ $(COVERAGE_CFLAGS) \ $(LIBXML_CFLAGS) \ + $(LIBARCHIVE_CFLAGS) \ $(READLINE_CFLAGS) BUILT_SOURCES = -- 2.5.0