The logic set up in previous patch for exposing VIR_TEST_EXPENSIVE to individual tests is as follows: make check VIR_TEST_EXPENSIVE=0 => getenv("VIR_TEST_EXPENSIVE") sees "0" make check VIR_TEST_EXPENSIVE=1 => getenv("VIR_TEST_EXPENSIVE") sees "1" make check => getenv("VIR_TEST_EXPENSIVE") sees either "0" or "1", based on configure options cd tests; ./FOOtest => getenv("VIR_TEST_EXPENSIVE") sees whatever is in your environment (usually NULL, but possibly garbage) Merely checking if VIR_TEST_EXPENSIVE is set in the environment does the wrong thing; likewise, it is unsafe to assume the variable will always contain a valid number. As such, it helps to have helper functions, instead of making each expensive test repeat the probe of the environment. * tests/testutils.h (virTestGetExpensive): New prototype. * tests/testutils.c (virTestGetExpensive): Implement it. * tests/test-lib.sh (very_expensive_): Rename... (test_expensive): ...and tweak to use VIR_TEST_EXPENSIVE. Signed-off-by: Eric Blake <eblake(a)redhat.com&gt; --- v2: new patch followup, no v1 tests/test-lib.sh | 29 ++++++++++++++++++++++------- tests/testutils.c | 8 ++++++++ tests/testutils.h | 1 + 3 files changed, 31 insertions(+), 7 deletions(-) diff --git a/tests/test-lib.sh b/tests/test-lib.sh index 918bf73..2f79706 100644 --- a/tests/test-lib.sh +++ b/tests/test-lib.sh @@ -1,4 +1,22 @@ -# source this file; set up for tests +# test-lib.sh: source this file; set up for tests + +# Copyright (C) 2008-2013 Red Hat, Inc. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see +# <http://www.gnu.org/licenses/>;. +# +# Based on an idea from GNU coreutils test -z "$abs_srcdir" && abs_srcdir=$(pwd) test -z "$abs_builddir" && abs_builddir=$(pwd) @@ -158,15 +176,12 @@ require_selinux_() esac } -very_expensive_() +test_expensive() { - if test "$RUN_VERY_EXPENSIVE_TESTS" != yes; then + if test "$VIR_TEST_EXPENSIVE" != 1; then skip_test_ ' This test is very expensive, so it is disabled by default. -To run it anyway, rerun make check with the RUN_VERY_EXPENSIVE_TESTS -environment variable set to yes. E.g., - - env RUN_VERY_EXPENSIVE_TESTS=yes make check +To run it anyway, rerun: make check VIR_TEST_EXPENSIVE=1 ' fi } diff --git a/tests/testutils.c b/tests/testutils.c index 72aa5b3..c521552 100644 --- a/tests/testutils.c +++ b/tests/testutils.c @@ -66,6 +66,7 @@ static unsigned int testDebug = -1; static unsigned int testVerbose = -1; +static unsigned int testExpensive = -1;

static unsigned int testOOM = 0; static size_t testCounter = 0; @@ -581,6 +582,13 @@ virTestGetVerbose(void) { return testVerbose || virTestGetDebug(); } +unsigned int

+virTestGetExpensive(void) { + if (testExpensive == -1) + testExpensive = virTestGetFlag("VIR_TEST_EXPENSIVE"); + return testExpensive; +} + int virtTestMain(int argc, char **argv, int (*func)(void))

The gnulib testsuite is relatively stable - the only times it is likely to have a test change from pass to fail is on a gnulib submodule update or a major system change (such as moving from Fedora 18 to 19, or other large change to libc). While it is an important test for end users on arbitrary machines (to make sure that the portability glue works for their machine), it mostly wastes time for development testing (as most developers aren't making any of the major changes that would cause gnulib tests to alter behavior). Thus, it pays to make the tests optional at configure time, defaulting to off for development, on for tarballs, with autobuilders requesting it to be on. It also helps to allow a make-time override, via VIR_TEST_EXPENSIVE=[01] (much the way automake sets up V=[01] for overriding the configure time default of how verbose to be). Automake has some pretty hard-coded magic with regards to the TESTS variable; I had quite a job figuring out how to keep 'make distcheck' passing regardless of the configure option setting in use, while still disabling the tests at runtime when I did not configure them on and did not use the override variable. Thankfully, we require GNU make, which lets me hide some information from Automake's magic handling of TESTS. * bootstrap.conf (bootstrap_epilogue): Munge gnulib test variable. * configure.ac (--enable-expensive-tests): Add new enable switch. (VIR_TEST_EXPENSIVE_DEFAULT, WITH_EXPENSIVE_TESTS): Set new witnesses. * gnulib/tests/Makefile.am (TESTS): Make tests conditional on configure settings and the VIR_TEST_EXPENSIVE variable. * tests/Makefile.am (TESTS_ENVIRONMENT): Expose VIR_TEST_EXPENSIVE to all tests. * autobuild.sh: Enable all tests during autobuilds. * libvirt.spec.in (%configure): Likewise. * mingw-libvirt.spec.in (%mingw_configure): Likewise. * docs/hacking.html.in: Document the option. * HACKING: Regenerate. Signed-off-by: Eric Blake <eblake(a)redhat.com&gt; --- v2: Change the configure name from --enable-gnulib-tests to --enable-expensive-tests; make VIR_TEST_EXPENSIVE=0 work to give a one-shot disable of tests at make time; export the configure default on to all tests HACKING | 9 +++++++++ autobuild.sh | 3 +++ bootstrap.conf | 3 ++- configure.ac | 21 +++++++++++++++++++++ docs/hacking.html.in | 12 ++++++++++++ gnulib/tests/Makefile.am | 15 ++++++++++++++- libvirt.spec.in | 1 + mingw-libvirt.spec.in | 4 ++-- tests/Makefile.am | 2 ++ 9 files changed, 66 insertions(+), 4 deletions(-) diff --git a/HACKING b/HACKING index 207b9ed..256e8ae 100644 --- a/HACKING +++ b/HACKING @@ -107,6 +107,15 @@ and run the tests: Valgrind <http://valgrind.org/> is a test that checks for memory management issues, such as leaks or use of uninitialized variables. +Some tests are skipped by default in a development environment, based on the +time they take in comparison to the likelihood that those tests will turn up +problems during incremental builds. These tests default to being run when when +building from a tarball or with the configure option --enable-expensive-tests; +you can also force a one-time toggle of these tests by setting +VIR_TEST_EXPENSIVE to 0 or 1 at make time, as in: + + make check VIR_TEST_EXPENSIVE=1 + If you encounter any failing tests, the VIR_TEST_DEBUG environment variable may provide extra information to debug the failures. Larger values of VIR_TEST_DEBUG may provide larger amounts of information: diff --git a/autobuild.sh b/autobuild.sh index 7da8cb5..e5aa35c 100755 --- a/autobuild.sh +++ b/autobuild.sh @@ -18,6 +18,7 @@ cd build # Run with options not normally exercised by the rpm build, for # more complete code coverage. ../autogen.sh --prefix="$AUTOBUILD_INSTALL_ROOT" \ + --enable-expensive-tests \ --enable-test-coverage \ --disable-nls \ --enable-werror \ @@ -76,6 +77,7 @@ if test -x /usr/bin/i686-w64-mingw32-gcc ; then --build=$(uname -m)-w64-linux \ --host=i686-w64-mingw32 \ --prefix="$AUTOBUILD_INSTALL_ROOT/i686-w64-mingw32/sys-root/mingw" \ + --enable-expensive-tests \ --enable-werror \ --without-libvirtd \ --without-python @@ -96,6 +98,7 @@ if test -x /usr/bin/x86_64-w64-mingw32-gcc ; then --build=$(uname -m)-w64-linux \ --host=x86_64-w64-mingw32 \ --prefix="$AUTOBUILD_INSTALL_ROOT/x86_64-w64-mingw32/sys-root/mingw" \ + --enable-expensive-tests \ --enable-werror \ --without-libvirtd \ --without-python diff --git a/bootstrap.conf b/bootstrap.conf index f166a53..a1d1f07 100644 --- a/bootstrap.conf +++ b/bootstrap.conf @@ -244,9 +244,10 @@ gnulib_extra_files=" bootstrap_epilogue() { # Change paths in gnulib/tests/gnulib.mk from "../../.." to "../..", + # and make tests conditional by changing "TESTS" to "GNULIB_TESTS", # then ensure that gnulib/tests/Makefile.in is up-to-date. m=gnulib/tests/gnulib.mk - sed 's,\.\./\.\./\.\.,../..,g' $m > $m-t + sed 's,\.\./\.\./\.\.,../..,g; s/^TESTS /GNULIB_TESTS /' $m > $m-t mv -f $m-t $m ${AUTOMAKE-automake} gnulib/tests/Makefile } diff --git a/configure.ac b/configure.ac index a155790..2c8cb86 100644 --- a/configure.ac +++ b/configure.ac @@ -1987,6 +1987,27 @@ fi AC_MSG_RESULT([$withval]) AM_CONDITIONAL([WITH_TESTS], [test "$withval" = "yes"]) +AC_ARG_ENABLE([expensive-tests], + [AC_HELP_STRING([--enable-expensive-tests], + [set the default for enabling expensive tests (gnulib and long timeouts) ] + [@<:@default=check@:>@; use VIR_TEST_EXPENSIVE to override during make])], + [case $enableval in + 0|no) VIR_TEST_EXPENSIVE_DEFAULT=0 ;; + 1|yes) VIR_TEST_EXPENSIVE_DEFAULT=1 ;; + check) ;; + *) AC_MSG_ERROR([bad value ${enableval} for enable-expensive-tests option]) + ;; + esac], [enableval=check]) +if test "$enableval" = check; then + if test -d $srcdir/.git ; then + VIR_TEST_EXPENSIVE_DEFAULT=0 + else + VIR_TEST_EXPENSIVE_DEFAULT=1 + fi +fi +AC_SUBST([VIR_TEST_EXPENSIVE_DEFAULT]) +AM_CONDITIONAL([WITH_EXPENSIVE_TESTS], [test $VIR_TEST_EXPENSIVE_DEFAULT = 1]) + AC_ARG_ENABLE([test-coverage], AC_HELP_STRING([--enable-test-coverage], [turn on code coverage instrumentation @<:@default=no@:>@]), [case "${enableval}" in diff --git a/docs/hacking.html.in b/docs/hacking.html.in index 8120b19..0892b73 100644 --- a/docs/hacking.html.in +++ b/docs/hacking.html.in @@ -119,6 +119,18 @@ </p> <p> + Some tests are skipped by default in a development environment, + based on the time they take in comparison to the likelihood + that those tests will turn up problems during incremental builds. + These tests default to being run when when building from a + tarball or with the configure option --enable-expensive-tests; + you can also force a one-time toggle of these tests by + setting VIR_TEST_EXPENSIVE to 0 or 1 at make time, as in: + </p> +<pre> + make check VIR_TEST_EXPENSIVE=1 +</pre> + <p> If you encounter any failing tests, the VIR_TEST_DEBUG environment variable may provide extra information to debug the failures. Larger values of VIR_TEST_DEBUG may provide diff --git a/gnulib/tests/Makefile.am b/gnulib/tests/Makefile.am index 6a2f51b..74d71e9 100644 --- a/gnulib/tests/Makefile.am +++ b/gnulib/tests/Makefile.am @@ -1,4 +1,4 @@ -## Makefile for gnulib/lib -*-Makefile-*- +## Makefile for gnulib/lib ## Copyright (C) 2011, 2013 Red Hat, Inc. ## @@ -19,3 +19,16 @@ include gnulib.mk INCLUDES = $(GETTEXT_CPPFLAGS) + +GNULIB_TESTS0 = +GNULIB_TESTS1 = $(GNULIB_TESTS) +if WITH_EXPENSIVE_TESTS +## Automake requires that at least one conditional call out all tests to +## be run, for those tests to be shipped in the tarball +TESTS = $(GNULIB_TESTS) +endif +## However, we want to change the set of tests based on the make environment, +## where the default was set at configure time. Use GNU make constructs to +## hide our actions from Automake, so we don't get it too confused. +VIR_TEST_EXPENSIVE ?= $(VIR_TEST_EXPENSIVE_DEFAULT) +$(eval TESTS=$(GNULIB_TESTS$(VIR_TEST_EXPENSIVE))) diff --git a/libvirt.spec.in b/libvirt.spec.in index 79c5a2c..114b144 100644 --- a/libvirt.spec.in +++ b/libvirt.spec.in @@ -1408,6 +1408,7 @@ of recent versions of Linux (and other OSes). --with-qemu-user=%{qemu_user} \ --with-qemu-group=%{qemu_group} \ %{?enable_werror} \ + --enable-expensive-tests \ %{init_scripts} make %{?_smp_mflags} gzip -9 ChangeLog diff --git a/mingw-libvirt.spec.in b/mingw-libvirt.spec.in index aa39231..e13407e 100644 --- a/mingw-libvirt.spec.in +++ b/mingw-libvirt.spec.in @@ -156,8 +156,8 @@ autoreconf -if --without-parallels \ --without-netcf \ --without-audit \ - --without-dtrace - + --without-dtrace \ + --enable-expensive-tests %mingw_make %{?_smp_mflags} diff --git a/tests/Makefile.am b/tests/Makefile.am index 789de9f..66ae610 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -307,6 +307,7 @@ lv_abs_top_builddir=`cd '$(top_builddir)'; pwd` path_add = $(subst :,$(PATH_SEPARATOR),\ $(subst !,$(lv_abs_top_builddir)/,!daemon:!tools:!tests)) +VIR_TEST_EXPENSIVE ?= $(VIR_TEST_EXPENSIVE_DEFAULT) TESTS_ENVIRONMENT = \ abs_top_builddir=$(lv_abs_top_builddir) \ abs_top_srcdir=`cd '$(top_srcdir)'; pwd` \ @@ -318,6 +319,7 @@ TESTS_ENVIRONMENT = \ LIBVIRT_DRIVER_DIR="$(abs_top_builddir)/src/.libs" \ LIBVIRT_AUTOSTART=0 \ LC_ALL=C \ + VIR_TEST_EXPENSIVE=$(VIR_TEST_EXPENSIVE) \ $(VG)