Re: [libvirt] PATCH: 2/4: Generic test script infrastructure

This patch adds more helper functions to the tests/testutils.c file which make it trivial to verify OOM handling in our test suites. It provides a virtTestMain() function which is the main driver. This is given the original argc, argv and a function callback representing the test suite to run. Next, instead of a test suite defining a 'main()' function itself, it calls it something else like 'mymain()', and then declared VIRT_TEST_MAIN(mymain). This will cause the test suite to be run via the generic helper. In normal use the virtTestMain() function simply calls mymain() directly and everything runs as it does now. If you set the VIR_TEST_OOM environment variable to a positive integer it'll perform OOM testing. The value of the variable is the number of consequtive allocations to fails. eg VIR_TEST_OOM=1 will only fail a single allocation each time, while VIR_TEST_OOM=5 will fail batches of 5 allocations. As described in the previous patch, the way it works is - Run mymain() and get a count of allocations - Foreach n in count - Schedule the n'th alocation to fail - Run mymain() and validate that it returns EXIT_FAILURE So, now you can do VIR_TEST_OOM=1 make check And it'll perform OOM checking on any test suite leveraging this testutils infrastructure. You can also do it directly on individual testsuites VIR_TEST_OOM=1 ./qparamstest And notice when it runs, you'll have an extra test case at the end where it verifies each allocation: $ VIR_TEST_OOM=1 ./qparamtest 1) Parse foo=one&bar=two ... OK 2) Format foo=one&bar=two ... OK 3) Build foo=one&bar=two ... OK 4) Parse foo=one&foo=two ... OK 5) Format foo=one&foo=two ... OK 6) Build foo=one&foo=two ... OK 7) Parse foo=one&&foo=two ... OK 8) Format foo=one&&foo=two ... OK 9) Build foo=one&&foo=two ... OK 10) Parse foo=one;foo=two ... OK 11) Format foo=one;foo=two ... OK 12) Build foo=one;foo=two ... OK 13) Parse foo ... OK 14) Format foo ... OK 15) Build foo ... OK 16) Parse foo= ... OK 17) Format foo= ... OK 18) Build foo= ... OK 19) Parse foo=& ... OK 20) Format foo=& ... OK 21) Build foo=& ... OK 22) Parse foo=&& ... OK 23) Format foo=&& ... OK 24) Build foo=&& ... OK 25) Parse foo=one%20two ... OK 26) Format foo=one%20two ... OK 27) Build foo=one%20two ... OK 28) Parse =bogus&foo=one ... OK 29) Format =bogus&foo=one ... OK 30) Build =bogus&foo=one ... OK 31) New vargs ... OK 32) Add vargs ... OK 33) OOM of 78 allocs .............................................................................. OK If any fails it'll immediately stop and you'll get a FAIL message. The testutils.c code makes use of the backtrace() function in glibc to provide optional stack traces for every allocation. It will only print the binary name and address, so it needs postprocessing with oomtrace.pl to resolve into source code line numbers. So to find out which allocation was not handled correctly you'd re-run with VIR_TEST_DEBUG=1 VIR_TEST_OOM=1 ./qparamstest 2>&1| perl oomtrace.pl ....cut ... Failing an allocation at: /home/berrange/src/xen/libvirt-numa/tests/testutils.c:305 /home/berrange/src/xen/libvirt-numa/src/memory.c:74 /home/berrange/src/xen/libvirt-numa/src/memory.c:180 /home/berrange/src/xen/libvirt-numa/src/qparams.c:101 /home/berrange/src/xen/libvirt-numa/src/qparams.c:86 /home/berrange/src/xen/libvirt-numa/tests/qparamtest.c:152 /home/berrange/src/xen/libvirt-numa/tests/testutils.c:90 /home/berrange/src/xen/libvirt-numa/tests/qparamtest.c:222 /home/berrange/src/xen/libvirt-numa/tests/testutils.c:378 /home/berrange/src/xen/libvirt-numa/tests/qparamtest.c:228 ??:0 ??:0 ... FAILED So we can see the trace of the first allocation which was not handled. b/tests/oomtrace.pl | 31 ++++++++++ tests/Makefile.am | 1 tests/testutils.c | 153 ++++++++++++++++++++++++++++++++++++++++++++++------ tests/testutils.h | 9 +++ 4 files changed, 179 insertions(+), 15 deletions(-) Regards, Daniel diff -r 9f962ac84b09 tests/Makefile.am --- a/tests/Makefile.am Wed May 21 19:42:55 2008 -0400 +++ b/tests/Makefile.am Wed May 21 22:22:48 2008 -0400 @@ -33,6 +33,7 @@ $(COVERAGE_LDFLAGS) EXTRA_DIST = \ + oomtrace.pl \ test-lib.sh \ xmlrpcserver.py \ test_conf.sh \ diff -r 9f962ac84b09 tests/oomtrace.pl --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tests/oomtrace.pl Wed May 21 22:22:48 2008 -0400 @@ -0,0 +1,31 @@ +#!/usr/bin/perl + +use strict; +use warnings; + +my @data = <>; + + +my %trace; +my %lines; + +foreach (@data) { + if (/^\s*TRACE:\s+(\S+?)(?:\(.*\))?\s+\[0x(.*)\]\s*$/ ) { + $trace{$2} = $1; + } +} + +foreach my $key (keys %trace) { + my $val = $trace{$key}; + my $info = $val =~ /\?\?/ ? $val : `addr2line -e $val $key`; + $lines{$key} = $info; +} + + +foreach (@data) { + if (/^\s*TRACE:\s+(\S+?)(?:\(.*\))?\s+\[0x(.*)\]\s*$/ ) { + print $lines{$2}; + } else { + print; + } +} diff -r 9f962ac84b09 tests/testutils.c --- a/tests/testutils.c Wed May 21 19:42:55 2008 -0400 +++ b/tests/testutils.c Wed May 21 22:22:48 2008 -0400 @@ -24,6 +24,12 @@ #include <limits.h> #include "testutils.h" #include "internal.h" +#include "memory.h" +#include "util.h" + +#if HAVE_TRACE +#include <execinfo.h> +#endif #ifdef HAVE_PATHS_H #include <paths.h> @@ -37,6 +43,10 @@ #define DIFF_MSEC(T, U) \ ((((int) ((T)->tv_sec - (U)->tv_sec)) * 1000000.0 + \ ((int) ((T)->tv_usec - (U)->tv_usec))) / 1000.0) + +static int testOOM = 0; +static int testDebug = 0; +static int testCounter = 0; double virtTestCountAverage(double *items, int nitems) @@ -60,12 +70,13 @@ { int i, ret = 0; double *ts = NULL; - static int counter = 0; - counter++; + testCounter++; - fprintf(stderr, "%2d) %-65s ... ", counter, title); - fflush(stderr); + if (testOOM < 2) { + fprintf(stderr, "%2d) %-65s ... ", testCounter, title); + fflush(stderr); + } if (nloops > 1 && (ts = calloc(nloops, sizeof(double)))==NULL) @@ -83,13 +94,15 @@ ts[i] = DIFF_MSEC(&after, &before); } } - if (ret == 0 && ts) - fprintf(stderr, "OK [%.5f ms]\n", - virtTestCountAverage(ts, nloops)); - else if (ret == 0) - fprintf(stderr, "OK\n"); - else - fprintf(stderr, "FAILED\n"); + if (testOOM < 2) { + if (ret == 0 && ts) + fprintf(stderr, "OK [%.5f ms]\n", + virtTestCountAverage(ts, nloops)); + else if (ret == 0) + fprintf(stderr, "OK\n"); + else + fprintf(stderr, "FAILED\n"); + } free(ts); return ret; @@ -232,13 +245,14 @@ const char *expectEnd = expect + (strlen(expect)-1); const char *actualStart = actual; const char *actualEnd = actual + (strlen(actual)-1); - const char *debug; - if ((debug = getenv("DEBUG_TESTS")) == NULL) + if (testOOM < 2) return 0; - if (STREQ(debug, "") || - STREQ(debug, "1")) { + if (!testDebug) + return 0; + + if (testDebug < 2) { /* Skip to first character where they differ */ while (*expectStart && *actualStart && *actualStart == *expectStart) { @@ -272,3 +286,112 @@ return 0; } + +static void +virtTestErrorFuncQuiet(void *data ATTRIBUTE_UNUSED, + virErrorPtr err ATTRIBUTE_UNUSED) +{ } + +static void +virtTestErrorHook(void *data ATTRIBUTE_UNUSED) +{ +#if HAVE_TRACE + void *trace[30]; + int ntrace = ARRAY_CARDINALITY(trace); + int i; + char **symbols = NULL; + + ntrace = backtrace(trace, ntrace); + symbols = backtrace_symbols(trace, ntrace); + if (symbols) { + fprintf(stderr, "Failing an allocation at:\n"); + for (i = 0 ; i < ntrace ; i++) { + if (symbols[i]) + fprintf(stderr, " TRACE: %s\n", symbols[i]); + } + free(symbols); + } +#endif +} + + +int virtTestMain(int argc, + char **argv, + int (*func)(int, char **)) +{ +#if TEST_OOM + int ret; + int approxAlloc = 0; + int n; + char *oomStr = NULL, *debugStr; + int oomCount; + + if ((debugStr = getenv("VIR_TEST_DEBUG")) != NULL) { + virStrToLong_i(debugStr, NULL, 10, &testDebug); + + if (testDebug < 0) + testDebug = 0; + } + + if ((oomStr = getenv("VIR_TEST_OOM")) != NULL) { + virStrToLong_i(oomStr, NULL, 10, &oomCount); + + if (oomCount < 0) + oomCount = 0; + if (oomCount) + testOOM = 1; + } + + if (testOOM) + virAllocTestInit(); + + /* Run once to count allocs, and ensure it passes :-) */ + ret = (func)(argc, argv); + if (ret != EXIT_SUCCESS) + return EXIT_FAILURE; + + if (testDebug) + virAllocTestHook(virtTestErrorHook, NULL); + + + if (testOOM) { + /* Makes next test runs quiet... */ + testOOM++; + virSetErrorFunc(NULL, virtTestErrorFuncQuiet); + + approxAlloc = virAllocTestCount(); + testCounter++; + if (testDebug) + fprintf(stderr, "%d) OOM...\n", testCounter); + else + fprintf(stderr, "%d) OOM of %d allocs ", testCounter, approxAlloc); + + /* Run once for each alloc, failing a different one + and validating that the test case failed */ + for (n = 0; n < approxAlloc ; n++) { + if (!testDebug) { + fprintf(stderr, "."); + fflush(stderr); + } + virAllocTestOOM(n+1, oomCount); + + if (((func)(argc, argv)) != EXIT_FAILURE) { + ret = EXIT_FAILURE; + break; + } + } + + if (testDebug) + fprintf(stderr, " ... OOM of %d allocs", approxAlloc); + + if (ret == EXIT_SUCCESS) + fprintf(stderr, " OK\n"); + else + fprintf(stderr, " FAILED\n"); + } + return ret; + +#else + return (func)(argc, argv); +#endif +} diff -r 9f962ac84b09 tests/testutils.h --- a/tests/testutils.h Wed May 21 19:42:55 2008 -0400 +++ b/tests/testutils.h Wed May 21 22:22:48 2008 -0400 @@ -37,6 +37,15 @@ const char *expect, const char *actual); + int virtTestMain(int argc, + char **argv, + int (*func)(int, char **)); + +#define VIRT_TEST_MAIN(func) \ + int main(int argc, char **argv) { \ + return virtTestMain(argc,argv, func); \ + } + #ifdef __cplusplus } #endif -- |: Red Hat, Engineering, Boston -o- http://people.redhat.com/berrange/ :| |: http://libvirt.org -o- http://virt-manager.org -o- http://ovirt.org :| |: http://autobuild.org -o- http://search.cpan.org/~danberr/ :| |: GnuPG: 7D3B9505 -o- F3C9 553F A1DA 4AC2 5648 23C1 B3DF F742 7D3B 9505 :|