[libvirt] PATCH: 0/4: Add out-of-memory validation to the testsuite
The following series of patches add infrastructure neccessary to let us test OOM handling for any libvirt code using our malloc wrappers. Currently only a few places use this, but as this series will show, it already lets us find several bugs. As we convert more code away from malloc to VIR_MALLOC we'll get ever improved coverage. Not testing with malloc() directly also means we avoid having to debug code in libraries we link to. Although obviously it'd be nice if they had correct OOM handling, that's their job to debug not ours... Regards, Daniel. -- |: Red Hat, Engineering, Boston -o- http://people.redhat.com/berrange/ :| |: http://libvirt.org -o- http://virt-manager.org -o- http://ovirt.org :| |: http://autobuild.org -o- http://search.cpan.org/~danberr/ :| |: GnuPG: 7D3B9505 -o- F3C9 553F A1DA 4AC2 5648 23C1 B3DF F742 7D3B 9505 :|
This patch adds extra code to src/memory.c which allows us to force an OOM condition on specific allocations. This is not code you *ever* want to use in a production build, so its all conditional on TEST_OOM, which is enabled by passing --enable-test-oom to the configure script. The hooks work as follows... - The test suite first calls virAllocTestInit() to initialize the hooks. This causes it to start counting allocations. ....then run the code you want to check for OOM... - Next call virAllocTestCount() to find out how many allocations were made. - Given a number of allocations 'n', we need to repeat 'n' times... - Call virAllocTestOOM(n, m) to tell it to fail the n'th allocation upto the (n + m -1)'th allocation. eg, virAllocTestOOM(3, 2) will cause allocations 3 and 4 to fail. ... run the code you want to check and verify it reports OOM in the way you expect. It can be quite hard to find out just where allocation failure bugs are hiding. So there is also a virAllocTestHook() function which lets you register a callback to be invoked at the time an allocation is artificially failed. Obvious use for this is to capture a stack trace. configure.in | 29 +++++++++++++++++++++-- src/memory.c | 74 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++-- src/memory.h | 11 +++++++- 3 files changed, 109 insertions(+), 5 deletions(-) Regards, Daniel diff -r 9f962ac84b09 configure.in --- a/configure.in Wed May 21 19:42:55 2008 -0400 +++ b/configure.in Wed May 21 22:22:47 2008 -0400 @@ -883,19 +883,39 @@ AM_CONDITIONAL([ENABLE_XEN_TESTS], [test "$RUNNING_XEN" != "no" -a "$RUNNING_XEND" != "no"]) AC_ARG_ENABLE([test-coverage], -[ --enable-test-coverage turn on code coverage instrumentation], +[ --enable-test-coverage turn on code coverage instrumentation], [case "${enableval}" in yes|no) ;; *) AC_MSG_ERROR([bad value ${enableval} for test-coverage option]) ;; esac], [enableval=no]) +enable_coverage=$enableval -if test "${enableval}" = yes; then +if test "${enable_coverage}" = yes; then gl_COMPILER_FLAGS(-fprofile-arcs) gl_COMPILER_FLAGS(-ftest-coverage) AC_SUBST([COVERAGE_CFLAGS], [$COMPILER_FLAGS]) AC_SUBST([COVERAGE_LDFLAGS], [$COMPILER_FLAGS]) COMPILER_FLAGS= +fi + +AC_ARG_ENABLE([test-oom], +[ --enable-test-oom memory allocation failure checking], +[case "${enableval}" in + yes|no) ;; + *) AC_MSG_ERROR([bad value ${enableval} for test-oom option]) ;; + esac], + [enableval=no]) +enable_oom=$enableval + +if test "${enable_oom}" = yes; then + have_trace=yes + AC_CHECK_HEADER([execinfo.h],[],[have_trace=no]) + AC_CHECK_FUNC([backtrace],[],[have_trace=no]) + if test "$have_trace" = "yes"; then + AC_DEFINE([HAVE_TRACE], 1, [Whether backtrace() is available]) + fi + AC_DEFINE([TEST_OOM], 1, [Whether malloc OOM checking is enabled]) fi dnl Enable building the proxy? @@ -1042,6 +1062,11 @@ AC_MSG_NOTICE([ numactl: no]) fi AC_MSG_NOTICE([]) +AC_MSG_NOTICE([Test suite]) +AC_MSG_NOTICE([]) +AC_MSG_NOTICE([ Coverage: $enable_coverage]) +AC_MSG_NOTICE([ Alloc OOM: $enable_oom]) +AC_MSG_NOTICE([]) AC_MSG_NOTICE([Miscellaneous]) AC_MSG_NOTICE([]) AC_MSG_NOTICE([ Debug: $enable_debug]) diff -r 9f962ac84b09 src/memory.c --- a/src/memory.c Wed May 21 19:42:55 2008 -0400 +++ b/src/memory.c Wed May 21 22:22:47 2008 -0400 @@ -26,6 +26,59 @@ #include "memory.h" +#if TEST_OOM +static int testMallocNext = 0; +static int testMallocFailFirst = 0; +static int testMallocFailLast = 0; +static void (*testMallocHook)(void*) = NULL; +static void *testMallocHookData = NULL; + +void virAllocTestInit(void) +{ + testMallocNext = 1; + testMallocFailFirst = 0; + testMallocFailLast = 0; +} + +int virAllocTestCount(void) +{ + return testMallocNext - 1; +} + +void virAllocTestHook(void (*func)(void*), void *data) +{ + testMallocHook = func; + testMallocHookData = data; +} + +void virAllocTestOOM(int n, int m) +{ + testMallocNext = 1; + testMallocFailFirst = n; + testMallocFailLast = n + m - 1; +} + +static int virAllocTestFail(void) +{ + int fail = 0; + if (testMallocNext == 0) + return 0; + + fail = + testMallocNext >= testMallocFailFirst && + testMallocNext <= testMallocFailLast; + + //printf("Alloc %d %d in [%d-%d]\n", fail, testMallocNext, testMallocFailFirst, testMallocFailLast); + + if (fail && testMallocHook) + (testMallocHook)(testMallocHookData); + + testMallocNext++; + return fail; +} +#endif + + /* Return 1 if an array of N objects, each of size S, cannot exist due to size arithmetic overflow. S must be positive and N must be nonnegative. This is a macro, not an inline function, so that it @@ -55,12 +108,17 @@ */ int virAlloc(void *ptrptr, size_t size) { +#if TEST_OOM + if (virAllocTestFail()) { + *(void **)ptrptr = NULL; + return -1; + } +#endif + if (size == 0) { *(void **)ptrptr = NULL; return 0; } - - *(void **)ptrptr = calloc(1, size); if (*(void **)ptrptr == NULL) @@ -83,6 +141,13 @@ */ int virAllocN(void *ptrptr, size_t size, size_t count) { +#if TEST_OOM + if (virAllocTestFail()) { + *(void **)ptrptr = NULL; + return -1; + } +#endif + if (size == 0 || count == 0) { *(void **)ptrptr = NULL; return 0; @@ -111,6 +176,11 @@ int virReallocN(void *ptrptr, size_t size, size_t count) { void *tmp; +#if TEST_OOM + if (virAllocTestFail()) + return -1; +#endif + if (size == 0 || count == 0) { free(*(void **)ptrptr); *(void **)ptrptr = NULL; diff -r 9f962ac84b09 src/memory.h --- a/src/memory.h Wed May 21 19:42:55 2008 -0400 +++ b/src/memory.h Wed May 21 22:22:47 2008 -0400 @@ -30,7 +30,6 @@ int virAllocN(void *ptrptr, size_t size, size_t count) ATTRIBUTE_RETURN_CHECK; int virReallocN(void *ptrptr, size_t size, size_t count) ATTRIBUTE_RETURN_CHECK; void virFree(void *ptrptr); - /** * VIR_ALLOC: @@ -79,4 +78,14 @@ */ #define VIR_FREE(ptr) virFree(&(ptr)); + +#if TEST_OOM +void virAllocTestInit(void); +int virAllocTestCount(void); +void virAllocTestOOM(int n, int m); +void virAllocTestHook(void (*func)(void*), void *data); +#endif + + + #endif /* __VIR_MEMORY_H_ */ -- |: Red Hat, Engineering, Boston -o- http://people.redhat.com/berrange/ :| |: http://libvirt.org -o- http://virt-manager.org -o- http://ovirt.org :| |: http://autobuild.org -o- http://search.cpan.org/~danberr/ :| |: GnuPG: 7D3B9505 -o- F3C9 553F A1DA 4AC2 5648 23C1 B3DF F742 7D3B 9505 :|
"Daniel P. Berrange" <berrange@redhat.com> wrote:
This patch adds extra code to src/memory.c which allows us to force an OOM condition on specific allocations. This is not code you *ever* want to use in a production build, so its all conditional on TEST_OOM, which is enabled by passing --enable-test-oom to the configure script.
The hooks work as follows...
- The test suite first calls virAllocTestInit() to initialize the hooks. This causes it to start counting allocations.
....then run the code you want to check for OOM...
- Next call virAllocTestCount() to find out how many allocations were made.
- Given a number of allocations 'n', we need to repeat 'n' times...
- Call virAllocTestOOM(n, m) to tell it to fail the n'th allocation upto the (n + m -1)'th allocation. eg, virAllocTestOOM(3, 2) will cause allocations 3 and 4 to fail.
... run the code you want to check and verify it reports OOM in the way you expect.
It can be quite hard to find out just where allocation failure bugs are hiding. So there is also a virAllocTestHook() function which lets you register a callback to be invoked at the time an allocation is artificially failed. Obvious use for this is to capture a stack trace.
Cool!
diff -r 9f962ac84b09 configure.in ... +if test "${enable_oom}" = yes; then + have_trace=yes + AC_CHECK_HEADER([execinfo.h],[],[have_trace=no]) + AC_CHECK_FUNC([backtrace],[],[have_trace=no]) + if test "$have_trace" = "yes"; then + AC_DEFINE([HAVE_TRACE], 1, [Whether backtrace() is available])
Please use some prefix other than "HAVE_" here. The HAVE_* namespace belongs to autoconf, and if some other autoconf macro ever does AC_CHECK_FUNCS([trace]), that will also define (or not) HAVE_TRACE. For compound features like this, it is common to use the USE_ prefix. So you might want to USE_BACKTRACE instead. Otherwise, ACK.
On Thu, May 22, 2008 at 06:17:27PM +0100, Daniel P. Berrange wrote:
This patch adds extra code to src/memory.c which allows us to force an OOM condition on specific allocations. This is not code you *ever* want to use in a production build, so its all conditional on TEST_OOM, which is enabled by passing --enable-test-oom to the configure script.
The hooks work as follows...
- The test suite first calls virAllocTestInit() to initialize the hooks. This causes it to start counting allocations.
....then run the code you want to check for OOM...
- Next call virAllocTestCount() to find out how many allocations were made.
- Given a number of allocations 'n', we need to repeat 'n' times...
- Call virAllocTestOOM(n, m) to tell it to fail the n'th allocation upto the (n + m -1)'th allocation. eg, virAllocTestOOM(3, 2) will cause allocations 3 and 4 to fail.
... run the code you want to check and verify it reports OOM in the way you expect.
It can be quite hard to find out just where allocation failure bugs are hiding. So there is also a virAllocTestHook() function which lets you register a callback to be invoked at the time an allocation is artificially failed. Obvious use for this is to capture a stack trace.
Fine by me. +1 Daniel -- Red Hat Virtualization group http://redhat.com/virtualization/ Daniel Veillard | virtualization library http://libvirt.org/ veillard@redhat.com | libxml GNOME XML XSLT toolkit http://xmlsoft.org/ http://veillard.com/ | Rpmfind RPM search engine http://rpmfind.net/
This patch adds more helper functions to the tests/testutils.c file which make it trivial to verify OOM handling in our test suites. It provides a virtTestMain() function which is the main driver. This is given the original argc, argv and a function callback representing the test suite to run. Next, instead of a test suite defining a 'main()' function itself, it calls it something else like 'mymain()', and then declared VIRT_TEST_MAIN(mymain). This will cause the test suite to be run via the generic helper. In normal use the virtTestMain() function simply calls mymain() directly and everything runs as it does now. If you set the VIR_TEST_OOM environment variable to a positive integer it'll perform OOM testing. The value of the variable is the number of consequtive allocations to fails. eg VIR_TEST_OOM=1 will only fail a single allocation each time, while VIR_TEST_OOM=5 will fail batches of 5 allocations. As described in the previous patch, the way it works is - Run mymain() and get a count of allocations - Foreach n in count - Schedule the n'th alocation to fail - Run mymain() and validate that it returns EXIT_FAILURE So, now you can do VIR_TEST_OOM=1 make check And it'll perform OOM checking on any test suite leveraging this testutils infrastructure. You can also do it directly on individual testsuites VIR_TEST_OOM=1 ./qparamstest And notice when it runs, you'll have an extra test case at the end where it verifies each allocation: $ VIR_TEST_OOM=1 ./qparamtest 1) Parse foo=one&bar=two ... OK 2) Format foo=one&bar=two ... OK 3) Build foo=one&bar=two ... OK 4) Parse foo=one&foo=two ... OK 5) Format foo=one&foo=two ... OK 6) Build foo=one&foo=two ... OK 7) Parse foo=one&&foo=two ... OK 8) Format foo=one&&foo=two ... OK 9) Build foo=one&&foo=two ... OK 10) Parse foo=one;foo=two ... OK 11) Format foo=one;foo=two ... OK 12) Build foo=one;foo=two ... OK 13) Parse foo ... OK 14) Format foo ... OK 15) Build foo ... OK 16) Parse foo= ... OK 17) Format foo= ... OK 18) Build foo= ... OK 19) Parse foo=& ... OK 20) Format foo=& ... OK 21) Build foo=& ... OK 22) Parse foo=&& ... OK 23) Format foo=&& ... OK 24) Build foo=&& ... OK 25) Parse foo=one%20two ... OK 26) Format foo=one%20two ... OK 27) Build foo=one%20two ... OK 28) Parse =bogus&foo=one ... OK 29) Format =bogus&foo=one ... OK 30) Build =bogus&foo=one ... OK 31) New vargs ... OK 32) Add vargs ... OK 33) OOM of 78 allocs .............................................................................. OK If any fails it'll immediately stop and you'll get a FAIL message. The testutils.c code makes use of the backtrace() function in glibc to provide optional stack traces for every allocation. It will only print the binary name and address, so it needs postprocessing with oomtrace.pl to resolve into source code line numbers. So to find out which allocation was not handled correctly you'd re-run with VIR_TEST_DEBUG=1 VIR_TEST_OOM=1 ./qparamstest 2>&1| perl oomtrace.pl ....cut ... Failing an allocation at: /home/berrange/src/xen/libvirt-numa/tests/testutils.c:305 /home/berrange/src/xen/libvirt-numa/src/memory.c:74 /home/berrange/src/xen/libvirt-numa/src/memory.c:180 /home/berrange/src/xen/libvirt-numa/src/qparams.c:101 /home/berrange/src/xen/libvirt-numa/src/qparams.c:86 /home/berrange/src/xen/libvirt-numa/tests/qparamtest.c:152 /home/berrange/src/xen/libvirt-numa/tests/testutils.c:90 /home/berrange/src/xen/libvirt-numa/tests/qparamtest.c:222 /home/berrange/src/xen/libvirt-numa/tests/testutils.c:378 /home/berrange/src/xen/libvirt-numa/tests/qparamtest.c:228 ??:0 ??:0 ... FAILED So we can see the trace of the first allocation which was not handled. b/tests/oomtrace.pl | 31 ++++++++++ tests/Makefile.am | 1 tests/testutils.c | 153 ++++++++++++++++++++++++++++++++++++++++++++++------ tests/testutils.h | 9 +++ 4 files changed, 179 insertions(+), 15 deletions(-) Regards, Daniel diff -r 9f962ac84b09 tests/Makefile.am --- a/tests/Makefile.am Wed May 21 19:42:55 2008 -0400 +++ b/tests/Makefile.am Wed May 21 22:22:48 2008 -0400 @@ -33,6 +33,7 @@ $(COVERAGE_LDFLAGS) EXTRA_DIST = \ + oomtrace.pl \ test-lib.sh \ xmlrpcserver.py \ test_conf.sh \ diff -r 9f962ac84b09 tests/oomtrace.pl --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tests/oomtrace.pl Wed May 21 22:22:48 2008 -0400 @@ -0,0 +1,31 @@ +#!/usr/bin/perl + +use strict; +use warnings; + +my @data = <>; + + +my %trace; +my %lines; + +foreach (@data) { + if (/^\s*TRACE:\s+(\S+?)(?:\(.*\))?\s+\[0x(.*)\]\s*$/ ) { + $trace{$2} = $1; + } +} + +foreach my $key (keys %trace) { + my $val = $trace{$key}; + my $info = $val =~ /\?\?/ ? $val : `addr2line -e $val $key`; + $lines{$key} = $info; +} + + +foreach (@data) { + if (/^\s*TRACE:\s+(\S+?)(?:\(.*\))?\s+\[0x(.*)\]\s*$/ ) { + print $lines{$2}; + } else { + print; + } +} diff -r 9f962ac84b09 tests/testutils.c --- a/tests/testutils.c Wed May 21 19:42:55 2008 -0400 +++ b/tests/testutils.c Wed May 21 22:22:48 2008 -0400 @@ -24,6 +24,12 @@ #include <limits.h> #include "testutils.h" #include "internal.h" +#include "memory.h" +#include "util.h" + +#if HAVE_TRACE +#include <execinfo.h> +#endif #ifdef HAVE_PATHS_H #include <paths.h> @@ -37,6 +43,10 @@ #define DIFF_MSEC(T, U) \ ((((int) ((T)->tv_sec - (U)->tv_sec)) * 1000000.0 + \ ((int) ((T)->tv_usec - (U)->tv_usec))) / 1000.0) + +static int testOOM = 0; +static int testDebug = 0; +static int testCounter = 0; double virtTestCountAverage(double *items, int nitems) @@ -60,12 +70,13 @@ { int i, ret = 0; double *ts = NULL; - static int counter = 0; - counter++; + testCounter++; - fprintf(stderr, "%2d) %-65s ... ", counter, title); - fflush(stderr); + if (testOOM < 2) { + fprintf(stderr, "%2d) %-65s ... ", testCounter, title); + fflush(stderr); + } if (nloops > 1 && (ts = calloc(nloops, sizeof(double)))==NULL) @@ -83,13 +94,15 @@ ts[i] = DIFF_MSEC(&after, &before); } } - if (ret == 0 && ts) - fprintf(stderr, "OK [%.5f ms]\n", - virtTestCountAverage(ts, nloops)); - else if (ret == 0) - fprintf(stderr, "OK\n"); - else - fprintf(stderr, "FAILED\n"); + if (testOOM < 2) { + if (ret == 0 && ts) + fprintf(stderr, "OK [%.5f ms]\n", + virtTestCountAverage(ts, nloops)); + else if (ret == 0) + fprintf(stderr, "OK\n"); + else + fprintf(stderr, "FAILED\n"); + } free(ts); return ret; @@ -232,13 +245,14 @@ const char *expectEnd = expect + (strlen(expect)-1); const char *actualStart = actual; const char *actualEnd = actual + (strlen(actual)-1); - const char *debug; - if ((debug = getenv("DEBUG_TESTS")) == NULL) + if (testOOM < 2) return 0; - if (STREQ(debug, "") || - STREQ(debug, "1")) { + if (!testDebug) + return 0; + + if (testDebug < 2) { /* Skip to first character where they differ */ while (*expectStart && *actualStart && *actualStart == *expectStart) { @@ -272,3 +286,112 @@ return 0; } + +static void +virtTestErrorFuncQuiet(void *data ATTRIBUTE_UNUSED, + virErrorPtr err ATTRIBUTE_UNUSED) +{ } + +static void +virtTestErrorHook(void *data ATTRIBUTE_UNUSED) +{ +#if HAVE_TRACE + void *trace[30]; + int ntrace = ARRAY_CARDINALITY(trace); + int i; + char **symbols = NULL; + + ntrace = backtrace(trace, ntrace); + symbols = backtrace_symbols(trace, ntrace); + if (symbols) { + fprintf(stderr, "Failing an allocation at:\n"); + for (i = 0 ; i < ntrace ; i++) { + if (symbols[i]) + fprintf(stderr, " TRACE: %s\n", symbols[i]); + } + free(symbols); + } +#endif +} + + +int virtTestMain(int argc, + char **argv, + int (*func)(int, char **)) +{ +#if TEST_OOM + int ret; + int approxAlloc = 0; + int n; + char *oomStr = NULL, *debugStr; + int oomCount; + + if ((debugStr = getenv("VIR_TEST_DEBUG")) != NULL) { + virStrToLong_i(debugStr, NULL, 10, &testDebug); + + if (testDebug < 0) + testDebug = 0; + } + + if ((oomStr = getenv("VIR_TEST_OOM")) != NULL) { + virStrToLong_i(oomStr, NULL, 10, &oomCount); + + if (oomCount < 0) + oomCount = 0; + if (oomCount) + testOOM = 1; + } + + if (testOOM) + virAllocTestInit(); + + /* Run once to count allocs, and ensure it passes :-) */ + ret = (func)(argc, argv); + if (ret != EXIT_SUCCESS) + return EXIT_FAILURE; + + if (testDebug) + virAllocTestHook(virtTestErrorHook, NULL); + + + if (testOOM) { + /* Makes next test runs quiet... */ + testOOM++; + virSetErrorFunc(NULL, virtTestErrorFuncQuiet); + + approxAlloc = virAllocTestCount(); + testCounter++; + if (testDebug) + fprintf(stderr, "%d) OOM...\n", testCounter); + else + fprintf(stderr, "%d) OOM of %d allocs ", testCounter, approxAlloc); + + /* Run once for each alloc, failing a different one + and validating that the test case failed */ + for (n = 0; n < approxAlloc ; n++) { + if (!testDebug) { + fprintf(stderr, "."); + fflush(stderr); + } + virAllocTestOOM(n+1, oomCount); + + if (((func)(argc, argv)) != EXIT_FAILURE) { + ret = EXIT_FAILURE; + break; + } + } + + if (testDebug) + fprintf(stderr, " ... OOM of %d allocs", approxAlloc); + + if (ret == EXIT_SUCCESS) + fprintf(stderr, " OK\n"); + else + fprintf(stderr, " FAILED\n"); + } + return ret; + +#else + return (func)(argc, argv); +#endif +} diff -r 9f962ac84b09 tests/testutils.h --- a/tests/testutils.h Wed May 21 19:42:55 2008 -0400 +++ b/tests/testutils.h Wed May 21 22:22:48 2008 -0400 @@ -37,6 +37,15 @@ const char *expect, const char *actual); + int virtTestMain(int argc, + char **argv, + int (*func)(int, char **)); + +#define VIRT_TEST_MAIN(func) \ + int main(int argc, char **argv) { \ + return virtTestMain(argc,argv, func); \ + } + #ifdef __cplusplus } #endif -- |: Red Hat, Engineering, Boston -o- http://people.redhat.com/berrange/ :| |: http://libvirt.org -o- http://virt-manager.org -o- http://ovirt.org :| |: http://autobuild.org -o- http://search.cpan.org/~danberr/ :| |: GnuPG: 7D3B9505 -o- F3C9 553F A1DA 4AC2 5648 23C1 B3DF F742 7D3B 9505 :|
"Daniel P. Berrange" <berrange@redhat.com> wrote:
This patch adds more helper functions to the tests/testutils.c file which make it trivial to verify OOM handling in our test suites.
Very nice! ACK
diff -r 9f962ac84b09 tests/oomtrace.pl --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tests/oomtrace.pl Wed May 21 22:22:48 2008 -0400 @@ -0,0 +1,31 @@ +#!/usr/bin/perl +use strict; +use warnings;
If you run ...| perl oomtrace.pl > /dev/full, it succeeds in spite of the write error. Insert these 8 lines and it will report the disk-full error. If my File::Coda module were standard, I'd say "use it". In the mean time, I use this (commented version at the URL): (my $ME = $0) =~ s|.*/||; # use File::Coda; # http://meyering.net/code/Coda/ END { defined fileno STDOUT or return; close STDOUT and return; warn "$ME: failed to close standard output: $!\n"; $? ||= 1; } ...
diff -r 9f962ac84b09 tests/testutils.c --- a/tests/testutils.c Wed May 21 19:42:55 2008 -0400 +++ b/tests/testutils.c Wed May 21 22:22:48 2008 -0400 ... +static int testOOM = 0; +static int testDebug = 0; +static int testCounter = 0;
If these can be unsigned, using an unsigned type (like size_t or uintmax_t) would make their declarations more readable, in that the reviewer wouldn't have to wonder if/when they go negative.
+int virtTestMain(int argc, + char **argv, + int (*func)(int, char **)) +{ +#if TEST_OOM + int ret; + int approxAlloc = 0; + int n; + char *oomStr = NULL, *debugStr; + int oomCount; + + if ((debugStr = getenv("VIR_TEST_DEBUG")) != NULL) { + virStrToLong_i(debugStr, NULL, 10, &testDebug); + + if (testDebug < 0) + testDebug = 0; + } + + if ((oomStr = getenv("VIR_TEST_OOM")) != NULL) { + virStrToLong_i(oomStr, NULL, 10, &oomCount);
When virStrToLong_i returns < 0, oomCount is not initialized, so when the string is invalid, either diagnose it or default oomCount to 0.
On Thu, May 22, 2008 at 06:28:25PM +0100, Daniel P. Berrange wrote:
This patch adds more helper functions to the tests/testutils.c file which make it trivial to verify OOM handling in our test suites.
It provides a virtTestMain() function which is the main driver. This is given the original argc, argv and a function callback representing the test suite to run. Next, instead of a test suite defining a 'main()' function itself, it calls it something else like 'mymain()', and then declared VIRT_TEST_MAIN(mymain). This will cause the test suite to be run via the generic helper.
In normal use the virtTestMain() function simply calls mymain() directly and everything runs as it does now.
If you set the VIR_TEST_OOM environment variable to a positive integer it'll perform OOM testing. The value of the variable is the number of consequtive allocations to fails. eg VIR_TEST_OOM=1 will only fail a single allocation each time, while VIR_TEST_OOM=5 will fail batches of 5 allocations.
As described in the previous patch, the way it works is
- Run mymain() and get a count of allocations
- Foreach n in count - Schedule the n'th alocation to fail - Run mymain() and validate that it returns EXIT_FAILURE
So, now you can do
VIR_TEST_OOM=1 make check
And it'll perform OOM checking on any test suite leveraging this testutils infrastructure. You can also do it directly on individual testsuites
VIR_TEST_OOM=1 ./qparamstest
okay, sounds cool. The problem is to make realistic testing while systematic one would explodes time-wise. I guess the 1 setting is probably sufficient Patch looks fine to me, +1 Daniel -- Red Hat Virtualization group http://redhat.com/virtualization/ Daniel Veillard | virtualization library http://libvirt.org/ veillard@redhat.com | libxml GNOME XML XSLT toolkit http://xmlsoft.org/ http://veillard.com/ | Rpmfind RPM search engine http://rpmfind.net/
The capabilities.c file was not checking for NULL pointers when cleaning up after some failed allocations, and so deferencing a NULL. The qparams.c file was not calling virRaiseError upon failure so there wasn't any indication of why it failed The qemu_conf.c file was not free'ing the sound card data in several places capabilities.c | 15 ++++++++++++++- qemu_conf.c | 10 +++++++++- qparams.c | 27 +++++++++++++++++++++++---- 3 files changed, 46 insertions(+), 6 deletions(-) Dan. diff -r 9f962ac84b09 src/capabilities.c --- a/src/capabilities.c Wed May 21 19:42:55 2008 -0400 +++ b/src/capabilities.c Wed May 21 22:22:47 2008 -0400 @@ -44,7 +44,7 @@ virCapsPtr caps; if (VIR_ALLOC(caps) < 0) - goto no_memory; + return NULL; if ((caps->host.arch = strdup(arch)) == NULL) goto no_memory; @@ -61,6 +61,9 @@ static void virCapabilitiesFreeHostNUMACell(virCapsHostNUMACellPtr cell) { + if (cell == NULL) + return; + VIR_FREE(cell->cpus); VIR_FREE(cell); } @@ -69,6 +72,9 @@ virCapabilitiesFreeGuestDomain(virCapsGuestDomainPtr dom) { int i; + if (dom == NULL) + return; + VIR_FREE(dom->info.emulator); VIR_FREE(dom->info.loader); for (i = 0 ; i < dom->info.nmachines ; i++) @@ -82,6 +88,8 @@ static void virCapabilitiesFreeGuestFeature(virCapsGuestFeaturePtr feature) { + if (feature == NULL) + return; VIR_FREE(feature->name); VIR_FREE(feature); } @@ -90,6 +98,9 @@ virCapabilitiesFreeGuest(virCapsGuestPtr guest) { int i; + if (guest == NULL) + return; + VIR_FREE(guest->ostype); VIR_FREE(guest->arch.name); @@ -120,6 +131,8 @@ void virCapabilitiesFree(virCapsPtr caps) { int i; + if (caps == NULL) + return; for (i = 0 ; i < caps->nguests ; i++) virCapabilitiesFreeGuest(caps->guests[i]); diff -r 7c1231eebae9 src/qemu_conf.c --- a/src/qemu_conf.c Thu May 22 12:31:13 2008 -0400 +++ b/src/qemu_conf.c Thu May 22 12:31:46 2008 -0400 @@ -215,6 +215,7 @@ struct qemud_vm_input_def *input = def->inputs; struct qemud_vm_chr_def *serial = def->serials; struct qemud_vm_chr_def *parallel = def->parallels; + struct qemud_vm_sound_def *sound = def->sounds; while (disk) { struct qemud_vm_disk_def *prev = disk; @@ -239,6 +240,11 @@ while (parallel) { struct qemud_vm_chr_def *prev = parallel; parallel = parallel->next; + free(prev); + } + while (sound) { + struct qemud_vm_sound_def *prev = sound; + sound = sound->next; free(prev); } xmlFree(def->keymap); @@ -2187,8 +2193,10 @@ } check = check->next; } - if (collision) + if (collision) { + free(sound); continue; + } def->nsounds++; sound->next = NULL; diff -r 9f962ac84b09 src/qparams.c --- a/src/qparams.c Wed May 21 19:42:55 2008 -0400 +++ b/src/qparams.c Wed May 21 22:22:47 2008 -0400 @@ -30,6 +30,14 @@ #include "memory.h" #include "qparams.h" +static void +qparam_report_oom(void) +{ + const char *virerr = __virErrorMsg(VIR_ERR_NO_MEMORY, NULL); + __virRaiseError(NULL, NULL, NULL, VIR_FROM_NONE, VIR_ERR_NO_MEMORY, VIR_ERR_ERROR, + virerr, NULL, NULL, -1, -1, virerr, NULL); +} + struct qparam_set * new_qparam_set (int init_alloc, ...) { @@ -39,12 +47,15 @@ if (init_alloc <= 0) init_alloc = 1; - if (VIR_ALLOC(ps) < 0) + if (VIR_ALLOC(ps) < 0) { + qparam_report_oom(); return NULL; + } ps->n = 0; ps->alloc = init_alloc; if (VIR_ALLOC_N(ps->p, ps->alloc) < 0) { VIR_FREE (ps); + qparam_report_oom(); return NULL; } @@ -88,7 +99,7 @@ { if (ps->n >= ps->alloc) { if (VIR_REALLOC_N(ps->p, ps->alloc * 2) < 0) { - perror ("realloc"); + qparam_report_oom(); return -1; } ps->alloc *= 2; @@ -104,12 +115,15 @@ char *pname, *pvalue; pname = strdup (name); - if (!pname) + if (!pname) { + qparam_report_oom(); return -1; + } pvalue = strdup (value); if (!pvalue) { VIR_FREE (pname); + qparam_report_oom(); return -1; } @@ -143,6 +157,7 @@ } if (virBufferError(&buf)) { + qparam_report_oom(); return NULL; } @@ -169,7 +184,10 @@ const char *end, *eq; ps = new_qparam_set (0, NULL); - if (!ps) return NULL; + if (!ps) { + qparam_report_oom(); + return NULL; + } if (!query || query[0] == '\0') return ps; @@ -240,6 +258,7 @@ return ps; out_of_memory: + qparam_report_oom(); free_qparam_set (ps); return NULL; } -- |: Red Hat, Engineering, Boston -o- http://people.redhat.com/berrange/ :| |: http://libvirt.org -o- http://virt-manager.org -o- http://ovirt.org :| |: http://autobuild.org -o- http://search.cpan.org/~danberr/ :| |: GnuPG: 7D3B9505 -o- F3C9 553F A1DA 4AC2 5648 23C1 B3DF F742 7D3B 9505 :|
"Daniel P. Berrange" <berrange@redhat.com> wrote:
The capabilities.c file was not checking for NULL pointers when cleaning up after some failed allocations, and so deferencing a NULL.
The qparams.c file was not calling virRaiseError upon failure so there wasn't any indication of why it failed
The qemu_conf.c file was not free'ing the sound card data in several places
Looks fine. ACK.
On Thu, May 22, 2008 at 06:36:16PM +0100, Daniel P. Berrange wrote:
The capabilities.c file was not checking for NULL pointers when cleaning up after some failed allocations, and so deferencing a NULL.
The qparams.c file was not calling virRaiseError upon failure so there wasn't any indication of why it failed
The qemu_conf.c file was not free'ing the sound card data in several places
Looks fine, +1 Daniel -- Red Hat Virtualization group http://redhat.com/virtualization/ Daniel Veillard | virtualization library http://libvirt.org/ veillard@redhat.com | libxml GNOME XML XSLT toolkit http://xmlsoft.org/ http://veillard.com/ | Rpmfind RPM search engine http://rpmfind.net/
This switches over a number of the test cases to make use of the new virTestMain() function and thus gain support for allocation checking So you can now actually see some checks when running VIR_TEST_OOM=1 make check Oh, and for added fun you can check if there are any leaks in the cleanup paths by layering in valgrind VIR_TEST_OOM=1 make valgrind which is superbly slow but finds a few things :-) nodeinfotest.c | 28 ++++++++++++++++------------ qemuxml2argvtest.c | 16 ++++++++++------ qemuxml2xmltest.c | 13 ++++++++----- qparamtest.c | 8 ++++++-- sexpr2xmltest.c | 13 ++++++++----- statstest.c | 11 +++++++---- xencapstest.c | 32 ++++++++++++++++++-------------- xmconfigtest.c | 11 +++++++---- xml2sexprtest.c | 10 ++++++---- 9 files changed, 86 insertions(+), 56 deletions(-) Regards, Daniel diff -r 7713868fe647 tests/nodeinfotest.c --- a/tests/nodeinfotest.c Thu May 22 12:59:37 2008 -0400 +++ b/tests/nodeinfotest.c Thu May 22 13:01:25 2008 -0400 @@ -3,13 +3,14 @@ #include <stdio.h> #include <stdlib.h> #include <string.h> +#include <unistd.h> #include "testutils.h" #include "internal.h" #include "nodeinfo.h" static char *progname; -static char *abs_top_srcdir; +static char *abs_srcdir; #define MAX_FILE 4096 @@ -56,17 +57,17 @@ static int linuxTestNodeInfo(const void *data) { char cpuinfo[PATH_MAX]; char output[PATH_MAX]; - snprintf(cpuinfo, PATH_MAX, "%s/tests/nodeinfodata/linux-%s.cpuinfo", - abs_top_srcdir, (const char*)data); - snprintf(output, PATH_MAX, "%s/tests/nodeinfodata/linux-%s.txt", - abs_top_srcdir, (const char*)data); + snprintf(cpuinfo, PATH_MAX, "%s/nodeinfodata/linux-%s.cpuinfo", + abs_srcdir, (const char*)data); + snprintf(output, PATH_MAX, "%s/nodeinfodata/linux-%s.txt", + abs_srcdir, (const char*)data); return linuxTestCompareFiles(cpuinfo, output); } #endif -int -main(int argc, char **argv) +static int +mymain(int argc, char **argv) { int ret = 0; #ifdef __linux__ @@ -79,16 +80,17 @@ "nodeinfo-5", "nodeinfo-6", }; + char cwd[PATH_MAX]; - abs_top_srcdir = getenv("abs_top_srcdir"); - if (!abs_top_srcdir) - return EXIT_FAILURE; + abs_srcdir = getenv("abs_srcdir"); + if (!abs_srcdir) + abs_srcdir = getcwd(cwd, sizeof(cwd)); progname = argv[0]; if (argc > 1) { fprintf(stderr, "Usage: %s\n", progname); - exit(EXIT_FAILURE); + return(EXIT_FAILURE); } virInitialize(); @@ -98,5 +100,7 @@ ret = -1; #endif - exit(ret==0 ? EXIT_SUCCESS : EXIT_FAILURE); + return(ret==0 ? EXIT_SUCCESS : EXIT_FAILURE); } + +VIRT_TEST_MAIN(mymain) diff -r 7713868fe647 tests/qemuxml2argvtest.c --- a/tests/qemuxml2argvtest.c Thu May 22 12:59:37 2008 -0400 +++ b/tests/qemuxml2argvtest.c Thu May 22 13:01:25 2008 -0400 @@ -43,6 +43,7 @@ if (!(vmdef = qemudParseVMDef(NULL, &driver, xmlData, "test"))) goto fail; + memset(&vm, 0, sizeof vm); vm.def = vmdef; vm.pid = -1; vm.id = -1; @@ -115,8 +116,8 @@ -int -main(int argc, char **argv) +static int +mymain(int argc, char **argv) { int ret = 0; char cwd[PATH_MAX]; @@ -125,14 +126,15 @@ if (argc > 1) { fprintf(stderr, "Usage: %s\n", progname); - exit(EXIT_FAILURE); + return (EXIT_FAILURE); } abs_srcdir = getenv("abs_srcdir"); if (!abs_srcdir) abs_srcdir = getcwd(cwd, sizeof(cwd)); - driver.caps = testQemuCapsInit(); + if ((driver.caps = testQemuCapsInit()) == NULL) + return EXIT_FAILURE; #define DO_TEST(name, extraFlags) \ do { \ @@ -181,11 +183,13 @@ virCapabilitiesFree(driver.caps); - exit(ret==0 ? EXIT_SUCCESS : EXIT_FAILURE); + return(ret==0 ? EXIT_SUCCESS : EXIT_FAILURE); } + +VIRT_TEST_MAIN(mymain) #else -int main (void) { exit (77); /* means 'test skipped' for automake */ } +int main (void) { return (77); /* means 'test skipped' for automake */ } #endif /* WITH_QEMU */ diff -r 7713868fe647 tests/qemuxml2xmltest.c --- a/tests/qemuxml2xmltest.c Thu May 22 12:59:37 2008 -0400 +++ b/tests/qemuxml2xmltest.c Thu May 22 13:01:25 2008 -0400 @@ -70,8 +70,8 @@ } -int -main(int argc, char **argv) +static int +mymain(int argc, char **argv) { int ret = 0; char cwd[PATH_MAX]; @@ -80,14 +80,15 @@ if (argc > 1) { fprintf(stderr, "Usage: %s\n", progname); - exit(EXIT_FAILURE); + return (EXIT_FAILURE); } abs_srcdir = getenv("abs_srcdir"); if (!abs_srcdir) abs_srcdir = getcwd(cwd, sizeof(cwd)); - driver.caps = testQemuCapsInit(); + if ((driver.caps = testQemuCapsInit()) == NULL) + return (EXIT_FAILURE); #define DO_TEST(name) \ if (virtTestRun("QEMU XML-2-XML " name, \ @@ -129,8 +130,10 @@ virCapabilitiesFree(driver.caps); - exit(ret==0 ? EXIT_SUCCESS : EXIT_FAILURE); + return (ret==0 ? EXIT_SUCCESS : EXIT_FAILURE); } + +VIRT_TEST_MAIN(mymain) #else diff -r 7713868fe647 tests/qparamtest.c --- a/tests/qparamtest.c Thu May 22 12:59:37 2008 -0400 +++ b/tests/qparamtest.c Thu May 22 13:01:25 2008 -0400 @@ -7,6 +7,7 @@ #include "testutils.h" #include "qparams.h" #include "util.h" +#include "memory.h" struct qparamParseDataEntry { const char *name; @@ -181,7 +182,8 @@ static const struct qparamParseDataEntry const params6[] = { { "foo", "one" } }; int -main(void) +mymain(int argc ATTRIBUTE_UNUSED, + char **argv ATTRIBUTE_UNUSED) { int ret = 0; @@ -220,5 +222,7 @@ if (virtTestRun("Add vargs", 1, qparamTestAddVargs, NULL) < 0) ret = -1; - exit(ret==0 ? EXIT_SUCCESS : EXIT_FAILURE); + return(ret==0 ? EXIT_SUCCESS : EXIT_FAILURE); } + +VIRT_TEST_MAIN(mymain) diff -r 7713868fe647 tests/sexpr2xmltest.c --- a/tests/sexpr2xmltest.c Thu May 22 12:59:37 2008 -0400 +++ b/tests/sexpr2xmltest.c Thu May 22 13:01:25 2008 -0400 @@ -69,8 +69,8 @@ } -int -main(int argc, char **argv) +static int +mymain(int argc, char **argv) { int ret = 0; char cwd[PATH_MAX]; @@ -79,7 +79,7 @@ if (argc > 1) { fprintf(stderr, "Usage: %s\n", progname); - exit(EXIT_FAILURE); + return(EXIT_FAILURE); } abs_srcdir = getenv("abs_srcdir"); @@ -88,7 +88,7 @@ if (argc > 1) { fprintf(stderr, "Usage: %s\n", progname); - exit(EXIT_FAILURE); + return(EXIT_FAILURE); } #define DO_TEST(in, out, version) \ @@ -139,8 +139,11 @@ DO_TEST("fv-sound", "fv-sound", 1); DO_TEST("fv-sound-all", "fv-sound-all", 1); - exit(ret==0 ? EXIT_SUCCESS : EXIT_FAILURE); + return(ret==0 ? EXIT_SUCCESS : EXIT_FAILURE); } + +VIRT_TEST_MAIN(mymain) + #else /* WITHOUT_XEN */ int main(void) diff -r 7713868fe647 tests/statstest.c --- a/tests/statstest.c Thu May 22 12:59:37 2008 -0400 +++ b/tests/statstest.c Thu May 22 13:01:25 2008 -0400 @@ -45,8 +45,9 @@ #endif -int -main(void) +static int +mymain(int argc ATTRIBUTE_UNUSED, + char **argv ATTRIBUTE_UNUSED) { int ret = 0; #if __linux__ && WITH_XEN @@ -54,7 +55,7 @@ * register a handler to stop error messages cluttering * up display */ - if (!getenv("DEBUG_TESTS")) + if (!getenv("VIR_TEST_DEBUG")) virSetErrorFunc(NULL, testQuietError); #define DO_TEST(dev, num) \ @@ -177,5 +178,7 @@ DO_TEST("/dev/xvda15", 51727); #endif - exit(ret==0 ? EXIT_SUCCESS : EXIT_FAILURE); + return(ret==0 ? EXIT_SUCCESS : EXIT_FAILURE); } + +VIRT_TEST_MAIN(mymain) diff -r 7713868fe647 tests/xencapstest.c --- a/tests/xencapstest.c Thu May 22 12:59:37 2008 -0400 +++ b/tests/xencapstest.c Thu May 22 13:01:25 2008 -0400 @@ -3,6 +3,7 @@ #include <stdio.h> #include <stdlib.h> #include <string.h> +#include <unistd.h> #ifdef WITH_XEN @@ -12,7 +13,7 @@ #include "xen_internal.h" static char *progname; -static char *abs_top_srcdir; +static char *abs_srcdir; #define MAX_FILE 4096 @@ -31,12 +32,12 @@ char cpuinfo[PATH_MAX]; char capabilities[PATH_MAX]; - snprintf(xml, sizeof xml - 1, "%s/tests/%s", - abs_top_srcdir, xml_rel); - snprintf(cpuinfo, sizeof cpuinfo - 1, "%s/tests/%s", - abs_top_srcdir, cpuinfo_rel); - snprintf(capabilities, sizeof capabilities - 1, "%s/tests/%s", - abs_top_srcdir, capabilities_rel); + snprintf(xml, sizeof xml - 1, "%s/%s", + abs_srcdir, xml_rel); + snprintf(cpuinfo, sizeof cpuinfo - 1, "%s/%s", + abs_srcdir, cpuinfo_rel); + snprintf(capabilities, sizeof capabilities - 1, "%s/%s", + abs_srcdir, capabilities_rel); if (virtTestLoadFile(xml, &expectxml, MAX_FILE) < 0) goto fail; @@ -147,21 +148,22 @@ } -int -main(int argc, char **argv) +static int +mymain(int argc, char **argv) { int ret = 0; + char cwd[PATH_MAX]; progname = argv[0]; if (argc > 1) { fprintf(stderr, "Usage: %s\n", progname); - exit(EXIT_FAILURE); + return(EXIT_FAILURE); } - abs_top_srcdir = getenv("abs_top_srcdir"); - if (!abs_top_srcdir) - return 1; + abs_srcdir = getenv("abs_srcdir"); + if (!abs_srcdir) + abs_srcdir = getcwd(cwd, sizeof(cwd)); virInitialize(); @@ -213,8 +215,10 @@ ret = -1; - exit(ret==0 ? EXIT_SUCCESS : EXIT_FAILURE); + return(ret==0 ? EXIT_SUCCESS : EXIT_FAILURE); } + +VIRT_TEST_MAIN(mymain) #else /* !WITH_XEN */ diff -r 7713868fe647 tests/xmconfigtest.c --- a/tests/xmconfigtest.c Thu May 22 12:59:37 2008 -0400 +++ b/tests/xmconfigtest.c Thu May 22 13:01:25 2008 -0400 @@ -170,8 +170,8 @@ } -int -main(int argc, char **argv) +static int +mymain(int argc, char **argv) { int ret = 0; char cwd[PATH_MAX]; @@ -180,7 +180,7 @@ if (argc > 1) { fprintf(stderr, "Usage: %s\n", progname); - exit(EXIT_FAILURE); + return(EXIT_FAILURE); } abs_srcdir = getenv("abs_srcdir"); @@ -223,8 +223,11 @@ DO_TEST("fullvirt-sound", 2); - exit(ret==0 ? EXIT_SUCCESS : EXIT_FAILURE); + return(ret==0 ? EXIT_SUCCESS : EXIT_FAILURE); } + +VIRT_TEST_MAIN(mymain) + #else /* WITHOUT_XEN */ int main(void) diff -r 7713868fe647 tests/xml2sexprtest.c --- a/tests/xml2sexprtest.c Thu May 22 12:59:37 2008 -0400 +++ b/tests/xml2sexprtest.c Thu May 22 13:01:25 2008 -0400 @@ -77,8 +77,8 @@ } -int -main(int argc, char **argv) +static int +mymain(int argc, char **argv) { int ret = 0; char cwd[PATH_MAX]; @@ -91,7 +91,7 @@ if (argc > 1) { fprintf(stderr, "Usage: %s\n", progname); - exit(EXIT_FAILURE); + return(EXIT_FAILURE); } #define DO_TEST(in, out, name, version) \ @@ -145,8 +145,10 @@ DO_TEST("fv-sound", "fv-sound", "fvtest", 1); - exit(ret==0 ? EXIT_SUCCESS : EXIT_FAILURE); + return(ret==0 ? EXIT_SUCCESS : EXIT_FAILURE); } + +VIRT_TEST_MAIN(mymain) #else /* WITH_XEN */ -- |: Red Hat, Engineering, Boston -o- http://people.redhat.com/berrange/ :| |: http://libvirt.org -o- http://virt-manager.org -o- http://ovirt.org :| |: http://autobuild.org -o- http://search.cpan.org/~danberr/ :| |: GnuPG: 7D3B9505 -o- F3C9 553F A1DA 4AC2 5648 23C1 B3DF F742 7D3B 9505 :|
"Daniel P. Berrange" <berrange@redhat.com> wrote:
This switches over a number of the test cases to make use of the new virTestMain() function and thus gain support for allocation checking
So you can now actually see some checks when running
In addition, this is changing s/exit/return/ and s/abs_top_srcdir/abs_srcdir/ [BTW, thanks. This latter has prodded me to do similar automated clean-up to coreutils' ~300 test scripts. ] ACK
VIR_TEST_OOM=1 make check
Oh, and for added fun you can check if there are any leaks in the cleanup paths by layering in valgrind
VIR_TEST_OOM=1 make valgrind
On Thu, May 22, 2008 at 06:38:47PM +0100, Daniel P. Berrange wrote:
This switches over a number of the test cases to make use of the new virTestMain() function and thus gain support for allocation checking
So you can now actually see some checks when running
VIR_TEST_OOM=1 make check
Oh, and for added fun you can check if there are any leaks in the cleanup paths by layering in valgrind
Great, +1
VIR_TEST_OOM=1 make valgrind
which is superbly slow but finds a few things :-)
Finnaly putting those fast CPUs to good use ... excellent :-) Daniel -- Red Hat Virtualization group http://redhat.com/virtualization/ Daniel Veillard | virtualization library http://libvirt.org/ veillard@redhat.com | libxml GNOME XML XSLT toolkit http://xmlsoft.org/ http://veillard.com/ | Rpmfind RPM search engine http://rpmfind.net/
participants (3)
-
Daniel P. Berrange -
Daniel Veillard -
Jim Meyering