On 13.02.2014 11:14, Daniel P. Berrange wrote:
The previous OOM testing support would re-run the entire "main" method each iteration, failing a different malloc each time. When a test suite has 'n' allocations, the number of repeats requires is (n * (n + 1) ) / 2. This gets very large, very quickly.
This new OOM testing support instead integrates at the virtTestRun level, so each individual test case gets repeated, instead of the entire test suite. This means the values of 'n' are orders of magnitude smaller.
The simple usage is
$ VIR_TEST_OOM=1 ./qemuxml2argvtest ... 29) QEMU XML-2-ARGV clock-utc ... OK Test OOM for nalloc=36 .................................... OK 30) QEMU XML-2-ARGV clock-localtime ... OK Test OOM for nalloc=36 .................................... OK 31) QEMU XML-2-ARGV clock-france ... OK Test OOM for nalloc=38 ...................................... OK ...
the second lines reports how many mallocs have to be failed, and thus how many repeats of the test will be run.
If it crashes, then running under valgrind will often show the problem
$ VIR_TEST_OOM=1 ../run valgrind ./qemuxml2argvtest
When debugging problems it is also helpful to select an individual test case
$ VIR_TEST_RANGE=30 VIR_TEST_OOM=1 ../run valgrind ./qemuxml2argvtest
When things get really tricky, it is possible to request that just specific allocs are failed. eg to fail allocs 5 -> 12, use
$ VIR_TEST_RANGE=30 VIR_TEST_OOM=1:5-12 ../run valgrind ./qemuxml2argvtest
In the worse case, you might want to know the stack trace of the alloc which was failed then VIR_TEST_OOM_TRACE can be set. If it is set to 1 then it will only print if it thinks a mistake happened. This is often not reliable, so setting it to 2 will make it print the stack trace for every alloc that is failed.
$ VIR_TEST_OOM_TRACE=2 VIR_TEST_RANGE=30 VIR_TEST_OOM=1:5-5 ../run valgrind ./qemuxml2argvtest 30) QEMU XML-2-ARGV clock-localtime ... OK Test OOM for nalloc=36 !virAllocN /home/berrange/src/virt/libvirt/src/util/viralloc.c:180 virHashCreateFull /home/berrange/src/virt/libvirt/src/util/virhash.c:144 virDomainDefParseXML /home/berrange/src/virt/libvirt/src/conf/domain_conf.c:11745 virDomainDefParseNode /home/berrange/src/virt/libvirt/src/conf/domain_conf.c:12646 virDomainDefParse /home/berrange/src/virt/libvirt/src/conf/domain_conf.c:12590 testCompareXMLToArgvFiles /home/berrange/src/virt/libvirt/tests/qemuxml2argvtest.c:106 virtTestRun /home/berrange/src/virt/libvirt/tests/testutils.c:250 mymain /home/berrange/src/virt/libvirt/tests/qemuxml2argvtest.c:418 (discriminator 2) virtTestMain /home/berrange/src/virt/libvirt/tests/testutils.c:750 ?? ??:0 _start ??:? FAILED
Signed-off-by: Daniel P. Berrange <berrange@redhat.com> --- cfg.mk | 2 +- docs/internals/oomtesting.html.in | 213 ++++++++++++++++++++++++++++++++++++++ docs/sitemap.html.in | 4 + tests/qemuargv2xmltest.c | 14 +-- tests/qemuxml2argvtest.c | 12 ++- tests/qemuxmlnstest.c | 18 ++-- tests/testutils.c | 195 +++++++++++++++++++++++++++++++++- tests/testutils.h | 2 + 8 files changed, 438 insertions(+), 22 deletions(-) create mode 100644 docs/internals/oomtesting.html.in
diff --git a/tests/testutils.c b/tests/testutils.c index 32fe374..52d07eb 100644 --- a/tests/testutils.c +++ b/tests/testutils.c @@ -47,6 +47,13 @@ #include "virprocess.h" #include "virstring.h"
+#ifdef TEST_OOM +# ifdef TEST_OOM_TRACE +# include <dlfcn.h> +# include <execinfo.h> +# endif +#endif + #ifdef HAVE_PATHS_H # include <paths.h> #endif @@ -64,12 +71,37 @@ static unsigned int testDebug = -1; static unsigned int testVerbose = -1; static unsigned int testExpensive = -1;
+#ifdef TEST_OOM +static unsigned int testOOM = 0; +static unsigned int testOOMStart = -1; +static unsigned int testOOMEnd = -1; +static unsigned int testOOMTrace = 0; +# ifdef TEST_OOM_TRACE +void *testAllocStack[30]; +int ntestAllocStack; +# endif +#endif +static bool testOOMActive = false; + static size_t testCounter = 0; static size_t testStart = 0; static size_t testEnd = 0;
char *progname;
+bool virtTestOOMActive(void) +{ + return testOOMActive; +} + +#ifdef TEST_OOM_TRACE +static void virTestAllocHook(int nalloc ATTRIBUTE_UNUSED, + void *opaque ATTRIBUTE_UNUSED) +{ + ntestAllocStack = backtrace(testAllocStack, ARRAY_CARDINALITY(testAllocStack)); +} +#endif + void virtTestResult(const char *name, int ret, const char *msg, ...) { va_list vargs; @@ -108,6 +140,35 @@ void virtTestResult(const char *name, int ret, const char *msg, ...) va_end(vargs); }
+#ifdef TEST_OOM_TRACE +static void +virTestShowTrace(void) +{ + size_t j; + for (j = 2; j < ntestAllocStack; j++) { + Dl_info info; + char *cmd; + + dladdr(testAllocStack[j], &info); + if (info.dli_fname && + strstr(info.dli_fname, ".so")) { + if (virAsprintf(&cmd, "addr2line -f -e %s %p", + info.dli_fname, + ((void*)((unsigned long long)testAllocStack[j] + - (unsigned long long)info.dli_fbase))) < 0) + continue; + } else { + if (virAsprintf(&cmd, "addr2line -f -e %s %p", + (char*)(info.dli_fname ? info.dli_fname : "<unknown>"), + testAllocStack[j]) < 0) + continue; + }
For some reason I'm seeing an error here: tests $ VIR_TEST_OOM_TRACE=2 VIR_TEST_OOM=1 ./qemuxml2argvtest <snip/> </capabilities> 1) QEMU XML-2-ARGV minimal ... OK Test OOM for nalloc=42 !sh: addr2line: command not found sh: addr2line: command not found sh: addr2line: command not found sh: addr2line: command not found But I DO have addr2line: tests $ which addr2line /usr/bin/addr2line ACK with this squashed in: diff --git a/configure.ac b/configure.ac index 0d505d3..15cd190 100644 --- a/configure.ac +++ b/configure.ac @@ -422,6 +422,8 @@ AC_PATH_PROG([OVSVSCTL], [ovs-vsctl], [ovs-vsctl], [/sbin:/usr/sbin:/usr/local/sbin:$PATH]) AC_PATH_PROG([SCRUB], [scrub], [scrub], [/sbin:/usr/sbin:/usr/local/sbin:$PATH]) +AC_PATH_PROG([ADDR2LINE], [addr2line], [addr2line], + [/sbin:/usr/bin:/usr/sbin:/usr/local/sbin:$PATH]) AC_DEFINE_UNQUOTED([DMIDECODE],["$DMIDECODE"], [Location or name of the dmidecode program]) @@ -452,6 +454,8 @@ if test -n "$RMMOD"; then fi AC_DEFINE_UNQUOTED([SCRUB],["$SCRUB"], [Location or name of the scrub program (for wiping algorithms)]) +AC_DEFINE_UNQUOTED([ADDR2LINE],["$ADDR2LINE"], + [Location of addr2line program]) dnl Specific dir for HTML output ? AC_ARG_WITH([html-dir], [AS_HELP_STRING([--with-html-dir=path], diff --git a/tests/testutils.c b/tests/testutils.c index 52d07eb..3f42b13 100644 --- a/tests/testutils.c +++ b/tests/testutils.c @@ -152,13 +152,13 @@ virTestShowTrace(void) dladdr(testAllocStack[j], &info); if (info.dli_fname && strstr(info.dli_fname, ".so")) { - if (virAsprintf(&cmd, "addr2line -f -e %s %p", + if (virAsprintf(&cmd, ADDR2LINE " -f -e %s %p", info.dli_fname, ((void*)((unsigned long long)testAllocStack[j] - (unsigned long long)info.dli_fbase))) < 0) continue; } else { - if (virAsprintf(&cmd, "addr2line -f -e %s %p", + if (virAsprintf(&cmd, ADDR2LINE " -f -e %s %p", (char*)(info.dli_fname ? info.dli_fname : "<unknown>"), testAllocStack[j]) < 0) continue; Michal