[PATCH 0 of 3] #4 Add reporting capabilities
This patch set will allow cimtest to generate reports. With a little extra help from a cron script, this can be used to do automated test runs. Will need to follow up with a patch that includes the error log messages in the FAIL block.
# HG changeset patch # User Kaitlin Rupert <karupert@us.ibm.com> # Date 1218470726 25200 # Node ID dfd2ac2440a2543fffe6e8c6ae27445e14932ca0 # Parent 680b9475757c1576af1addae83645ab1d1bb6971 [TEST] #2 Enable test infrastructure to support writing to a temporary file. Allow the option to print to a file in addition to stdout. This will allow a test suite to read back the file after the test execution is complete. If a previous log exists at the start of the run, it is removed and a new one is created. Updates: -Add a new line at the end of each line when writing out to the log Signed-off-by: Kaitlin Rupert <karupert@us.ibm.com> diff -r 680b9475757c -r dfd2ac2440a2 lib/Reporter.py --- a/lib/Reporter.py Thu Jul 31 15:17:28 2008 -0700 +++ b/lib/Reporter.py Mon Aug 11 09:05:26 2008 -0700 @@ -19,10 +19,13 @@ # License along with this library; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA # +from CimTest.ReturnCodes import PASS, FAIL, SKIP, XFAIL + class Reporter: - def __init__(self, verbosity=1): + def __init__(self, verbosity=1, log_fd=None): self.verbosity = verbosity + self.log_fd = log_fd def __red(self, str): return "\033[01;31m%s\033[00m" % str @@ -36,11 +39,38 @@ def __blue(self, str): return "\033[01;34m%s\033[00m" % str - def __out(self, str): - """We might not always be just printing output to stdout, so this should - be used for all output.""" - # Right now we just mimic print. - print(str) + def __out(self, str, status, bug): + def no_color(string): + return string + + colors = { "FAIL" : self.__red, + "PASS" : self.__green, + "SKIP" : self.__yellow, + "XFAIL" : self.__blue, + } + + fn = colors.get(status, no_color) + + if status == XFAIL: + print "%s: %s\tBug: %s" % (str, fn(status), bug) + else: + print "%s: %s" % (str, fn(status)) + + if self.log_fd is not None: + if status == XFAIL: + self.log_fd.write("%s: %s\tBug: %s\n" % (str, status, bug)) + else: + self.log_fd.write("%s: %s\n" % (str, status)) + + def results(self, str, status, bug): + + rc = { FAIL : "FAIL", + PASS : "PASS", + SKIP : "SKIP", + XFAIL : "XFAIL" + } + + self.__out(str, rc[status], bug) def debug(self, level, str): """Produces debug output if appropriate for current verbosity level. @@ -49,20 +79,6 @@ priority output will be printed in the most levels, while low priority output will only be printed when verbosity is high.""" if level <= self.verbosity: - self.__out(str) + print str + self.log_fd.write("%s\n" % str) - def pass_test(self, test_name): - str = self.__green("PASS") - self.__out("%s: %s" % (test_name, str)) - - def fail_test(self, test_name): - str = self.__red("FAIL") - self.__out("%s: %s" % (test_name, str)) - - def xfail_test(self, test_name, bug): - str = self.__blue("XFAIL") - self.__out("%s: %s\tBug: %s" % (test_name, str, bug)) - - def skip_test(self, test_name): - str = self.__yellow("SKIP") - self.__out("%s: %s" % (test_name, str)) diff -r 680b9475757c -r dfd2ac2440a2 lib/TestSuite.py --- a/lib/TestSuite.py Thu Jul 31 15:17:28 2008 -0700 +++ b/lib/TestSuite.py Mon Aug 11 09:05:26 2008 -0700 @@ -24,48 +24,59 @@ DEFAULT_RPC_URL = "http://morbo.linux.ibm.com/xenotest/testrun/api" +DEFAULT_LOG_FILE = "run_report.txt" + import Reporter import re +import os +from CimTest.ReturnCodes import PASS, FAIL, XFAIL, SKIP class TestSuite: """Test Suite class to make the output of driving test suites a bit more consistant""" - def __init__(self): - self.rep = Reporter.Reporter(verbosity=5) + def __init__(self, log=False, file_name=None): + if log == True: + if file_name is None: + self.log_file = DEFAULT_LOG_FILE + else: + self.log_file = file_name - def ok(self, group, test, output=""): - self.rep.pass_test("%s - %s" % (group, test)) + if os.path.exists(self.log_file): + os.remove(self.log_file) + self.log_fd = open(self.log_file, "w") + else: + self.log_file = None + self.log_fd = None - def skip(self, group, test, output=""): - self.rep.skip_test("%s - %s" % (group, test)) + self.rep = Reporter.Reporter(verbosity=5, log_fd=self.log_fd) + + def print_results(self, group, test, status, output=""): + bug = None + if status == XFAIL: + err = "Test error: returned XFAIL without a valid bug string." + bug = err + if len(output) > 0: + try: + str = re.search('Bug:<[0-9]*>', output).group() + bug = re.search("Bug:<([0-9]+)>", str).group(1) + if len(str) > 0: + if output == str: + #No need to pring bug twice + output = "" + except: + #If we hit a problem, make sure bug = error msg + bug = err + + self.rep.results("%s - %s" % (group, test), status, bug) if output: self.rep.debug(1, output) - def fail(self, group, test, output=""): - self.rep.fail_test("%s - %s" % (group, test)) - if output: - self.rep.debug(1, output) - - def xfail(self, group, test, output=""): - err = "Test error: returned XFAIL without a valid bug string." - bug = err - if len(output) > 0: - try: - str = re.search('Bug:<[0-9]*>', output).group() - bug = re.search("Bug:<([0-9]+)>", str).group(1) - if len(str) > 0: - if output == str: - #No need to pring bug twice - output = "" - except: - #If we hit a problem, make sure bug is equal to the error msg - bug = err - self.rep.xfail_test("%s - %s" % (group, test), bug) - if output: - self.rep.debug(1, output) + def debug(self, str): + self.rep.debug(1, str) def finish(self): - pass + if self.log_fd is not None: + self.log_fd.close() class RPCTestSuite: """Test Suite class to make the output of driving test suites a bit more consistant diff -r 680b9475757c -r dfd2ac2440a2 suites/libvirt-cim/main.py --- a/suites/libvirt-cim/main.py Thu Jul 31 15:17:28 2008 -0700 +++ b/suites/libvirt-cim/main.py Mon Aug 11 09:05:26 2008 -0700 @@ -153,14 +153,7 @@ os_status = os.WEXITSTATUS(status) - if os_status == PASS: - testsuite.ok(test['group'], test['test']) - elif os_status == SKIP: - testsuite.skip(test['group'], test['test'], output) - elif os_status == XFAIL: - testsuite.xfail(test['group'], test['test'], output) - else: - testsuite.fail(test['group'], test['test'], output) + testsuite.print_results(test['group'], test['test'], os_status, output) testsuite.finish()
# HG changeset patch # User Kaitlin Rupert <karupert@us.ibm.com> # Date 1218470737 25200 # Node ID b1b0cc8fbbd13706fd3a30c2343251688acf8dcf # Parent dfd2ac2440a2543fffe6e8c6ae27445e14932ca0 [TEST] #3 Add reporting library. Include CIMOM type in email heading. Updates from 2 to 3: -Changed tabs in the build_report_body() call with space padded fields. Updates from 1 to 2: -Read from the file using xreadlines() -Reorganize some of the report building pieces -Fixed issue where mail was being sent to the from address Signed-off-by: Kaitlin Rupert <karupert@us.ibm.com> diff -r dfd2ac2440a2 -r b1b0cc8fbbd1 suites/libvirt-cim/lib/XenKvmLib/reporting.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/suites/libvirt-cim/lib/XenKvmLib/reporting.py Mon Aug 11 09:05:37 2008 -0700 @@ -0,0 +1,176 @@ +# +# Copyright 2008 IBM Corp. +# +# Authors: +# Kaitlin Rupert <karupert@us.ibm.com> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +import os +import sys +import commands +import smtplib +from time import gmtime, strftime +from VirtLib import utils + +def get_cmd_val(cmd, ip): + rc, out = utils.run_remote(ip, cmd) + if rc != 0: + return "Unknown" + return out + +def get_libvirt_ver(ip): + libvirt_ver = "Unknown" + hyp_ver = "Unknown" + cmd = "virsh version" + virsh_ver = get_cmd_val(cmd, ip) + if virsh_ver != "Unknown": + if len(virsh_ver.splitlines()) == 4: + if virsh_ver.splitlines()[0].find("libvir"): + libvirt_ver = virsh_ver.splitlines()[0].split()[4] + + if virsh_ver.splitlines()[3].find("hypervisor"): + hyp_ver = virsh_ver.splitlines()[3].split("hypervisor")[1] + hyp_ver = hyp_ver.split(": ")[1] + + return libvirt_ver, hyp_ver + + +def get_cimom_ver(ip): + cimom = get_cmd_val("ps -ef | grep cimserver | grep -v grep", ip) + if cimom != "Unknown": + cimom = "Pegasus" + else: + cimom = get_cmd_val("ps -ef | grep sfcb | grep -v grep", ip) + if cimom != "Unknown": + cimom = "sfcb" + + if cimom == "Pegasus": + cimom_ver = get_cmd_val("cimserver -v", ip) + elif cimom == "sfcb": + cimom_ver = get_cmd_val("sfcbd -v", ip) + else: + cimom_ver = "unknown version" + + return cimom, cimom_ver + + +def get_env_data(rev, changeset, ip): + distro = get_cmd_val("cat /etc/issue | awk 'NR<=1'", ip) + kernel_ver = get_cmd_val("uname -r", ip) + + libvirt_ver, hyp_ver = get_libvirt_ver(ip) + + cimom, cimom_ver = get_cimom_ver(ip) + + env = "Distro: %s\nKernel: %s\nlibvirt: %s\nHypervisor: %s\nCIMOM: %s %s\n"\ + % (distro, kernel_ver, libvirt_ver, hyp_ver, cimom, cimom_ver) + + lc_ver = "Libvirt-cim revision: %s\nLibvirt-cim changeset: %s\n" % \ + (rev, changeset) + + return env + lc_ver + +def parse_run_output(log_file): + rvals = { 'PASS' : 0, + 'FAIL' : 0, + 'XFAIL' : 0, + 'SKIP' : 0, + } + + tstr = { 'PASS' : "", + 'FAIL' : "", + 'XFAIL' : "", + 'SKIP' : "", + } + + fd = open(log_file, "r") + + run_output = "" + for line in fd.xreadlines(): + for type, val in rvals.iteritems(): + if type in line: + if type == "FAIL" and "py: FAIL" not in line: + continue + rvals[type] += 1 + tstr[type] += "%s" % line + run_output += line + + fd.close() + + return rvals, tstr, run_output + +def build_report_body(rvals, tstr, div): + results = "" + test_total = 0 + for type, val in rvals.iteritems(): + results += " %-10s: %d\n" % (type, val) + test_total += val + + results_total = " -----------------\n %-10s: %d\n" % ("Total", test_total) + + test_block = "" + for type, str in tstr.iteritems(): + if type == "PASS" or str == "": + continue + test_block += "%s Test Summary:\n%s\n%s" % (type, str, div) + + return results, results_total, test_block + +def gen_report(rev, changeset, virt, ip, log_file): + date = strftime("%b %d %Y", gmtime()) + + cimom, cimom_ver = get_cimom_ver(ip) + + heading = "%s on %s Test Run Summary for %s" % (virt, cimom, date) + sys_env = get_env_data(rev, changeset, ip) + + divider = "=================================================\n" + + rvals, tstr, run_output = parse_run_output(log_file) + + res, res_total, test_block = build_report_body(rvals, tstr, divider) + + report = divider + heading + "\n" + divider + sys_env + divider + res \ + + res_total + divider + test_block + "Full report:\n" \ + + run_output + + fd = open(log_file, "w") + rc = fd.write(report) + if rc is not None: + print "Error %s writing report to: %s." % (rc, log_file) + fd.close() + + return report, heading + + +def send_report(to_addr, from_addr, relay, report, heading): + headers = "From: %s\r\nTo: %s\r\nSubject: %s\r\n\r\n" % (from_addr, to_addr, + heading) + + message = headers + report + + try: + server = smtplib.SMTP(relay) + result = server.sendmail(from_addr, to_addr, message) + server.quit() + + if result: + for recip in result.keys(): + print "Could not deliver mail to: %s" % recip + + except Exception, details: + print "Encountered a problem mailing report: %s" % details +
# HG changeset patch # User Kaitlin Rupert <karupert@us.ibm.com> # Date 1218470745 25200 # Node ID 0aad4b07fac5359f56174085deaebdfd3a66efd8 # Parent b1b0cc8fbbd13706fd3a30c2343251688acf8dcf [TEST] #2 Add test report generation. Using the --report option, you can generate a report and have it mailed to the address supplied. How to call using the report option: ./runtests libvirt-cim -i localhost -c -d -v LXC -g VirtualSystemManagementService -t 01_definesystem_name.py --report <my_to_addr@test.com> The user will need to create a .cimtestrc file in their home directory in the following format: [email] relay = my.mail.relay.com from = Joe User <joe@test.com> Will need to add revision number support to cimtest and then add the cimtest revision number to the report. Updates: -Refactored reporting related pieces into their own library file Signed-off-by: Kaitlin Rupert <karupert@us.ibm.com> diff -r b1b0cc8fbbd1 -r 0aad4b07fac5 suites/libvirt-cim/main.py --- a/suites/libvirt-cim/main.py Mon Aug 11 09:05:37 2008 -0700 +++ b/suites/libvirt-cim/main.py Mon Aug 11 09:05:45 2008 -0700 @@ -29,12 +29,12 @@ sys.path.append('../../lib') import TestSuite import commands -from VirtLib import utils from VirtLib import groups -from CimTest.ReturnCodes import PASS, SKIP, XFAIL from CimTest.Globals import platform_sup sys.path.append('./lib') from XenKvmLib.classes import get_typed_class +import ConfigParser +from XenKvmLib.reporting import gen_report, send_report parser = OptionParser() parser.add_option("-i", "--ip", dest="ip", default="localhost", @@ -54,8 +54,11 @@ help="Virt type, select from 'Xen' & 'KVM' & 'XenFV' & 'LXC'(default: Xen). ") parser.add_option("-d", "--debug-output", action="store_true", dest="debug", help="Duplicate the output to stderr") +parser.add_option("--report", dest="report", + help="Send report using mail info: --report=<recipient addr>") TEST_SUITE = 'cimtest' +CIMTEST_RCFILE = '%s/.cimtestrc' % os.environ['HOME'] def set_python_path(): previous_pypath = os.environ.get('PYTHONPATH') @@ -82,6 +85,28 @@ print "Cleaned log files." +def get_rcfile_vals(): + if not os.access(CIMTEST_RCFILE, os.R_OK): + print "\nCould not access the %s file for this user." % CIMTEST_RCFILE + print "Create this file and add the appropriate relay:" + print "\tfrom = me@isp.com\n\trelay = my.relay\n" + return None, None + + try: + conf = ConfigParser.ConfigParser() + if not conf.read(CIMTEST_RCFILE): + return None, None + + addr = conf.get("email", "from") + relay = conf.get("email", "relay") + + except Exception, details: + print "\n%s" % details + print "\nPlease verify the format of the %s file\n" % CIMTEST_RCFILE + return None, None + + return addr, relay + def get_version(virt, ip): conn = WBEMConnection('http://%s' % ip, (os.getenv('CIM_USER'), os.getenv('CIM_PASS')), @@ -101,7 +126,10 @@ def main(): (options, args) = parser.parse_args() - + to_addr = None + from_addr = None + relay = None + div = "--------------------------------------------------------------------" if options.test and not options.group: parser.print_help() @@ -114,7 +142,15 @@ os.environ['CIMOM_PORT'] = str(options.port) # - testsuite = TestSuite.TestSuite() + if options.report: + from_addr, relay = get_rcfile_vals() + + if from_addr == None or relay == None: + return 1 + + to_addr = options.report + + testsuite = TestSuite.TestSuite(log=True) set_python_path() @@ -139,9 +175,10 @@ revision, changeset = get_version(options.virt, options.ip) - print "Testing " + options.virt + " hypervisor" + print "\nTesting " + options.virt + " hypervisor" - for test in test_list: + for test in test_list: + testsuite.debug(div) t_path = os.path.join(TEST_SUITE, test['group']) os.environ['CIM_TC'] = test['test'] cdto = 'cd %s' % t_path @@ -155,7 +192,16 @@ testsuite.print_results(test['group'], test['test'], os_status, output) + testsuite.debug("%s\n" % div) testsuite.finish() + + msg_body, heading = gen_report(revision, changeset, options.virt, + options.ip, testsuite.log_file) + + if options.report: + print "Sending mail from %s to %s using %s relay.\n" % \ + (from_addr, to_addr, relay) + send_report(to_addr, from_addr, relay, msg_body, heading) if __name__ == '__main__': sys.exit(main())
Kaitlin - Excellent! All the alignment looks fine for me. +1 from me. libvirt-cim-bounces@redhat.com wrote on 2008-08-14 05:25:18:
This patch set will allow cimtest to generate reports. With a little extra help from a cron script, this can be used to do automated test runs.
Will need to follow up with a patch that includes the error log messages in the FAIL block.
_______________________________________________ Libvirt-cim mailing list Libvirt-cim@redhat.com https://www.redhat.com/mailman/listinfo/libvirt-cim
participants (2)
-
Guo Lian Yun -
Kaitlin Rupert