# HG changeset patch # User Kaitlin Rupert <karupert(a)us.ibm.com&gt; # Date 1213136844 25200 # Node ID 63757929dd9a15671ae59fb54b89121bbc93d50d # Parent 502dddef8c34eeb85b571df0ee97f0ee0676861b [TEST] Enable test infrastructure to support writing to a temporary file. Allow the option to print to a file in addition to stdout. This will allow a test suite to read back the file after the test execution is complete. If a previous log exists at the start of the run, it is removed and a new one is created. Signed-off-by: Kaitlin Rupert <karupert(a)us.ibm.com&gt; diff -r 502dddef8c34 -r 63757929dd9a lib/Reporter.py --- a/lib/Reporter.py Tue Jul 01 14:12:48 2008 -0700 +++ b/lib/Reporter.py Tue Jun 10 15:27:24 2008 -0700 @@ -19,10 +19,13 @@ # License along with this library; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA # +from CimTest.ReturnCodes import PASS, FAIL, SKIP, XFAIL + class Reporter: - def __init__(self, verbosity=1): + def __init__(self, verbosity=1, log_fd=None): self.verbosity = verbosity + self.log_fd = log_fd def __red(self, str): return "\033[01;31m%s\033[00m" % str @@ -36,11 +39,38 @@ def __blue(self, str): return "\033[01;34m%s\033[00m" % str - def __out(self, str): - """We might not always be just printing output to stdout, so this should - be used for all output.""" - # Right now we just mimic print. - print(str) + def __out(self, str, status, bug): + def no_color(string): + return string + + colors = { "FAIL" : self.__red, + "PASS" : self.__green, + "SKIP" : self.__yellow, + "XFAIL" : self.__blue, + } + + fn = colors.get(status, no_color) + + if status == XFAIL: + print "%s: %s\tBug: %s" % (str, fn(status), bug) + else: + print "%s: %s" % (str, fn(status)) + + if self.log_fd is not None: + if status == XFAIL: + self.log_fd.write("%s: %s\tBug: %s" % (str, status, bug)) + else: + self.log_fd.write("%s: %s" % (str, status)) + + def results(self, str, status, bug): + + rc = { FAIL : "FAIL", + PASS : "PASS", + SKIP : "SKIP", + XFAIL : "XFAIL" + } + + self.__out(str, rc[status], bug) def debug(self, level, str): """Produces debug output if appropriate for current verbosity level. @@ -49,20 +79,5 @@ priority output will be printed in the most levels, while low priority output will only be printed when verbosity is high.""" if level <= self.verbosity: - self.__out(str) + print(str) - def pass_test(self, test_name): - str = self.__green("PASS") - self.__out("%s: %s" % (test_name, str)) - - def fail_test(self, test_name): - str = self.__red("FAIL") - self.__out("%s: %s" % (test_name, str)) - - def xfail_test(self, test_name, bug): - str = self.__blue("XFAIL") - self.__out("%s: %s\tBug: %s" % (test_name, str, bug)) - - def skip_test(self, test_name): - str = self.__yellow("SKIP") - self.__out("%s: %s" % (test_name, str)) diff -r 502dddef8c34 -r 63757929dd9a lib/TestSuite.py --- a/lib/TestSuite.py Tue Jul 01 14:12:48 2008 -0700 +++ b/lib/TestSuite.py Tue Jun 10 15:27:24 2008 -0700 @@ -24,48 +24,59 @@ DEFAULT_RPC_URL = "http://morbo.linux.ibm.com/xenotest/testrun/api" +DEFAULT_LOG_FILE = "run_report.txt" + import Reporter import re +import os +from CimTest.ReturnCodes import PASS, FAIL, XFAIL, SKIP class TestSuite: """Test Suite class to make the output of driving test suites a bit more consistant""" - def __init__(self): - self.rep = Reporter.Reporter(verbosity=5) + def __init__(self, log=False, file_name=None): + if log == True: + if file_name is None: + self.log_file = DEFAULT_LOG_FILE + else: + self.log_file = file_name - def ok(self, group, test, output=""): - self.rep.pass_test("%s - %s" % (group, test)) + if os.path.exists(self.log_file): + os.remove(self.log_file) + self.log_fd = open(self.log_file, "w") + else: + self.log_file = None + self.log_fd = None - def skip(self, group, test, output=""): - self.rep.skip_test("%s - %s" % (group, test)) + self.rep = Reporter.Reporter(verbosity=5, log_fd=self.log_fd) + + def print_results(self, group, test, status, output=""): + bug = None + if status == XFAIL: + err = "Test error: returned XFAIL without a valid bug string." + bug = err + if len(output) > 0: + try: + str = re.search('Bug:<[0-9]*>', output).group() + bug = re.search("Bug:<([0-9]+)>", str).group(1) + if len(str) > 0: + if output == str: + #No need to pring bug twice + output = "" + except: + #If we hit a problem, make sure bug = error msg + bug = err + + self.rep.results("%s - %s" % (group, test), status, bug) if output: self.rep.debug(1, output) - def fail(self, group, test, output=""): - self.rep.fail_test("%s - %s" % (group, test)) - if output: - self.rep.debug(1, output) - - def xfail(self, group, test, output=""): - err = "Test error: returned XFAIL without a valid bug string." - bug = err - if len(output) > 0: - try: - str = re.search('Bug:<[0-9]*>', output).group() - bug = re.search("Bug:<([0-9]+)>", str).group(1) - if len(str) > 0: - if output == str: - #No need to pring bug twice - output = "" - except: - #If we hit a problem, make sure bug is equal to the error msg - bug = err - self.rep.xfail_test("%s - %s" % (group, test), bug) - if output: - self.rep.debug(1, output) + def debug(self, str): + self.rep.debug(1, str) def finish(self): - pass + if self.log_fd is not None: + self.log_fd.close() class RPCTestSuite: """Test Suite class to make the output of driving test suites a bit more consistant diff -r 502dddef8c34 -r 63757929dd9a suites/libvirt-cim/main.py --- a/suites/libvirt-cim/main.py Tue Jul 01 14:12:48 2008 -0700 +++ b/suites/libvirt-cim/main.py Tue Jun 10 15:27:24 2008 -0700 @@ -153,14 +153,7 @@ os_status = os.WEXITSTATUS(status) - if os_status == PASS: - testsuite.ok(test['group'], test['test']) - elif os_status == SKIP: - testsuite.skip(test['group'], test['test'], output) - elif os_status == XFAIL: - testsuite.xfail(test['group'], test['test'], output) - else: - testsuite.fail(test['group'], test['test'], output) + testsuite.print_results(test['group'], test['test'], os_status) testsuite.finish()

# HG changeset patch # User Kaitlin Rupert <karupert(a)us.ibm.com&gt; # Date 1213136844 25200 # Node ID 723430f4ceb0770b1a133c6babfc4fd63c17f7a5 # Parent 63757929dd9a15671ae59fb54b89121bbc93d50d [TEST] Add test report generation. Using the --report option, you can generate a report and have it mailed to the address supplied. How to call using the report option: ./runtests libvirt-cim -i localhost -c -d -v LXC -g VirtualSystemManagementService -t 01_definesystem_name.py --report <my_to_addr(a)test.com&gt; The user will need to create a .cimtestrc file in their home directory in the following format: [email] relay = my.mail.relay.com from = Joe User <joe(a)test.com&gt; Will need to add revision number support to cimtest and then add the cimtest revision number to the report. Signed-off-by: Kaitlin Rupert <karupert(a)us.ibm.com&gt; diff -r 63757929dd9a -r 723430f4ceb0 suites/libvirt-cim/main.py --- a/suites/libvirt-cim/main.py Tue Jun 10 15:27:24 2008 -0700 +++ b/suites/libvirt-cim/main.py Tue Jun 10 15:27:24 2008 -0700 @@ -25,6 +25,8 @@ from optparse import OptionParser import os import sys +import smtplib +from time import gmtime, strftime from pywbem import WBEMConnection sys.path.append('../../lib') import TestSuite @@ -35,6 +37,7 @@ from CimTest.Globals import platform_sup sys.path.append('./lib') from XenKvmLib.classes import get_typed_class +import ConfigParser parser = OptionParser() parser.add_option("-i", "--ip", dest="ip", default="localhost", @@ -54,8 +57,11 @@ help="Virt type, select from 'Xen' & 'KVM' & 'XenFV' & 'LXC'(default: Xen). ") parser.add_option("-d", "--debug-output", action="store_true", dest="debug", help="Duplicate the output to stderr") +parser.add_option("--report", dest="report", + help="Send report using mail info: --report=<recipient addr>") TEST_SUITE = 'cimtest' +CIMTEST_RCFILE = '%s/.cimtestrc' % os.environ['HOME'] def set_python_path(): previous_pypath = os.environ.get('PYTHONPATH') @@ -82,6 +88,28 @@ print "Cleaned log files." +def get_rcfile_vals(): + if not os.access(CIMTEST_RCFILE, os.R_OK): + print "\nCould not access the %s file for this user." % CIMTEST_RCFILE + print "Create this file and add the appropriate relay:" + print "\tfrom = me(a)isp.com\n\trelay = my.relay\n" + return None, None + + try: + conf = ConfigParser.ConfigParser() + if not conf.read(CIMTEST_RCFILE): + return None, None + + addr = conf.get("email", "from") + relay = conf.get("email", "relay") + + except Exception, details: + print "\n%s" % details + print "\nPlease verify the format of the %s file\n" % CIMTEST_RCFILE + return None, None + + return addr, relay + def get_version(virt, ip): conn = WBEMConnection('http://%s' % ip, (os.getenv('CIM_USER'), os.getenv('CIM_PASS')), @@ -99,9 +127,139 @@ changeset = 'Unknown' return revision, changeset +def get_cmd_val(cmd, ip): + rc, out = utils.run_remote(ip, cmd) + if rc != 0: + return "Unknown" + return out + +def get_env_data(rev, changeset, ip): + distro = get_cmd_val("cat /etc/issue | awk 'NR<=1'", ip) + kernel_ver = get_cmd_val("uname -r", ip) + + libvirt_ver = "Unknown" + hyp_ver = "Unknown" + cmd = "virsh version" + virsh_ver = get_cmd_val(cmd, ip) + if virsh_ver != "Unknown": + if len(virsh_ver.splitlines()) == 4: + if virsh_ver.splitlines()[0].find("libvir"): + libvirt_ver = virsh_ver.splitlines()[0].split()[4] + + if virsh_ver.splitlines()[3].find("hypervisor"): + hyp_ver = virsh_ver.splitlines()[3].split("hypervisor")[1] + hyp_ver = hyp_ver.split(": ")[1] + + cimom = get_cmd_val("ps -ef | grep cimserver | grep -v grep", ip) + if cimom != "Unknown": + cimom = "Pegasus" + else: + cimom = get_cmd_val("ps -ef | grep sfcb | grep -v grep", ip) + if cimom != "Unknown": + cimom = "sfcb" + + if cimom == "Pegasus": + cimom_ver = get_cmd_val("cimserver -v", ip) + elif cimom == "sfcb": + cimom_ver = get_cmd_val("sfcbd -v", ip) + else: + cimom_ver = "unknown version" + + env = "Distro: %s\nKernel: %s\nlibvirt: %s\nHypervisor: %s\nCIMOM: %s %s\n"\ + % (distro, kernel_ver, libvirt_ver, hyp_ver, cimom, cimom_ver) + + lc_ver = "Libvirt-cim revision: %s\nLibvirt-cim changeset: %s\n" % \ + (rev, changeset) + + return env + lc_ver + +def build_report(run_output, rev, changeset, virt, ip): + rvals = { 'PASS' : 0, + 'FAIL' : 0, + 'XFAIL' : 0, + 'SKIP' : 0, + } + + tstr = { 'PASS' : "", + 'FAIL' : "", + 'XFAIL' : "", + 'SKIP' : "", + } + + date = strftime("%b %d %Y", gmtime()) + + heading = "%s Test Run Summary %s" % (virt, date) + sys_env = get_env_data(rev, changeset, ip) + + divider = "=================================================\n" + + for line in run_output.splitlines(): + for type, val in rvals.iteritems(): + if type in line: + if type == "FAIL" and "py: FAIL" not in line: + continue + rvals[type] += 1 + tstr[type] += "%s\n" % line + + results = "" + test_total = 0 + for type, val in rvals.iteritems(): + results += " %s:\t%d\n" % (type, val) + test_total += val + + results_total = " -----------------\n Total:\t%d\n" % test_total + + test_block = "" + for type, str in tstr.iteritems(): + if type == "PASS" or str == "": + continue + test_block += "%s Test Summary:\n%s\n%s" % (type, str, divider) + + report = divider + heading + "\n" + divider + sys_env + divider + results \ + + results_total + divider + test_block + "Full report:\n" \ + + run_output + + return report, heading + +def gen_report(rev, changeset, virt, ip, log_file): + fd = open(log_file, "r") + run_results = fd.read() + fd.close() + + msg_body, heading = build_report(run_results, rev, changeset, virt, ip) + + fd = open(log_file, "w") + rc = fd.write(msg_body) + if rc is not None: + print "Error %s writing report to: %s." % (rc, log_file) + fd.close() + + return msg_body, heading + +def send_report(to_addr, from_addr, relay, report, heading): + headers = "From: %s\r\nTo: %s\r\nSubject: %s\r\n\r\n" % (from_addr, to_addr, + heading) + + message = headers + report + + try: + server = smtplib.SMTP(relay) + result = server.sendmail(to_addr, from_addr, message) + server.quit() + + if result: + for recip in result.keys(): + print "Could not deliver mail to: %s" % recip + + except Exception, details: + print "Encountered a problem mailing report: %s" % details + def main(): (options, args) = parser.parse_args() - + to_addr = None + from_addr = None + relay = None + div = "--------------------------------------------------------------------" if options.test and not options.group: parser.print_help() @@ -114,7 +272,15 @@ os.environ['CIMOM_PORT'] = str(options.port) # - testsuite = TestSuite.TestSuite() + if options.report: + from_addr, relay = get_rcfile_vals() + + if from_addr == None or relay == None: + return 1 + + to_addr = options.report + + testsuite = TestSuite.TestSuite(log=True) set_python_path() @@ -139,9 +305,10 @@ revision, changeset = get_version(options.virt, options.ip) - print "Testing " + options.virt + " hypervisor" + print "\nTesting " + options.virt + " hypervisor" - for test in test_list: + for test in test_list: + testsuite.debug(div) t_path = os.path.join(TEST_SUITE, test['group']) os.environ['CIM_TC'] = test['test'] cdto = 'cd %s' % t_path @@ -155,7 +322,16 @@ testsuite.print_results(test['group'], test['test'], os_status) + testsuite.debug("%s\n" % div) testsuite.finish() + + msg_body, heading = gen_report(revision, changeset, options.virt, + options.ip, testsuite.log_file) + + if options.report: + print "Sending mail from %s to %s using %s relay.\n" % \ + (from_addr, to_addr, relay) + send_report(to_addr, from_addr, relay, msg_body, heading) if __name__ == '__main__': sys.exit(main())