[libvirt] [test-API PATCH 0/5] add TESTCASE_check optional function support
In some cases, we need to check if the testing environment is satisfied to run a certain testcase. The testcase only will be executed If some specific hardware is present on box. The patches add a optional check function support. For example: a testcase named testa.py with testa_check() defined in this file. The framework will run test_check(params) first, return 0 means pass, 1 means failure. If the check function failed. the testa(params) will not be run. "Skip" is marked in output. # testa.py required_params = () optional_params = ('options') def testa_check(params): logger = params['logger'] logger.info("I am from testa_check") return 1 def testa(params): logger = params['logger'] logger.info("I am from testa") return 1 The Output: Checking Testing Environment... Linux localhost.localdomain 3.2.5-3.fc16.x86_64 #1 SMP Thu Feb 9 01:24:38 UTC 2012 x86_64 x86_64 x86_64 GNU/Linux Virsh command line tool of libvirt: 0.9.10 libvirtd (libvirt) 0.9.10 Default URI: qemu:///system QEMU emulator version 0.15.0 (qemu-kvm-0.15.0), Copyright (c) 2003-2008 Fabrice Bellard Start Testing: Case Count: 1 Log File: log/20120416142755/libvirt_test001 test:testa 14:27:56|INFO |I am from testa_check 14:27:56|INFO |Failed to meet testing requirement Result: Skip Summary: Total:1 [Pass:0 Fail:0 Skip:1] The testcase.conf file: test:testa options value1
Original, the clean flag work with previous testcase to form a separate call request to generator. the patch will append ":clean" to previous testcase to mark that the previous testcase needs to clean after running. It make counting of testcase number easier. And rename the function and variable, add more comments --- mapper.py | 71 +++++++++++++++++++++++++++++------------------------------- 1 files changed, 34 insertions(+), 37 deletions(-) diff --git a/mapper.py b/mapper.py index c2c44da..5cf12e3 100644 --- a/mapper.py +++ b/mapper.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# libvirt-test-API is copyright 2010 Red Hat, Inc. +# libvirt-test-API is copyright 2010, 2012 Red Hat, Inc. # # libvirt-test-API is free software: you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by @@ -17,8 +17,6 @@ # the purpose is to get useful information about a testrun. # -import copy - class Mapper(object): def __init__(self, testcases_list): @@ -27,54 +25,53 @@ class Mapper(object): def module_casename_func_map(self): """ generate a new list of dictionary change key from module:casename to module:casename:func + if clean flag is set, key will be module:casename:func:clean """ - tripped_cases_list = [] - prev_testcasename = '' - prev_testcases_params = '' + new_case_list = [] for testcase in self.testcases_list: - tripped_case = {} - testcase_name = testcase.keys()[0] - if ":" in testcase_name: - casename = testcase_name.split(":")[1] + case = {} + mod_case = testcase.keys()[0] + if ":" in mod_case: + casename = mod_case.split(":")[1] func = casename - if testcase_name == 'sleep': - tripped_cases_list.append(testcase) + if mod_case == 'sleep': + new_case_list.append(testcase) continue - if testcase_name == 'clean': - func = casename + "_clean" - tripped_case[prev_testcasename + ":" + func] = prev_testcases_params - tripped_cases_list.append(tripped_case) - continue + if mod_case == 'clean': + if not new_case_list: + return None - testcases_params = testcase.values()[0] - tripped_case[testcase_name + ":" + func] = testcases_params - tripped_cases_list.append(tripped_case) + previous_case = new_case_list.pop() + key = previous_case.keys()[0] + ':clean' + case[key] = previous_case.values()[0] + new_case_list.append(case) + continue - prev_testcasename = testcase_name - prev_testcases_params = testcases_params + cases_params = testcase.values()[0] + case[mod_case + ":" + func] = cases_params + new_case_list.append(case) - return tripped_cases_list + return new_case_list - def module_casename_cleanfunc_map(self): - """generate a new data format - keys of dictionay are of module:casename:casename_clean + def module_casename_func_noflag(self): + """remove sleep and clean data + generate a new data format """ - tripped_cases_list = [] + new_case_list = [] for testcase in self.testcases_list: - tripped_case = {} - testcase_name = testcase.keys()[0] - if ":" in testcase_name: - casename = testcase_name.split(":")[1] - func = casename + "_clean" + case = {} + mod_case = testcase.keys()[0] - if testcase_name == 'sleep' or testcase_name == 'clean': + if mod_case == 'sleep' or mod_case == 'clean': continue - testcases_params = testcase.values()[0] - tripped_case[testcase_name + ":" + func] = testcases_params - tripped_cases_list.append(tripped_case) + func = mod_case.split(":")[1] + + cases_params = testcase.values()[0] + case[mod_case + ":" + func] = cases_params + new_case_list.append(case) - return tripped_cases_list + return new_case_list -- 1.7.7.5
*calling testcase_check() if testcase defined it. On pass, run testcase(), otherwise, skip the testcase. *add "Skip" counting *because we don't treat clean flag as a seprate call, so we run clean function based on the check of clean flag on testcase name that returned by mapper. --- generator.py | 125 +++++++++++++++++++++++++++++---------------------------- 1 files changed, 64 insertions(+), 61 deletions(-) diff --git a/generator.py b/generator.py index 62b0d66..5059587 100644 --- a/generator.py +++ b/generator.py @@ -39,18 +39,19 @@ for dist in os.listdir('dist'): class FuncGen(object): """ To generate a callable testcase""" def __init__(self, cases_func_ref_dict, + cases_checkfunc_ref_dict, activity, logfile, testrunid, testid, log_xml_parser, lockfile, bugstxt, loglevel): self.cases_func_ref_dict = cases_func_ref_dict + self.cases_checkfunc_ref_dict = cases_checkfunc_ref_dict self.logfile = logfile self.testrunid = testrunid self.testid = testid self.lockfile = lockfile self.bugstxt = bugstxt self.loglevel = loglevel - self.testcase_number = 0 self.fmt = format.Format(logfile) self.log_xml_parser = log_xml_parser @@ -61,23 +62,21 @@ class FuncGen(object): self.env = env_parser.Envparser("env.cfg") mapper_obj = mapper.Mapper(activity) - pkg_casename_func = mapper_obj.module_casename_func_map() + case_list = mapper_obj.module_casename_func_map() - for test_procedure in pkg_casename_func: + for test_procedure in case_list: log_xml_parser.add_testprocedure_xml(testrunid, testid, test_procedure) - self.cases_ref_names = [] - for case in pkg_casename_func: - case_ref_name = case.keys()[0] - if case_ref_name[-6:] != "_clean": - self.testcase_number += 1 - self.cases_ref_names.append(case_ref_name) - - self.cases_params_list = [] - for case in pkg_casename_func: + self.case_name_list = [] + for case in case_list: + mod_case_func = case.keys()[0] + self.case_name_list.append(mod_case_func) + + self.case_params_list = [] + for case in case_list: case_params = case.values()[0] - self.cases_params_list.append(case_params) + self.case_params_list.append(case_params) def __call__(self): retflag = self.generator() @@ -115,7 +114,7 @@ class FuncGen(object): envlog = log.EnvLog(self.logfile, self.loglevel) env_logger = envlog.env_log() - loop_number = len(self.cases_ref_names) + casenumber = len(self.case_name_list) start_time = time.strftime("%Y-%m-%d %H:%M:%S") env_logger.info("Checking Testing Environment... ") @@ -125,92 +124,96 @@ class FuncGen(object): sys.exit(1) else: env_logger.info("\nStart Testing:") - env_logger.info(" Case Count: %s" % self.testcase_number) + env_logger.info(" Case Count: %s" % casenumber) env_logger.info(" Log File: %s\n" % self.logfile) caselog = log.CaseLog(self.logfile, self.loglevel) case_logger = caselog.case_log() - retflag = 0 - for i in range(loop_number): + # retflag: [pass, fail, skip] + retflag = [0, 0, 0] + for i in range(casenumber): - case_ref_name = self.cases_ref_names[i] - pkg_casename = case_ref_name.rsplit(":", 1)[0] - funcname = case_ref_name.rsplit(":", 1)[-1] + clean_flag = False - if "_clean" not in funcname: - cleanoper = 0 - else: - cleanoper = 1 + mod_case_func = self.case_name_list[i] + mod_case = mod_case_func.rsplit(":", 1)[0] + if mod_case_func.endswith(':clean'): + mod_case_func = mod_case_func[:-6] + clean_flag = True + self.fmt.print_start(mod_case, env_logger) - if not cleanoper: - self.fmt.print_start(pkg_casename, env_logger) - else: - self.fmt.print_string(12*" " + "Cleaning...", env_logger) + case_params = self.case_params_list[i] + case_params['logger'] = case_logger - case_params = self.cases_params_list[i] + if self.cases_checkfunc_ref_dict.has_key(mod_case_func): + if self.cases_checkfunc_ref_dict[mod_case_func](case_params): + case_logger.info("Failed to meet testing requirement") + self.fmt.print_end(mod_case, 2, env_logger) + retflag[2] += 1 + continue case_start_time = time.strftime("%Y-%m-%d %H:%M:%S") - ret = -1 - clean_ret = -1 + ret = 0 try: try: - if case_ref_name != 'sleep': - case_params['logger'] = case_logger - - existed_bug_list = self.bug_check(pkg_casename) + existed_bug_list = self.bug_check(mod_case) if len(existed_bug_list) == 0: - if case_ref_name == 'sleep': - sleepsecs = case_params['sleep'] + if mod_case_func == 'sleep': + sleepsecs = case_params.get('sleep', 0) case_logger.info("sleep %s seconds" % sleepsecs) time.sleep(int(sleepsecs)) ret = 0 else: - ret = self.cases_func_ref_dict[case_ref_name](case_params) - if cleanoper: - clean_ret = ret - ret = 0 + ret = self.cases_func_ref_dict[mod_case_func](case_params) + # In the case where testcase return -1 on error + if ret < 0: ret = 1 + + if clean_flag: + clean_func = mod_case_func + '_clean' + self.fmt.print_string(12*" " + "Cleaning...", env_logger) + # the return value of clean function is optional + clean_ret = self.cases_func_ref_dict[clean_func](case_params) + if clean_ret and clean_ret == 1: + self.fmt.print_string(21*" " + "Fail", env_logger) + continue + + self.fmt.print_string(21*" " + "Done", env_logger) + else: case_logger.info("about the testcase , bug existed:") for existed_bug in existed_bug_list: case_logger.info("%s" % existed_bug) - ret = 100 - self.fmt.print_end(pkg_casename, ret, env_logger) + # use 2 to represent skip value + ret = 2 continue except Exception, e: case_logger.error(traceback.format_exc()) continue finally: case_end_time = time.strftime("%Y-%m-%d %H:%M:%S") - if ret == -1: - ret = 1 - elif ret == 100: - retflag += 0 - else: - pass - retflag += ret + if ret == 0: + retflag[0] += 1 + elif ret == 1: + retflag[1] += 1 + elif ret == 2: + retflag[2] += 1 - if not cleanoper: - self.fmt.print_end(pkg_casename, ret, env_logger) - else: - if clean_ret < 1: - self.fmt.print_string(21*" " + "Done", env_logger) - else: - self.fmt.print_string(21*" " + "Fail", env_logger) + self.fmt.print_end(mod_case, ret, env_logger) # close hypervisor connection envck.close_hypervisor_connection() end_time = time.strftime("%Y-%m-%d %H:%M:%S") env_logger.info("\nSummary:") - env_logger.info(" Total:%s [Pass:%s Fail:%s]" % \ - (self.testcase_number, (self.testcase_number - retflag), retflag)) + env_logger.info(" Total:%s [Pass:%s Fail:%s Skip:%s]" % \ + (casenumber, retflag[0], retflag[1], retflag[2])) - result = (retflag and "FAIL") or "PASS" + result = (retflag[1] and "FAIL") or "PASS" fcntl.lockf(self.lockfile.fileno(), fcntl.LOCK_EX) self.log_xml_parser.add_test_summary(self.testrunid, self.testid, @@ -219,7 +222,7 @@ class FuncGen(object): end_time, self.logfile) fcntl.lockf(self.lockfile.fileno(), fcntl.LOCK_UN) - return retflag + return retflag[1] def __case_info_save(self, case, testrunid): """ Save data of each test into a file under the testrunid directory -- 1.7.7.5
*make get_optionalfunc_call_dict work for both testcase_check() and testcase_clean() *imporve the code readability --- proxy.py | 26 ++++++++++++++------------ 1 files changed, 14 insertions(+), 12 deletions(-) diff --git a/proxy.py b/proxy.py index 49a0420..32a8eb2 100644 --- a/proxy.py +++ b/proxy.py @@ -21,10 +21,10 @@ import exception class Proxy(object): - """ The Proxy class is used for getting real function call reference """ + """ The Proxy class is used for getting function reference """ def __init__(self, testcases_names): - """ Argument case_list is test case list """ + """ initialize a list of references to testcases module """ self.testcases_names = testcases_names self.testcase_ref_dict = {} @@ -73,8 +73,8 @@ class Proxy(object): (func, modcase)) return func_dict - def get_clearfunc_call_dict(self): - """ Return a clearing function reference dictionary. """ + def get_optionalfunc_call_dict(self, suffix): + """ get optional function that is present in testcase""" func_dict = {} for testcase_name in self.testcases_names: # Get module, casename @@ -85,14 +85,15 @@ class Proxy(object): module = elements[0] casename = elements[1] - func = casename + '_clean' + func = casename + '_' + suffix - casemod_ref = self.testcase_ref_dict[testcase_name] - var_func_names = dir(casemod_ref) + modcase = module + ':' + casename + key = modcase + ':' + casename - key = module + ':' + casename + ':' + func + casemod_ref = self.testcase_ref_dict[modcase] + var_func_names = dir(casemod_ref) - # the clean function is optional, we get its reference + # function is optional, we get its reference # only if it exists in testcases if func in var_func_names: func_ref = getattr(casemod_ref, func) @@ -114,16 +115,17 @@ class Proxy(object): module = elements[0] casename = elements[1] - casemod_ref = self.testcase_ref_dict[testcase_name] + modcase = module + ':' + casename + casemod_ref = self.testcase_ref_dict[modcase] var_func_names = dir(casemod_ref) if 'required_params' in var_func_names \ and 'optional_params' in var_func_names: - case_params[testcase_name] = \ + case_params[modcase] = \ [casemod_ref.required_params, casemod_ref.optional_params] else: raise exception.TestCaseError\ - ("required_params or optional_params not found in %s" % testcase_name) + ("required_params or optional_params not found in %s" % modcase) return case_params def has_clean_function(self, testcase_name): -- 1.7.7.5
*proxy_obj.get_optionalfunc_call_dict('check') to get references to 'check' function that defined in testcases, and pass them to generator for running later --- libvirt-test-api.py | 6 +++++- 1 files changed, 5 insertions(+), 1 deletions(-) diff --git a/libvirt-test-api.py b/libvirt-test-api.py index 7b38aaa..c53a3b6 100644 --- a/libvirt-test-api.py +++ b/libvirt-test-api.py @@ -125,6 +125,9 @@ class Main(object): unique_testcase_keys = filterobj.unique_testcase_cleansuffix() cases_func_ref_dict = proxy_obj.get_func_call_dict(unique_testcase_keys) + # get check function reference if that is defined in testcase file + cases_checkfunc_ref_dict = proxy_obj.get_optionalfunc_call_dict('check') + # create a null list, then, initilize generator to # get the callable testcase function # and put it into procs list for running. @@ -142,6 +145,7 @@ class Main(object): else: logfile = os.path.join('log/%s' % testrunid, logname) procs.append(generator.FuncGen(cases_func_ref_dict, + cases_checkfunc_ref_dict, activity, logfile, testrunid, @@ -199,7 +203,7 @@ class Main(object): if options_list[0]['options'].has_key("cleanup"): if options_list[0]['options']["cleanup"] == "enable": print "Clean up Testing Environment..." - cases_clearfunc_ref_dict = proxy_obj.get_clearfunc_call_dict() + cases_clearfunc_ref_dict = proxy_obj.get_optionalfunc_call_dict('clean') log.Log.counter = 0 for activity in activities_list: logname = log.Log.get_log_name() -- 1.7.7.5
*env_clear.py *utils/format.py we use 2 for skip flag --- env_clear.py | 26 +++++++++++++------------- utils/format.py | 2 +- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/env_clear.py b/env_clear.py index 2e5bcf5..a5dc4fa 100644 --- a/env_clear.py +++ b/env_clear.py @@ -30,17 +30,17 @@ class EnvClear(object): self.loglevel = loglevel mapper_obj = mapper.Mapper(activity) - clean_pkg_casename_func = mapper_obj.module_casename_cleanfunc_map() + mod_casename_func = mapper_obj.module_casename_func_noflag() - self.cases_ref_names = [] - for case in clean_pkg_casename_func: - case_ref_name = case.keys()[0] - self.cases_ref_names.append(case_ref_name) + self.case_name_list = [] + for case in mod_casename_func: + mod_case_func = case.keys()[0] + self.case_name_list.append(mod_case_func) - self.cases_params_list = [] - for case in clean_pkg_casename_func: + self.case_params_list = [] + for case in mod_casename_func: case_params = case.values()[0] - self.cases_params_list.append(case_params) + self.case_params_list.append(case_params) def __call__(self): retflag = self.env_clear() @@ -52,15 +52,15 @@ class EnvClear(object): envlog = log.EnvLog(self.logfile, self.loglevel) logger = envlog.env_log() - testcase_number = len(self.cases_ref_names) + testcase_number = len(self.case_name_list) for i in range(testcase_number): - case_ref_name = self.cases_ref_names[i] - case_params = self.cases_params_list[i] + mod_case_func = self.case_name_list[i] + case_params = self.case_params_list[i] case_params['logger'] = logger - if self.cases_clearfunc_ref_dict.has_key(case_ref_name): - self.cases_clearfunc_ref_dict[case_ref_name](case_params) + if self.cases_clearfunc_ref_dict.has_key(mod_case_func): + self.cases_clearfunc_ref_dict[mod_case_func](case_params) return 0 diff --git a/utils/format.py b/utils/format.py index 7ee5eca..9f228c4 100644 --- a/utils/format.py +++ b/utils/format.py @@ -57,7 +57,7 @@ class Format(object): if flag == 1: result = 'FAIL' console_result = '\033[1;31mFAIL\033[1;m' - if flag == 100: + if flag == 2: result = 'Skip' console_result = '\033[1;38mSkip\033[1;m' -- 1.7.7.5
participants (1)
-
Guannan Ren