1:30 a.m.
In some cases, we need to check if the testing environment is satisfied
to run a certain testcase. The testcase only will be executed If some
specific hardware is present on box. The patches add a optional check
function support.
For example: a testcase named testa.py with testa_check() defined in
this file. The framework will run test_check(params) first, return 0
means pass, 1 means failure. If the check function failed. the testa(params)
will not be run. "Skip" is marked in output.
# testa.py
required_params = ()
optional_params = ('options')
def testa_check(params):
logger = params['logger']
logger.info("I am from testa_check")
return 1
def testa(params):
logger = params['logger']
logger.info("I am from testa")
return 1
The Output:
Checking Testing Environment...
Linux localhost.localdomain 3.2.5-3.fc16.x86_64 #1 SMP Thu Feb 9 01:24:38 UTC 2012
x86_64 x86_64 x86_64 GNU/Linux
Virsh command line tool of libvirt: 0.9.10
libvirtd (libvirt) 0.9.10
Default URI: qemu:///system
QEMU emulator version 0.15.0 (qemu-kvm-0.15.0), Copyright (c) 2003-2008 Fabrice
Bellard
Start Testing:
Case Count: 1
Log File: log/20120416142755/libvirt_test001
test:testa
14:27:56|INFO |I am from testa_check
14:27:56|INFO |Failed to meet testing requirement
Result: Skip
Summary:
Total:1 [Pass:0 Fail:0 Skip:1]
The testcase.conf file:
test:testa
options
value1
1:30 a.m.
New subject: [libvirt] [test-API PATCH 1/5] mapper: don't treat clean flag as a testcase
Original, the clean flag work with previous testcase
to form a separate call request to generator.
the patch will append ":clean" to previous testcase
to mark that the previous testcase needs to clean after running.
It make counting of testcase number easier.
And rename the function and variable, add more comments
---
mapper.py | 71 +++++++++++++++++++++++++++++-------------------------------
1 files changed, 34 insertions(+), 37 deletions(-)
diff --git a/mapper.py b/mapper.py
index c2c44da..5cf12e3 100644
--- a/mapper.py
+++ b/mapper.py
@@ -1,6 +1,6 @@
#!/usr/bin/env python
#
-# libvirt-test-API is copyright 2010 Red Hat, Inc.
+# libvirt-test-API is copyright 2010, 2012 Red Hat, Inc.
#
# libvirt-test-API is free software: you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by
@@ -17,8 +17,6 @@
# the purpose is to get useful information about a testrun.
#
-import copy
-
class Mapper(object):
def __init__(self, testcases_list):
@@ -27,54 +25,53 @@ class Mapper(object):
def module_casename_func_map(self):
""" generate a new list of dictionary
change key from module:casename to module:casename:func
+ if clean flag is set, key will be module:casename:func:clean
"""
- tripped_cases_list = []
- prev_testcasename = ''
- prev_testcases_params = ''
+ new_case_list = []
for testcase in self.testcases_list:
- tripped_case = {}
- testcase_name = testcase.keys()[0]
- if ":" in testcase_name:
- casename = testcase_name.split(":")[1]
+ case = {}
+ mod_case = testcase.keys()[0]
+ if ":" in mod_case:
+ casename = mod_case.split(":")[1]
func = casename
- if testcase_name == 'sleep':
- tripped_cases_list.append(testcase)
+ if mod_case == 'sleep':
+ new_case_list.append(testcase)
continue
- if testcase_name == 'clean':
- func = casename + "_clean"
- tripped_case[prev_testcasename + ":" + func] =
prev_testcases_params
- tripped_cases_list.append(tripped_case)
- continue
+ if mod_case == 'clean':
+ if not new_case_list:
+ return None
- testcases_params = testcase.values()[0]
- tripped_case[testcase_name + ":" + func] = testcases_params
- tripped_cases_list.append(tripped_case)
+ previous_case = new_case_list.pop()
+ key = previous_case.keys()[0] + ':clean'
+ case[key] = previous_case.values()[0]
+ new_case_list.append(case)
+ continue
- prev_testcasename = testcase_name
- prev_testcases_params = testcases_params
+ cases_params = testcase.values()[0]
+ case[mod_case + ":" + func] = cases_params
+ new_case_list.append(case)
- return tripped_cases_list
+ return new_case_list
- def module_casename_cleanfunc_map(self):
- """generate a new data format
- keys of dictionay are of module:casename:casename_clean
+ def module_casename_func_noflag(self):
+ """remove sleep and clean data
+ generate a new data format
"""
- tripped_cases_list = []
+ new_case_list = []
for testcase in self.testcases_list:
- tripped_case = {}
- testcase_name = testcase.keys()[0]
- if ":" in testcase_name:
- casename = testcase_name.split(":")[1]
- func = casename + "_clean"
+ case = {}
+ mod_case = testcase.keys()[0]
- if testcase_name == 'sleep' or testcase_name == 'clean':
+ if mod_case == 'sleep' or mod_case == 'clean':
continue
- testcases_params = testcase.values()[0]
- tripped_case[testcase_name + ":" + func] = testcases_params
- tripped_cases_list.append(tripped_case)
+ func = mod_case.split(":")[1]
+
+ cases_params = testcase.values()[0]
+ case[mod_case + ":" + func] = cases_params
+ new_case_list.append(case)
- return tripped_cases_list
+ return new_case_list
--
1.7.7.5
1:30 a.m.
New subject: [libvirt] [test-API PATCH 2/5] generator: calling testcase_check if it is present in testcase
*calling testcase_check() if testcase defined it. On pass,
run testcase(), otherwise, skip the testcase.
*add "Skip" counting
*because we don't treat clean flag as a seprate call, so
we run clean function based on the check of clean flag
on testcase name that returned by mapper.
---
generator.py | 125 +++++++++++++++++++++++++++++----------------------------
1 files changed, 64 insertions(+), 61 deletions(-)
diff --git a/generator.py b/generator.py
index 62b0d66..5059587 100644
--- a/generator.py
+++ b/generator.py
@@ -39,18 +39,19 @@ for dist in os.listdir('dist'):
class FuncGen(object):
""" To generate a callable testcase"""
def __init__(self, cases_func_ref_dict,
+ cases_checkfunc_ref_dict,
activity, logfile,
testrunid, testid,
log_xml_parser, lockfile,
bugstxt, loglevel):
self.cases_func_ref_dict = cases_func_ref_dict
+ self.cases_checkfunc_ref_dict = cases_checkfunc_ref_dict
self.logfile = logfile
self.testrunid = testrunid
self.testid = testid
self.lockfile = lockfile
self.bugstxt = bugstxt
self.loglevel = loglevel
- self.testcase_number = 0
self.fmt = format.Format(logfile)
self.log_xml_parser = log_xml_parser
@@ -61,23 +62,21 @@ class FuncGen(object):
self.env = env_parser.Envparser("env.cfg")
mapper_obj = mapper.Mapper(activity)
- pkg_casename_func = mapper_obj.module_casename_func_map()
+ case_list = mapper_obj.module_casename_func_map()
- for test_procedure in pkg_casename_func:
+ for test_procedure in case_list:
log_xml_parser.add_testprocedure_xml(testrunid,
testid,
test_procedure)
- self.cases_ref_names = []
- for case in pkg_casename_func:
- case_ref_name = case.keys()[0]
- if case_ref_name[-6:] != "_clean":
- self.testcase_number += 1
- self.cases_ref_names.append(case_ref_name)
-
- self.cases_params_list = []
- for case in pkg_casename_func:
+ self.case_name_list = []
+ for case in case_list:
+ mod_case_func = case.keys()[0]
+ self.case_name_list.append(mod_case_func)
+
+ self.case_params_list = []
+ for case in case_list:
case_params = case.values()[0]
- self.cases_params_list.append(case_params)
+ self.case_params_list.append(case_params)
def __call__(self):
retflag = self.generator()
@@ -115,7 +114,7 @@ class FuncGen(object):
envlog = log.EnvLog(self.logfile, self.loglevel)
env_logger = envlog.env_log()
- loop_number = len(self.cases_ref_names)
+ casenumber = len(self.case_name_list)
start_time = time.strftime("%Y-%m-%d %H:%M:%S")
env_logger.info("Checking Testing Environment... ")
@@ -125,92 +124,96 @@ class FuncGen(object):
sys.exit(1)
else:
env_logger.info("\nStart Testing:")
- env_logger.info(" Case Count: %s" % self.testcase_number)
+ env_logger.info(" Case Count: %s" % casenumber)
env_logger.info(" Log File: %s\n" % self.logfile)
caselog = log.CaseLog(self.logfile, self.loglevel)
case_logger = caselog.case_log()
- retflag = 0
- for i in range(loop_number):
+ # retflag: [pass, fail, skip]
+ retflag = [0, 0, 0]
+ for i in range(casenumber):
- case_ref_name = self.cases_ref_names[i]
- pkg_casename = case_ref_name.rsplit(":", 1)[0]
- funcname = case_ref_name.rsplit(":", 1)[-1]
+ clean_flag = False
- if "_clean" not in funcname:
- cleanoper = 0
- else:
- cleanoper = 1
+ mod_case_func = self.case_name_list[i]
+ mod_case = mod_case_func.rsplit(":", 1)[0]
+ if mod_case_func.endswith(':clean'):
+ mod_case_func = mod_case_func[:-6]
+ clean_flag = True
+ self.fmt.print_start(mod_case, env_logger)
- if not cleanoper:
- self.fmt.print_start(pkg_casename, env_logger)
- else:
- self.fmt.print_string(12*" " + "Cleaning...",
env_logger)
+ case_params = self.case_params_list[i]
+ case_params['logger'] = case_logger
- case_params = self.cases_params_list[i]
+ if self.cases_checkfunc_ref_dict.has_key(mod_case_func):
+ if self.cases_checkfunc_ref_dict[mod_case_func](case_params):
+ case_logger.info("Failed to meet testing requirement")
+ self.fmt.print_end(mod_case, 2, env_logger)
+ retflag[2] += 1
+ continue
case_start_time = time.strftime("%Y-%m-%d %H:%M:%S")
- ret = -1
- clean_ret = -1
+ ret = 0
try:
try:
- if case_ref_name != 'sleep':
- case_params['logger'] = case_logger
-
- existed_bug_list = self.bug_check(pkg_casename)
+ existed_bug_list = self.bug_check(mod_case)
if len(existed_bug_list) == 0:
- if case_ref_name == 'sleep':
- sleepsecs = case_params['sleep']
+ if mod_case_func == 'sleep':
+ sleepsecs = case_params.get('sleep', 0)
case_logger.info("sleep %s seconds" % sleepsecs)
time.sleep(int(sleepsecs))
ret = 0
else:
- ret = self.cases_func_ref_dict[case_ref_name](case_params)
- if cleanoper:
- clean_ret = ret
- ret = 0
+ ret = self.cases_func_ref_dict[mod_case_func](case_params)
+ # In the case where testcase return -1 on error
+ if ret < 0: ret = 1
+
+ if clean_flag:
+ clean_func = mod_case_func + '_clean'
+ self.fmt.print_string(12*" " +
"Cleaning...", env_logger)
+ # the return value of clean function is optional
+ clean_ret =
self.cases_func_ref_dict[clean_func](case_params)
+ if clean_ret and clean_ret == 1:
+ self.fmt.print_string(21*" " +
"Fail", env_logger)
+ continue
+
+ self.fmt.print_string(21*" " +
"Done", env_logger)
+
else:
case_logger.info("about the testcase , bug existed:")
for existed_bug in existed_bug_list:
case_logger.info("%s" % existed_bug)
- ret = 100
- self.fmt.print_end(pkg_casename, ret, env_logger)
+ # use 2 to represent skip value
+ ret = 2
continue
except Exception, e:
case_logger.error(traceback.format_exc())
continue
finally:
case_end_time = time.strftime("%Y-%m-%d %H:%M:%S")
- if ret == -1:
- ret = 1
- elif ret == 100:
- retflag += 0
- else:
- pass
- retflag += ret
+ if ret == 0:
+ retflag[0] += 1
+ elif ret == 1:
+ retflag[1] += 1
+ elif ret == 2:
+ retflag[2] += 1
- if not cleanoper:
- self.fmt.print_end(pkg_casename, ret, env_logger)
- else:
- if clean_ret < 1:
- self.fmt.print_string(21*" " + "Done",
env_logger)
- else:
- self.fmt.print_string(21*" " + "Fail",
env_logger)
+ self.fmt.print_end(mod_case, ret, env_logger)
# close hypervisor connection
envck.close_hypervisor_connection()
end_time = time.strftime("%Y-%m-%d %H:%M:%S")
env_logger.info("\nSummary:")
- env_logger.info(" Total:%s [Pass:%s Fail:%s]" % \
- (self.testcase_number, (self.testcase_number - retflag), retflag))
+ env_logger.info(" Total:%s [Pass:%s Fail:%s Skip:%s]" % \
+ (casenumber, retflag[0], retflag[1], retflag[2]))
- result = (retflag and "FAIL") or "PASS"
+ result = (retflag[1] and "FAIL") or "PASS"
fcntl.lockf(self.lockfile.fileno(), fcntl.LOCK_EX)
self.log_xml_parser.add_test_summary(self.testrunid,
self.testid,
@@ -219,7 +222,7 @@ class FuncGen(object):
end_time,
self.logfile)
fcntl.lockf(self.lockfile.fileno(), fcntl.LOCK_UN)
- return retflag
+ return retflag[1]
def __case_info_save(self, case, testrunid):
""" Save data of each test into a file under the testrunid
directory
--
1.7.7.5
1:30 a.m.
New subject: [libvirt] [test-API PATCH 3/5] proxy: generalize function get_optionalfunc_call_dict
*make get_optionalfunc_call_dict work for both testcase_check()
and testcase_clean()
*imporve the code readability
---
proxy.py | 26 ++++++++++++++------------
1 files changed, 14 insertions(+), 12 deletions(-)
diff --git a/proxy.py b/proxy.py
index 49a0420..32a8eb2 100644
--- a/proxy.py
+++ b/proxy.py
@@ -21,10 +21,10 @@
import exception
class Proxy(object):
- """ The Proxy class is used for getting real function call reference
"""
+ """ The Proxy class is used for getting function reference
"""
def __init__(self, testcases_names):
- """ Argument case_list is test case list """
+ """ initialize a list of references to testcases module
"""
self.testcases_names = testcases_names
self.testcase_ref_dict = {}
@@ -73,8 +73,8 @@ class Proxy(object):
(func, modcase))
return func_dict
- def get_clearfunc_call_dict(self):
- """ Return a clearing function reference dictionary.
"""
+ def get_optionalfunc_call_dict(self, suffix):
+ """ get optional function that is present in
testcase"""
func_dict = {}
for testcase_name in self.testcases_names:
# Get module, casename
@@ -85,14 +85,15 @@ class Proxy(object):
module = elements[0]
casename = elements[1]
- func = casename + '_clean'
+ func = casename + '_' + suffix
- casemod_ref = self.testcase_ref_dict[testcase_name]
- var_func_names = dir(casemod_ref)
+ modcase = module + ':' + casename
+ key = modcase + ':' + casename
- key = module + ':' + casename + ':' + func
+ casemod_ref = self.testcase_ref_dict[modcase]
+ var_func_names = dir(casemod_ref)
- # the clean function is optional, we get its reference
+ # function is optional, we get its reference
# only if it exists in testcases
if func in var_func_names:
func_ref = getattr(casemod_ref, func)
@@ -114,16 +115,17 @@ class Proxy(object):
module = elements[0]
casename = elements[1]
- casemod_ref = self.testcase_ref_dict[testcase_name]
+ modcase = module + ':' + casename
+ casemod_ref = self.testcase_ref_dict[modcase]
var_func_names = dir(casemod_ref)
if 'required_params' in var_func_names \
and 'optional_params' in var_func_names:
- case_params[testcase_name] = \
+ case_params[modcase] = \
[casemod_ref.required_params, casemod_ref.optional_params]
else:
raise exception.TestCaseError\
- ("required_params or optional_params not found in %s" %
testcase_name)
+ ("required_params or optional_params not found in %s" %
modcase)
return case_params
def has_clean_function(self, testcase_name):
--
1.7.7.5
1:30 a.m.
New subject: [libvirt] [test-API PATCH 4/5] libvirt-test-api: get reference to check function in testcases
*proxy_obj.get_optionalfunc_call_dict('check') to get references
to 'check' function that defined in testcases, and pass them to
generator for running later
---
libvirt-test-api.py | 6 +++++-
1 files changed, 5 insertions(+), 1 deletions(-)
diff --git a/libvirt-test-api.py b/libvirt-test-api.py
index 7b38aaa..c53a3b6 100644
--- a/libvirt-test-api.py
+++ b/libvirt-test-api.py
@@ -125,6 +125,9 @@ class Main(object):
unique_testcase_keys = filterobj.unique_testcase_cleansuffix()
cases_func_ref_dict = proxy_obj.get_func_call_dict(unique_testcase_keys)
+ # get check function reference if that is defined in testcase file
+ cases_checkfunc_ref_dict = proxy_obj.get_optionalfunc_call_dict('check')
+
# create a null list, then, initilize generator to
# get the callable testcase function
# and put it into procs list for running.
@@ -142,6 +145,7 @@ class Main(object):
else:
logfile = os.path.join('log/%s' % testrunid, logname)
procs.append(generator.FuncGen(cases_func_ref_dict,
+ cases_checkfunc_ref_dict,
activity,
logfile,
testrunid,
@@ -199,7 +203,7 @@ class Main(object):
if options_list[0]['options'].has_key("cleanup"):
if options_list[0]['options']["cleanup"] ==
"enable":
print "Clean up Testing Environment..."
- cases_clearfunc_ref_dict = proxy_obj.get_clearfunc_call_dict()
+ cases_clearfunc_ref_dict =
proxy_obj.get_optionalfunc_call_dict('clean')
log.Log.counter = 0
for activity in activities_list:
logname = log.Log.get_log_name()
--
1.7.7.5
1:30 a.m.
New subject: [libvirt] [test-API PATCH 5/5] function and variable name improvement
*env_clear.py
*utils/format.py we use 2 for skip flag
---
env_clear.py | 26 +++++++++++++-------------
utils/format.py | 2 +-
2 files changed, 14 insertions(+), 14 deletions(-)
diff --git a/env_clear.py b/env_clear.py
index 2e5bcf5..a5dc4fa 100644
--- a/env_clear.py
+++ b/env_clear.py
@@ -30,17 +30,17 @@ class EnvClear(object):
self.loglevel = loglevel
mapper_obj = mapper.Mapper(activity)
- clean_pkg_casename_func = mapper_obj.module_casename_cleanfunc_map()
+ mod_casename_func = mapper_obj.module_casename_func_noflag()
- self.cases_ref_names = []
- for case in clean_pkg_casename_func:
- case_ref_name = case.keys()[0]
- self.cases_ref_names.append(case_ref_name)
+ self.case_name_list = []
+ for case in mod_casename_func:
+ mod_case_func = case.keys()[0]
+ self.case_name_list.append(mod_case_func)
- self.cases_params_list = []
- for case in clean_pkg_casename_func:
+ self.case_params_list = []
+ for case in mod_casename_func:
case_params = case.values()[0]
- self.cases_params_list.append(case_params)
+ self.case_params_list.append(case_params)
def __call__(self):
retflag = self.env_clear()
@@ -52,15 +52,15 @@ class EnvClear(object):
envlog = log.EnvLog(self.logfile, self.loglevel)
logger = envlog.env_log()
- testcase_number = len(self.cases_ref_names)
+ testcase_number = len(self.case_name_list)
for i in range(testcase_number):
- case_ref_name = self.cases_ref_names[i]
- case_params = self.cases_params_list[i]
+ mod_case_func = self.case_name_list[i]
+ case_params = self.case_params_list[i]
case_params['logger'] = logger
- if self.cases_clearfunc_ref_dict.has_key(case_ref_name):
- self.cases_clearfunc_ref_dict[case_ref_name](case_params)
+ if self.cases_clearfunc_ref_dict.has_key(mod_case_func):
+ self.cases_clearfunc_ref_dict[mod_case_func](case_params)
return 0
diff --git a/utils/format.py b/utils/format.py
index 7ee5eca..9f228c4 100644
--- a/utils/format.py
+++ b/utils/format.py
@@ -57,7 +57,7 @@ class Format(object):
if flag == 1:
result = 'FAIL'
console_result = '\033[1;31mFAIL\033[1;m'
- if flag == 100:
+ if flag == 2:
result = 'Skip'
console_result = '\033[1;38mSkip\033[1;m'
--
1.7.7.5
4603
days inactive
4603
days old
5 comments
1 participants
participants (1)
-
Guannan Ren