Re: [libvirt] [test-API][PATCH v2] Add volume upload and download cases

The cases only cover dir volume testing. v1: * test download storage volumes using storage download API. * test upload storage volumes using storage upload API. For upload case, only raw volume format is supported, other format will fail. The offset and length value should be chosen from 0 and 1048576, because upload size is set as 1M. * both case use blocking stream. * sample conf is added. v2: * move digest function to utils * rename cases with prefix 'dir_' to emphasise that they are only for dir vol testing Signed-off-by: Wayne Sun <gsun(a)redhat.com&gt; --- cases/storage_dir_vol_upload_download.conf | 127 ++++++++++++++++++++++ repos/storage/dir_vol_download.py | 131 +++++++++++++++++++++++ repos/storage/dir_vol_upload.py | 158 ++++++++++++++++++++++++++++ utils/utils.py | 24 ++++ 4 files changed, 440 insertions(+), 0 deletions(-) create mode 100644 cases/storage_dir_vol_upload_download.conf create mode 100644 repos/storage/dir_vol_download.py create mode 100644 repos/storage/dir_vol_upload.py diff --git a/cases/storage_dir_vol_upload_download.conf b/cases/storage_dir_vol_upload_download.conf new file mode 100644 index 0000000..fd22720 --- /dev/null +++ b/cases/storage_dir_vol_upload_download.conf @@ -0,0 +1,127 @@ +storage:create_dir_pool + poolname + $defaultpoolname + +storage:dir_vol_upload + poolname + $defaultpoolname + volname + $defaultvolumename + capacity + 10M + volformat + raw + offset + 0 + length + 0 +clean + +storage:dir_vol_upload + poolname + $defaultpoolname + volname + $defaultvolumename + capacity + 10M + volformat + raw + offset + 1048576 + length + 0 +clean + +storage:dir_vol_upload + poolname + $defaultpoolname + volname + $defaultvolumename + capacity + 10M + volformat + raw + offset + 0 + length + 1048576 +clean + +storage:dir_vol_upload + poolname + $defaultpoolname + volname + $defaultvolumename + capacity + 10M + volformat + raw + offset + 1048576 + length + 1048576 +clean + +storage:dir_vol_download + poolname + $defaultpoolname + volname + $defaultvolumename + capacity + 50M + volformat + raw + offset + 0 + length + 0 +clean + +storage:dir_vol_download + poolname + $defaultpoolname + volname + $defaultvolumename + capacity + 50M + volformat + qcow2 + offset + 1048576 + length + 0 +clean + +storage:dir_vol_download + poolname + $defaultpoolname + volname + $defaultvolumename + capacity + 50M + volformat + qed + offset + 0 + length + 1048576 +clean + +storage:dir_vol_download + poolname + $defaultpoolname + volname + $defaultvolumename + capacity + 50M + volformat + raw + offset + 1048576 + length + 1048576 +clean + +storage:destroy_pool + poolname + $defaultpoolname diff --git a/repos/storage/dir_vol_download.py b/repos/storage/dir_vol_download.py new file mode 100644 index 0000000..ddf293b --- /dev/null +++ b/repos/storage/dir_vol_download.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python +# dir storage volume download testing + +import os +import string +from xml.dom import minidom + +import libvirt +from libvirt import libvirtError + +from src import sharedmod +from utils import utils + +required_params = ('poolname', 'volname', 'volformat', 'capacity', 'offset', + 'length',) +optional_params = {'xml' : 'xmls/dir_volume.xml', + } + +def get_pool_path(poolobj): + """ get pool xml description + """ + poolxml = poolobj.XMLDesc(0) + + logger.debug("the xml description of pool is %s" % poolxml) + + doc = minidom.parseString(poolxml) + path_element = doc.getElementsByTagName('path')[0] + textnode = path_element.childNodes[0] + path_value = textnode.data + + return path_value + +def write_file(path, capacity): + """write test data to file + """ + logger.info("write %s data into file %s" % (capacity, path)) + out = utils.get_capacity_suffix_size(capacity) + f = open(path, 'w') + datastr = ''.join(string.lowercase + string.uppercase + + string.digits + '.' + '\n') + repeat = out['capacity_byte'] / 64 + data = ''.join(repeat * datastr) + f.write(data) + f.close() + +def handler(stream, data, file_): + return file_.write(data) + +def dir_vol_download(params): + """test volume download and check""" + + global logger + logger = params['logger'] + poolname = params['poolname'] + volname = params['volname'] + volformat = params['volformat'] + offset = int(params['offset']) + length = int(params['length']) + capacity = params['capacity'] + xmlstr = params['xml'] + + logger.info("the poolname is %s, volname is %s, volformat is %s" % + (poolname, volname, volformat)) + logger.info("download offset is: %s" % offset) + logger.info("the data length to download is: %s" % length) + + conn = sharedmod.libvirtobj['conn'] + try: + poolobj = conn.storagePoolLookupByName(poolname) + path_value = get_pool_path(poolobj) + volume_path = path_value + "/" + volname + + xmlstr = xmlstr.replace('VOLPATH', volume_path) + xmlstr = xmlstr.replace('SUFFIX', capacity[-1]) + xmlstr = xmlstr.replace('CAP', capacity[:-1]) + logger.debug("volume xml:\n%s" % xmlstr) + + logger.info("create %s %s volume" % (volname, volformat)) + vol = poolobj.createXML(xmlstr, 0) + + write_file(volume_path, capacity) + origdigest = utils.digest(volume_path, offset, length) + logger.debug("the md5 hex digest of data read from %s is: %s" % + (volume_path, origdigest)) + + st = conn.newStream(0) + + test_path = path_value + "/" + "vol_test" + + f = open(test_path, 'w') + logger.info("start download") + vol.download(st, offset, length, 0) + logger.info("downloaded all data") + st.recvAll(handler, f) + logger.info("finished stream") + st.finish() + f.close() + + newdigest = utils.digest(test_path, 0, 0) + logger.debug("the md5 hex digest of data read from %s is: %s" % + (test_path, newdigest)) + + if origdigest == newdigest: + logger.info("file digests match, download succeed") + else: + logger.error("file digests not match, download failed") + return 1 + + except libvirtError, e: + logger.error("libvirt call failed: " + str(e)) + return 1 + + return 0 + +def dir_vol_download_clean(params): + """clean testing environment""" + poolname = params['poolname'] + volname = params['volname'] + + conn = sharedmod.libvirtobj['conn'] + poolobj = conn.storagePoolLookupByName(poolname) + path_value = get_pool_path(poolobj) + test_path = path_value + "/" + "vol_test" + + vol = poolobj.storageVolLookupByName(volname) + vol.delete(0) + + if os.path.exists(test_path): + os.unlink(test_path) + + return 0 diff --git a/repos/storage/dir_vol_upload.py b/repos/storage/dir_vol_upload.py new file mode 100644 index 0000000..78188e9 --- /dev/null +++ b/repos/storage/dir_vol_upload.py @@ -0,0 +1,158 @@ +#!/usr/bin/env python +# dir storage volume upload testing, only raw format volume is +# supported, other format might fail. offset and length can +# only be chosen in 0 and 1048576. + +import os +import string +from xml.dom import minidom + +import libvirt +from libvirt import libvirtError + +from src import sharedmod +from utils import utils + +required_params = ('poolname', 'volname', 'volformat', 'capacity', + 'offset', 'length',) +optional_params = {'xml' : 'xmls/dir_volume.xml', + } + +def get_pool_path(poolobj): + """ get pool xml description + """ + poolxml = poolobj.XMLDesc(0) + + logger.debug("the xml description of pool is %s" % poolxml) + + doc = minidom.parseString(poolxml) + path_element = doc.getElementsByTagName('path')[0] + textnode = path_element.childNodes[0] + path_value = textnode.data + + return path_value + +def write_file(path): + """write 1M test data to file + """ + logger.info("write data into file %s" % path) + f = open(path, 'w') + datastr = ''.join(string.lowercase + string.uppercase + + string.digits + '.' + '\n') + data = ''.join(16384 * datastr) + f.write(data) + f.close() + +def handler(stream, data, file_): + return file_.read(data) + +def dir_vol_upload(params): + """test volume download and check""" + + global logger + logger = params['logger'] + poolname = params['poolname'] + volname = params['volname'] + volformat = params['volformat'] + offset = int(params['offset']) + length = int(params['length']) + capacity = params['capacity'] + xmlstr = params['xml'] + + logger.info("the poolname is %s, volname is %s, volformat is %s" % + (poolname, volname, volformat)) + logger.info("upload offset is: %s" % offset) + logger.info("the data length to upload is: %s" % length) + + conn = sharedmod.libvirtobj['conn'] + try: + poolobj = conn.storagePoolLookupByName(poolname) + path_value = get_pool_path(poolobj) + volume_path = path_value + "/" + volname + + xmlstr = xmlstr.replace('VOLPATH', volume_path) + xmlstr = xmlstr.replace('SUFFIX', capacity[-1]) + xmlstr = xmlstr.replace('CAP', capacity[:-1]) + logger.debug("volume xml:\n%s" % xmlstr) + + logger.info("create %s %s volume" % (volname, volformat)) + vol = poolobj.createXML(xmlstr, 0) + + test_path = path_value + "/" + "vol_test" + write_file(test_path) + olddigest = utils.digest(test_path, 0, 0) + logger.debug("the old file digest is: %s" % olddigest) + + if offset: + origdigestpre = utils.digest(volume_path, 0, offset) + else: + origdigestpre = '' + logger.debug("the original pre region digest is: %s" % origdigestpre) + + origdigestpost = utils.digest(volume_path, offset + 1024 * 1024, 0) + logger.debug("the original post region digest is: %s" % origdigestpost) + + st = conn.newStream(0) + + f = open(test_path, 'r') + logger.info("start upload") + vol.upload(st, offset, length, 0) + logger.info("sent all data") + st.sendAll(handler, f) + logger.info("finished stream") + st.finish() + f.close() + + newdigest = utils.digest(volume_path, offset, 1024 * 1024) + logger.debug("the new file digest is: %s" % olddigest) + + if offset: + newdigestpre = utils.digest(volume_path, 0, offset) + else: + newdigestpre = '' + logger.debug("the new pre region digest is: %s" % origdigestpre) + + newdigestpost = utils.digest(volume_path, offset + 1024 * 1024, 0) + logger.debug("the new post region digest is: %s" % origdigestpost) + + if newdigestpre == origdigestpre: + logger.info("file pre region digests match") + else: + logger.error("file pre region digests not match") + return 1 + + if olddigest == newdigest: + logger.info("file digests match") + else: + logger.error("file digests not match") + return 1 + + if newdigestpost == origdigestpost: + logger.info("file post region digests match") + else: + logger.error("file post region digests not match") + return 1 + + except libvirtError, e: + logger.error("libvirt call failed: " + str(e)) + return 1 + + return 0 + +def dir_vol_upload_clean(params): + """clean testing environment""" + poolname = params['poolname'] + volname = params['volname'] + + conn = sharedmod.libvirtobj['conn'] + poolobj = conn.storagePoolLookupByName(poolname) + path_value = get_pool_path(poolobj) + test_path = path_value + "/" + "vol_test" + + vol = poolobj.storageVolLookupByName(volname) + vol.delete(0) + + if os.path.exists(test_path): + os.unlink(test_path) + + return 0 diff --git a/utils/utils.py b/utils/utils.py index 1ed673a..d3b0dc1 100644 --- a/utils/utils.py +++ b/utils/utils.py @@ -29,6 +29,7 @@ import struct import pexpect import string import subprocess +import hashlib import libvirt from xml.dom import minidom from urlparse import urlparse @@ -725,6 +726,29 @@ def param_to_tuple(paramlist, length): else: return False +def digest(path, offset, length): + """read data from file with length bytes, begin at offset + and return md5 hexdigest + """ + f = open(path, 'r') + f.seek(offset) + m = hashlib.md5() + done = 0 + + while True: + want = 1024 + if length and length - done < want: + want = length - done + outstr = f.read(want) + got = len(outstr) + if got == 0: + break + done += got + m.update(outstr) + + f.close() + return m.hexdigest() + def run_wget_app(hostname, username, password, file_url, logger): """Simple test for wget app on specified host""" cmd_line = "wget -P /tmp %s -o /tmp/wget.log" % (file_url)