[PATCH v2 03/10] scripts: Add 'qemu-replies-tool' script for testing and modifying data for qemucapabilitiestest

The tool in the current shape functionally replaces tests/qemucapabilitiesnumbering.c It validates that the output '.replies' files conform to how we generate them from qemu and also allows programmatic modification of the '.replies' files if re-generation is not feasible any more. The main advantage is that JSON objects are parsed into native python types and thus the programatic modification is much more convenient. The tool will be later extended to also do validation that we properly handle the whole of QMP schema as well as help in reviewing the differences in the .replies file after qemu updates. Signed-off-by: Peter Krempa <pkrempa(a)redhat.com&gt; --- v2: - use sys.exit() instead of exit() - remove the 'from None' hack to avoid double exceptions as it's no longer needed - modify the positional argument to take a list of files - modify the handling of --repliesdir to work in addition to the positional list of files (IMO any interlocking is pointless) - added comments what the blocks in the modification example do scripts/meson.build | 1 + scripts/qemu-replies-tool.py | 239 +++++++++++++++++++++++++++++++++++ tests/meson.build | 11 ++ 3 files changed, 251 insertions(+) create mode 100755 scripts/qemu-replies-tool.py diff --git a/scripts/meson.build b/scripts/meson.build index 283b790d03..2798e302ab 100644 --- a/scripts/meson.build +++ b/scripts/meson.build @@ -31,6 +31,7 @@ scripts = [ 'meson-timestamp.py', 'mock-noinline.py', 'prohibit-duplicate-header.py', + 'qemu-replies-tool.py', ] foreach name : scripts diff --git a/scripts/qemu-replies-tool.py b/scripts/qemu-replies-tool.py new file mode 100755 index 0000000000..5180f2c69b --- /dev/null +++ b/scripts/qemu-replies-tool.py @@ -0,0 +1,239 @@ +#!/usr/bin/env python3 +# +# SPDX-License-Identifier: LGPL-2.1-or-later +# +# A "swiss army knife" tool for qemu capability probing '.replies' files. See +# below in 'description' for more information. + +from pathlib import Path +import argparse +import json +import os +import sys + + +class qrtException(Exception): + pass + + +# Load the 'replies' file into a list of (command, reply) tuples of parsed JSON +def qemu_replies_load(filename): + conv = [] + + with open(filename, "r") as fh: + command = None + jsonstr = '' + + try: + for line in fh: + jsonstr += line + + if line == '\n': + if command is None: + command = json.loads(jsonstr) + else: + conv.append((command, json.loads(jsonstr))) + command = None + + jsonstr = '' + + if command is not None and jsonstr != '': + conv.append((command, json.loads(jsonstr))) + command = None + jsonstr = '' + + except json.decoder.JSONDecodeError as je: + raise qrtException("JSON error:\n'%s'\nwhile processing snippet:\n'%s'" % (je, jsonstr)) + + if command is not None or jsonstr != '': + if command is not None: + errorstr = json.dumps(command, indent=2) + else: + errorstr = jsonstr + + raise qrtException("replies file error: Missing reply for command:\n'%s'" % errorstr) + + return conv + + +# Format the list of (command, reply) tuples into a string and compare it with +# the 'replies' file. Optionally regenerate the replies file if the output doesn't match +def qemu_replies_compare_or_replace(filename, conv, regenerate_on_error): + actual = '' + seq = 9999 # poison the initial counter state + + # possibly fix mis-ordererd 'id' fields + for (cmd, rep) in conv: + # 'qmp_capabilities' command restarts the numbering sequence + if cmd['execute'] == 'qmp_capabilities': + seq = 1 + + newid = 'libvirt-%d' % seq + cmd['id'] = newid + rep['id'] = newid + + seq += 1 + + # format the output string + if len(actual) != 0: + actual += '\n\n' + + actual += json.dumps(cmd, indent=2) + '\n\n' + json.dumps(rep, indent=2) + + expect = '' + actual += '\n' + + with open(filename, "r") as fh: + expect = fh.read() + + if actual != expect: + if regenerate_on_error: + with open(filename, "w") as fh: + fh.write(actual) + + raise qrtException("replies file error: Expected content of '%s' doesn't match actual content" % filename) + + +# Process the replies file programmatically here. +# The 'conv' argument contains the whole conversation as a list of +# (command, reply) tuples, where both command and reply are already parsed JSON +# and thus represented by native python types (dict, list, etc ...) +# +# The code below contains a few examples and hints how to use the programatic +# processing. Do not forget to use '--regenerate' flag to uptate the output files. +# +# Beware that this updates the output file which is used as input for any +# subsequent re-run of the tool which can re-apply the modification. +def modify_replies(conv): + return # remove this to enable modifications + version = None # filled with a dictionary with 'major', 'minor', 'micro' keys + + # find version of current qemu for later use + for (cmd, rep) in conv: + if cmd['execute'] == 'query-version': + version = rep['return']['qemu'] + break + + if version is None: + raise Exception("'query-version' not found in the .replies file") + + idx = -1 + # Find index of a command, in this case we're looking for the last + # invocation of given command + for i in range(len(conv)): + (cmd, rep) = conv[i] + + if cmd['execute'] == 'device-list-properties': + idx = i + + if idx == -1: + raise Exception("entry not found") + + # Prepare data for inserting a new command + + # Command definition and error are instantiated via native python types + cmd = {'execute': 'device-list-properties', + 'arguments': {'typename': 'example-device'}} + + reply_unsupp = {'error': {'class': 'DeviceNotFound', + 'desc': "Device 'example-device' not found"}} + + # Real reply data can be also parsed from JSON + reply = json.loads(''' + { + "return": [ + { + "name": "dummy_prop", + "type": "str" + }, + { + "name": "test", + "type": "str" + } + ] + } + ''') + + # insert command into the QMP conversation based on version of qemu + if version['major'] >= 8 and version['minor'] > 0: + conv.insert(idx, (cmd, reply)) + else: + conv.insert(idx, (cmd, reply_unsupp)) + + +def process_one(filename, args): + try: + conv = qemu_replies_load(filename) + + modify_replies(conv) + + qemu_replies_compare_or_replace(filename, conv, args.regenerate) + + except qrtException as e: + print("'%s' ... FAIL\n%s" % (filename, e)) + return False + + print("'%s' ... OK" % filename) + return True + + +description = '''A Swiss army knife tool for '.replies' files used by 'qemucapabilitiestest' + +This tool is used to validate, programmatically update or inspect the +'.*replies' normally stored files under 'tests/qemucapabilitiesdata'. + +By default the file(s) passed as positional argument areused. All '.replies' +iles in a directory can be processed by specifying '--repliesdir /path/to/dir' +argument. + +The default mode is validation which checks the following: + - each command has a reply and both are valid JSON + - numbering of the 'id' field is as expected + - the input file has the expected JSON formatting + +The tool can be also used to programmaticaly modify the '.replies' file by +editting the 'modify_replies' method directly in the source, or for +re-formatting and re-numbering the '.replies' file to conform with the required +format. To update the output file the '--regenerate' flag can be used or the +'VIR_TEST_REGENERATE_OUTPUT' environment variable must be set to '1'. +''' + +if os.environ.get('VIR_TEST_REGENERATE_OUTPUT', '0') == '1': + default_regenerate = True +else: + default_regenerate = False + +parser = argparse.ArgumentParser(formatter_class=argparse.RawDescriptionHelpFormatter, + description=description) + +parser.add_argument('--regenerate', action="store_true", default=default_regenerate, + help="regenerate output file if actual output doesn't match") + +parser.add_argument('--repliesdir', default='', + help='use all .replies files from the directory') + +parser.add_argument('replyfiles', nargs='*', + help='.replies file(s) to process') + +args = parser.parse_args() + +files = [] + +if args.replyfiles: + files += args.replyfiles + +if args.repliesdir: + files += Path(args.repliesdir).glob('*.replies') + +if len(files) == 0: + parser.print_help() + sys.exit(1) + +fail = False + +for file in files: + if not process_one(str(file), args): + fail = True + +if fail: + sys.exit(1) diff --git a/tests/meson.build b/tests/meson.build index 8969f29b08..419e2d5c44 100644 --- a/tests/meson.build +++ b/tests/meson.build @@ -605,6 +605,17 @@ foreach data : tests ) endforeach +test( + 'qemu replies check', + python3_prog, + args: [ + qemu_replies_tool_prog.full_path(), + '--repliesdir', + meson.project_source_root() / 'tests' / 'qemucapabilitiesdata' + ], + env: runutf8, +) + # helpers: # each entry is a dictionary with following items: -- 2.43.0