[PATCH 03/10] scripts: Add 'qemu-qmp-replies-tool' script for testing and modifying data for qemucapabilitiestest

The tool in the current shape functionally replaces tests/qemucapabilitiesnumbering.c It validates that the output '.replies' files conform to how we generate them from qemu and also allows programmatic modification of the '.replies' files if re-generation is not feasible any more. The main advantage is that JSON objects are parsed into native python types and thus the programatic modification is much more convenient. The tool will be later extended to also do validation that we properly handle the whole of QMP schema as well as help in reviewing the differences in the .replies file after qemu updates. Signed-off-by: Peter Krempa <pkrempa(a)redhat.com&gt; --- scripts/meson.build | 1 + scripts/qemu-replies-tool.py | 228 +++++++++++++++++++++++++++++++++++ tests/meson.build | 11 ++ 3 files changed, 240 insertions(+) create mode 100755 scripts/qemu-replies-tool.py diff --git a/scripts/meson.build b/scripts/meson.build index 283b790d03..2798e302ab 100644 --- a/scripts/meson.build +++ b/scripts/meson.build @@ -31,6 +31,7 @@ scripts = [ 'meson-timestamp.py', 'mock-noinline.py', 'prohibit-duplicate-header.py', + 'qemu-replies-tool.py', ] foreach name : scripts diff --git a/scripts/qemu-replies-tool.py b/scripts/qemu-replies-tool.py new file mode 100755 index 0000000000..57bb26a356 --- /dev/null +++ b/scripts/qemu-replies-tool.py @@ -0,0 +1,228 @@ +#!/usr/bin/env python3 +# +# SPDX-License-Identifier: LGPL-2.1-or-later +# +# A "swiss army knife" tool for qemu capability probing '.replies' files. See +# below in 'description' for more information. + +from pathlib import Path +import argparse +import json +import os + + +class qrtException(Exception): + pass + + +# Load the 'replies' file into a list of (command, reply) tuples of parsed JSON +def qemu_replies_load(filename): + conv = [] + + with open(filename, "r") as fh: + command = None + jsonstr = '' + + try: + for line in fh: + jsonstr += line + + if line == '\n': + if command is None: + command = json.loads(jsonstr) + else: + conv.append((command, json.loads(jsonstr))) + command = None + + jsonstr = '' + + if command is not None and jsonstr != '': + conv.append((command, json.loads(jsonstr))) + command = None + jsonstr = '' + + except json.decoder.JSONDecodeError as je: + raise qrtException("JSON error:\n'%s'\nwhile processing snippet:\n'%s'" % (je, jsonstr)) from None + + if command is not None or jsonstr != '': + if command is not None: + errorstr = json.dumps(command, indent=2) + else: + errorstr = jsonstr + + raise qrtException("replies file error: Missing reply for command:\n'%s'" % errorstr) + + return conv + + +# Format the list of (command, reply) tuples into a string and compare it with +# the 'replies' file. Optionally regenerate the replies file if the output doesn't match +def qemu_replies_compare_or_replace(filename, conv, regenerate_on_error): + actual = '' + seq = 9999 # poison the initial counter state + + # possibly fix mis-ordererd 'id' fields + for (cmd, rep) in conv: + # 'qmp_capabilities' command restarts the numbering sequence + if cmd['execute'] == 'qmp_capabilities': + seq = 1 + + newid = 'libvirt-%d' % seq + cmd['id'] = newid + rep['id'] = newid + + seq += 1 + + # format the output string + if len(actual) != 0: + actual += '\n\n' + + actual += json.dumps(cmd, indent=2) + '\n\n' + json.dumps(rep, indent=2) + + expect = '' + actual += '\n' + + with open(filename, "r") as fh: + expect = fh.read() + + if actual != expect: + if regenerate_on_error: + with open(filename, "w") as fh: + fh.write(actual) + + raise qrtException("replies file error: Expected content of '%s' doesn't match actual content" % filename) + + +# Process the replies file programmatically here. +# The 'conv' argument contains the whole conversation as a list of +# (command, reply) tuples, where both command and reply are already parsed JSON +# and thus represented by native python types (dict, list, etc ...) +# +# The code below contains a few examples and hints how to use the programatic +# processing. Do not forget to use '--regenerate' flag to uptate the output files. +# +# Beware that this updates the output file which is used as input for any +# subsequent re-run of the tool which can re-apply the modification. +def modify_replies(conv): + return # remove this to enable modifications + version = None # filled with a dictionary with 'major', 'minor', 'micro' keys + + for (cmd, rep) in conv: + if cmd['execute'] == 'query-version': + version = rep['return']['qemu'] + break + + if version is None: + raise Exception("'query-version' not found in the .replies file") + + idx = -1 + # Find an index of an entry + for i in range(len(conv)): + (cmd, rep) = conv[i] + + if cmd['execute'] == 'device-list-properties': + idx = i + + if idx == -1: + raise Exception("entry not found") + + cmd = {'execute': 'device-list-properties', + 'arguments': {'typename': 'example-device'}} + + reply_unsupp = {'error': {'class': 'DeviceNotFound', + 'desc': "Device 'example-device' not found"}} + + reply = json.loads(''' + { + "return": [ + { + "name": "dummy_prop", + "type": "str" + }, + { + "name": "test", + "type": "str" + } + ] + } + ''') + + if version['major'] >= 8 and version['minor'] > 0: + conv.insert(idx, (cmd, reply)) + else: + conv.insert(idx, (cmd, reply_unsupp)) + + +def process_one(filename, args): + try: + conv = qemu_replies_load(filename) + + modify_replies(conv) + + qemu_replies_compare_or_replace(filename, conv, args.regenerate) + + except qrtException as e: + print("'%s' ... FAIL\n%s" % (filename, e)) + return False + + print("'%s' ... OK" % filename) + return True + + +description = '''A Swiss army knife tool for '.replies' files used by 'qemucapabilitiestest' + +This tool is used to validate, programmatically update or inspect the +'.*replies' normally stored files under 'tests/qemucapabilitiesdata'. + +By default the file passed as argument is used. All '.replies' files in a +directory can be processed by specifying '--repliesdir /path/to/dir' argument. + +The default mode is validation which checks the following: + - each command has a reply and both are valid JSON + - numbering of the 'id' field is as expected + - the input file has the expected JSON formatting + +The tool can be also used to programmaticaly modify the '.replies' file by +editting the 'modify_replies' method directly in the source, or for +re-formatting and re-numbering the '.replies' file to conform with the required +format. To update the output file the '--regenerate' flag can be used or the +'VIR_TEST_REGENERATE_OUTPUT' environment variable must be set to '1'. +''' + +if os.environ.get('VIR_TEST_REGENERATE_OUTPUT', '0') == '1': + default_regenerate = True +else: + default_regenerate = False + +parser = argparse.ArgumentParser(formatter_class=argparse.RawDescriptionHelpFormatter, + description=description) + +parser.add_argument('--regenerate', action="store_true", default=default_regenerate, + help="regenerate output file if actual output doesn't match") + +parser.add_argument('--repliesdir', default='', + help='directory containing .replies files') + +parser.add_argument('replyfile', nargs='?', + help='path to .replies file to use') + +args = parser.parse_args() + +if not args.replyfile and not args.repliesdir: + parser.print_help() + exit(1) + +fail = False + +if args.replyfile: + if not process_one(args.replyfile, args): + fail = True +else: + files = Path(args.repliesdir).glob('*.replies') + + for file in files: + if not process_one(str(file), args): + fail = True + +if fail: + exit(1) diff --git a/tests/meson.build b/tests/meson.build index e1cd57654a..bef3ed3260 100644 --- a/tests/meson.build +++ b/tests/meson.build @@ -612,6 +612,17 @@ foreach data : tests ) endforeach +test( + 'qemu replies check', + python3_prog, + args: [ + qemu_replies_tool_prog.full_path(), + '--repliesdir', + meson.project_source_root() / 'tests' / 'qemucapabilitiesdata' + ], + env: runutf8, +) + # helpers: # each entry is a dictionary with following items: -- 2.43.0