[libvirt PATCH 0/9] Merge scripts in tests/cputestdata
This series merges the functionality of cpu-cpuid.py into cpu-gather.py and completes the transition. For background, see https://www.redhat.com/archives/libvir-list/2020-December/msg00706.html Tim Wiederhake (9): cpu-cpuid: Use argparse to parse arguments cpu-cpuid: Remove xmltodict usage in parseMap cpu-cpuid: Remove xmltodict usage in parseCPU cpu-cpuid: Merge addFeature functions cpu-cpuid: Merge checkFeature functions cpu-cpuid: Deduplicate register list cpu-gather: Use actions instead of flags for action argument cpu-gather: Factor out call to cpu-cpuid.py cpu-gather: Merge cpu-cpuid.py tests/cputestdata/cpu-cpuid.py | 229 -------------------------------- tests/cputestdata/cpu-gather.py | 178 ++++++++++++++++++++++--- 2 files changed, 156 insertions(+), 251 deletions(-) delete mode 100755 tests/cputestdata/cpu-cpuid.py -- 2.26.2
Using 'argparse' for argument handling simplifies merging this script with cpu-gather.py in a later patch. Signed-off-by: Tim Wiederhake <twiederh@redhat.com> --- tests/cputestdata/cpu-cpuid.py | 78 +++++++++++++++++++--------------- 1 file changed, 43 insertions(+), 35 deletions(-) diff --git a/tests/cputestdata/cpu-cpuid.py b/tests/cputestdata/cpu-cpuid.py index dac43debb6..6ca72d2262 100755 --- a/tests/cputestdata/cpu-cpuid.py +++ b/tests/cputestdata/cpu-cpuid.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 +import argparse import os import sys import json @@ -191,39 +192,46 @@ def formatCPUData(cpuData, path, comment): f.write("</cpudata>\n") -def diff(cpuMap, path): - base = path.replace(".json", "") - jsonFile = path - cpuDataFile = base + ".xml" - enabledFile = base + "-enabled.xml" - disabledFile = base + "-disabled.xml" - - cpuData = parseCPUData(cpuDataFile) - qemu = parseQemu(jsonFile, cpuMap) - - enabled = {"cpuid": {}} - disabled = {"cpuid": {}} - for feature in cpuMap.values(): - if checkFeature(qemu, feature): - addFeature(enabled, feature) - elif checkFeature(cpuData, feature): - addFeature(disabled, feature) - - formatCPUData(enabled, enabledFile, "Features enabled by QEMU") - formatCPUData(disabled, disabledFile, "Features disabled by QEMU") - - -if len(sys.argv) < 3: - print("Usage: %s diff json_file..." % sys.argv[0]) - sys.exit(1) - -action = sys.argv[1] -args = sys.argv[2:] - -if action == "diff": +def diff(args): cpuMap = parseMap() - for path in args: - diff(cpuMap, path) -else: - print("Unknown action: %s" % action) - sys.exit(1) + + for jsonFile in args.json_files: + cpuDataFile = jsonFile.replace(".json", ".xml") + enabledFile = jsonFile.replace(".json", "-enabled.xml") + disabledFile = jsonFile.replace(".json", "-disabled.xml") + + cpuData = parseCPUData(cpuDataFile) + qemu = parseQemu(jsonFile, cpuMap) + + enabled = dict() + disabled = dict() + for feature in cpuMap.values(): + if checkFeature(qemu, feature): + addFeature(enabled, feature) + elif checkFeature(cpuData, feature): + addFeature(disabled, feature) + + formatCPUData(enabled, enabledFile, "Features enabled by QEMU") + formatCPUData(disabled, disabledFile, "Features disabled by QEMU") + + +def main(): + parser = argparse.ArgumentParser(description="Diff cpuid results") + subparsers = parser.add_subparsers(dest="action", required=True) + diffparser = subparsers.add_parser( + "diff", + help="Diff json description of CPU model against known features.") + diffparser.add_argument( + "json_files", + nargs="+", + metavar="FILE", + type=os.path.realpath, + help="Path to one or more json CPU model descriptions.") + args = parser.parse_args() + + diff(args) + exit(0) + + +if __name__ == "__main__": + main() -- 2.26.2
On Mon, Jan 04, 2021 at 12:30:11 +0100, Tim Wiederhake wrote:
Using 'argparse' for argument handling simplifies merging this script with cpu-gather.py in a later patch.
Signed-off-by: Tim Wiederhake <twiederh@redhat.com> --- tests/cputestdata/cpu-cpuid.py | 78 +++++++++++++++++++--------------- 1 file changed, 43 insertions(+), 35 deletions(-)
diff --git a/tests/cputestdata/cpu-cpuid.py b/tests/cputestdata/cpu-cpuid.py index dac43debb6..6ca72d2262 100755 --- a/tests/cputestdata/cpu-cpuid.py +++ b/tests/cputestdata/cpu-cpuid.py ... +def main(): + parser = argparse.ArgumentParser(description="Diff cpuid results") + subparsers = parser.add_subparsers(dest="action", required=True) + diffparser = subparsers.add_parser( + "diff", + help="Diff json description of CPU model against known features.") + diffparser.add_argument( + "json_files", + nargs="+", + metavar="FILE", + type=os.path.realpath, + help="Path to one or more json CPU model descriptions.") + args = parser.parse_args() + + diff(args) + exit(0)
This exit(0) here is redundant, it's removed by the last patch in this series, and our syntax check doesn't like it. So I just removed this line before pushing. Jirka
'xmltodict' is a Python module that is not installed by default. Replace it, so the dependencies of cpu-gather.py do not change when both scripts are merged. Signed-off-by: Tim Wiederhake <twiederh@redhat.com> --- tests/cputestdata/cpu-cpuid.py | 39 ++++++++++++---------------------- 1 file changed, 13 insertions(+), 26 deletions(-) diff --git a/tests/cputestdata/cpu-cpuid.py b/tests/cputestdata/cpu-cpuid.py index 6ca72d2262..8e06baea85 100755 --- a/tests/cputestdata/cpu-cpuid.py +++ b/tests/cputestdata/cpu-cpuid.py @@ -5,6 +5,7 @@ import os import sys import json import xmltodict +import xml.etree.ElementTree def checkCPUIDFeature(cpuData, feature): @@ -132,37 +133,23 @@ def parseCPUData(path): return cpuData -def parseMapFeature(fType, data): - ret = {"type": fType} - - if fType == "cpuid": - fields = ["eax_in", "ecx_in", "eax", "ebx", "ecx", "edx"] - elif fType == "msr": - fields = ["index", "edx", "eax"] - - for field in fields: - attr = "@%s" % field - if attr in data: - ret[field] = int(data[attr], 0) - else: - ret[field] = 0 - - return ret - - def parseMap(): path = os.path.dirname(sys.argv[0]) path = os.path.join(path, "..", "..", "src", "cpu_map", "x86_features.xml") - with open(path, "rb") as f: - data = xmltodict.parse(f) - cpuMap = {} - for feature in data["cpus"]["feature"]: - for fType in ["cpuid", "msr"]: - if fType not in feature: - continue - cpuMap[feature["@name"]] = parseMapFeature(fType, feature[fType]) + cpuMap = dict() + for f in xml.etree.ElementTree.parse(path).getroot().iter("feature"): + if f[0].tag == "cpuid": + reg_list = ["eax_in", "ecx_in", "eax", "ebx", "ecx", "edx"] + elif f[0].tag == "msr": + reg_list = ["index", "eax", "edx"] + else: + continue + feature = {"type": f[0].tag} + for reg in reg_list: + feature[reg] = int(f[0].attrib.get(reg, "0"), 0) + cpuMap[f.attrib["name"]] = feature return cpuMap -- 2.26.2
'xmltodict' is a Python module that is not installed by default. Replace it, so the dependencies of cpu-gather.py do not change when both scripts are merged. Signed-off-by: Tim Wiederhake <twiederh@redhat.com> --- tests/cputestdata/cpu-cpuid.py | 35 +++++++++++----------------------- 1 file changed, 11 insertions(+), 24 deletions(-) diff --git a/tests/cputestdata/cpu-cpuid.py b/tests/cputestdata/cpu-cpuid.py index 8e06baea85..d570884db6 100755 --- a/tests/cputestdata/cpu-cpuid.py +++ b/tests/cputestdata/cpu-cpuid.py @@ -4,7 +4,6 @@ import argparse import os import sys import json -import xmltodict import xml.etree.ElementTree @@ -105,31 +104,19 @@ def parseQemu(path, features): def parseCPUData(path): - cpuData = {} - with open(path, "rb") as f: - data = xmltodict.parse(f) - - for leaf in data["cpudata"]["cpuid"]: - feature = {"type": "cpuid"} - feature["eax_in"] = int(leaf["@eax_in"], 0) - feature["ecx_in"] = int(leaf["@ecx_in"], 0) - for reg in ["eax", "ebx", "ecx", "edx"]: - feature[reg] = int(leaf["@" + reg], 0) + cpuData = dict() + for f in xml.etree.ElementTree.parse(path).getroot(): + if f.tag == "cpuid": + reg_list = ["eax_in", "ecx_in", "eax", "ebx", "ecx", "edx"] + elif f.tag == "msr": + reg_list = ["index", "eax", "edx"] + else: + continue + feature = {"type": f.tag} + for reg in reg_list: + feature[reg] = int(f.attrib.get(reg, "0"), 0) addFeature(cpuData, feature) - - if "msr" in data["cpudata"]: - if not isinstance(data["cpudata"]["msr"], list): - data["cpudata"]["msr"] = [data["cpudata"]["msr"]] - - for msr in data["cpudata"]["msr"]: - feature = {"type": "msr"} - feature["index"] = int(msr["@index"], 0) - feature["edx"] = int(msr["@edx"], 0) - feature["eax"] = int(msr["@eax"], 0) - - addFeature(cpuData, feature) - return cpuData -- 2.26.2
Prepare to deduplicate the list of relevant registers for cpuid and msr information. Signed-off-by: Tim Wiederhake <twiederh@redhat.com> --- tests/cputestdata/cpu-cpuid.py | 48 ++++++++++++---------------------- 1 file changed, 16 insertions(+), 32 deletions(-) diff --git a/tests/cputestdata/cpu-cpuid.py b/tests/cputestdata/cpu-cpuid.py index d570884db6..c9948da6f8 100755 --- a/tests/cputestdata/cpu-cpuid.py +++ b/tests/cputestdata/cpu-cpuid.py @@ -54,41 +54,25 @@ def checkFeature(cpuData, feature): return checkMSRFeature(cpuData, feature) -def addCPUIDFeature(cpuData, feature): - if "cpuid" not in cpuData: - cpuData["cpuid"] = {} - cpuid = cpuData["cpuid"] - - if feature["eax_in"] not in cpuid: - cpuid[feature["eax_in"]] = {} - leaf = cpuid[feature["eax_in"]] - - if feature["ecx_in"] not in leaf: - leaf[feature["ecx_in"]] = {"eax": 0, "ebx": 0, "ecx": 0, "edx": 0} - leaf = leaf[feature["ecx_in"]] - - for reg in ["eax", "ebx", "ecx", "edx"]: - leaf[reg] |= feature[reg] - - -def addMSRFeature(cpuData, feature): - if "msr" not in cpuData: - cpuData["msr"] = {} - msr = cpuData["msr"] - - if feature["index"] not in msr: - msr[feature["index"]] = {"edx": 0, "eax": 0} - msr = msr[feature["index"]] - - for reg in ["edx", "eax"]: - msr[reg] |= feature[reg] - - def addFeature(cpuData, feature): if feature["type"] == "cpuid": - addCPUIDFeature(cpuData, feature) + # cpuData["cpuid"][eax_in][ecx_in] = {eax:, ebx:, ecx:, edx:} + keyList = ["type", "eax_in", "ecx_in"] + regList = ["eax", "ebx", "ecx", "edx"] elif feature["type"] == "msr": - addMSRFeature(cpuData, feature) + # cpuData["msr"][index] = {eax:, edx:} + keyList = ["type", "index"] + regList = ["eax", "edx"] + else: + return + + for key in keyList: + if feature[key] not in cpuData: + cpuData[feature[key]] = dict() + cpuData = cpuData[feature[key]] + + for reg in regList: + cpuData[reg] = cpuData.get(reg, 0) | feature[reg] def parseQemu(path, features): -- 2.26.2
Prepare to deduplicate the list of relevant registers for cpuid and msr information. Signed-off-by: Tim Wiederhake <twiederh@redhat.com> --- tests/cputestdata/cpu-cpuid.py | 60 ++++++++++------------------------ 1 file changed, 18 insertions(+), 42 deletions(-) diff --git a/tests/cputestdata/cpu-cpuid.py b/tests/cputestdata/cpu-cpuid.py index c9948da6f8..e90165e563 100755 --- a/tests/cputestdata/cpu-cpuid.py +++ b/tests/cputestdata/cpu-cpuid.py @@ -7,51 +7,27 @@ import json import xml.etree.ElementTree -def checkCPUIDFeature(cpuData, feature): - eax_in = feature["eax_in"] - ecx_in = feature["ecx_in"] - eax = feature["eax"] - ebx = feature["ebx"] - ecx = feature["ecx"] - edx = feature["edx"] - - if "cpuid" not in cpuData: - return False - - cpuid = cpuData["cpuid"] - if eax_in not in cpuid or ecx_in not in cpuid[eax_in]: - return False - - leaf = cpuid[eax_in][ecx_in] - return ((eax > 0 and leaf["eax"] & eax == eax) or - (ebx > 0 and leaf["ebx"] & ebx == ebx) or - (ecx > 0 and leaf["ecx"] & ecx == ecx) or - (edx > 0 and leaf["edx"] & edx == edx)) - - -def checkMSRFeature(cpuData, feature): - index = feature["index"] - edx = feature["edx"] - eax = feature["eax"] - - if "msr" not in cpuData: - return False - - msr = cpuData["msr"] - if index not in msr: - return False - - msr = msr[index] - return ((edx > 0 and msr["edx"] & edx == edx) or - (eax > 0 and msr["eax"] & eax == eax)) - - def checkFeature(cpuData, feature): if feature["type"] == "cpuid": - return checkCPUIDFeature(cpuData, feature) + # cpuData["cpuid"][eax_in][ecx_in] = {eax:, ebx:, ecx:, edx:} + keyList = ["type", "eax_in", "ecx_in"] + regList = ["eax", "ebx", "ecx", "edx"] + elif feature["type"] == "msr": + # cpuData["msr"][index] = {eax:, edx:} + keyList = ["type", "index"] + regList = ["eax", "edx"] + else: + return False + + for key in keyList: + if feature[key] not in cpuData: + return False + cpuData = cpuData[feature[key]] - if feature["type"] == "msr": - return checkMSRFeature(cpuData, feature) + for reg in regList: + if feature[reg] > 0 and feature[reg] == feature[reg] & cpuData[reg]: + return True + return False def addFeature(cpuData, feature): -- 2.26.2
Signed-off-by: Tim Wiederhake <twiederh@redhat.com> --- tests/cputestdata/cpu-cpuid.py | 57 ++++++++++++---------------------- 1 file changed, 19 insertions(+), 38 deletions(-) diff --git a/tests/cputestdata/cpu-cpuid.py b/tests/cputestdata/cpu-cpuid.py index e90165e563..3e852d5d55 100755 --- a/tests/cputestdata/cpu-cpuid.py +++ b/tests/cputestdata/cpu-cpuid.py @@ -7,47 +7,36 @@ import json import xml.etree.ElementTree +_KEYS = { + "cpuid": ["eax_in", "ecx_in"], + "msr": ["index"], +} + +_REGS = { + "cpuid": ["eax", "ebx", "ecx", "edx"], + "msr": ["eax", "edx"], +} + + def checkFeature(cpuData, feature): - if feature["type"] == "cpuid": - # cpuData["cpuid"][eax_in][ecx_in] = {eax:, ebx:, ecx:, edx:} - keyList = ["type", "eax_in", "ecx_in"] - regList = ["eax", "ebx", "ecx", "edx"] - elif feature["type"] == "msr": - # cpuData["msr"][index] = {eax:, edx:} - keyList = ["type", "index"] - regList = ["eax", "edx"] - else: - return False - - for key in keyList: + for key in ["type"] + _KEYS.get(feature["type"], list()): if feature[key] not in cpuData: return False cpuData = cpuData[feature[key]] - for reg in regList: + for reg in _REGS.get(feature["type"], list()): if feature[reg] > 0 and feature[reg] == feature[reg] & cpuData[reg]: return True return False def addFeature(cpuData, feature): - if feature["type"] == "cpuid": - # cpuData["cpuid"][eax_in][ecx_in] = {eax:, ebx:, ecx:, edx:} - keyList = ["type", "eax_in", "ecx_in"] - regList = ["eax", "ebx", "ecx", "edx"] - elif feature["type"] == "msr": - # cpuData["msr"][index] = {eax:, edx:} - keyList = ["type", "index"] - regList = ["eax", "edx"] - else: - return - - for key in keyList: + for key in ["type"] + _KEYS.get(feature["type"], list()): if feature[key] not in cpuData: cpuData[feature[key]] = dict() cpuData = cpuData[feature[key]] - for reg in regList: + for reg in _REGS.get(feature["type"], list()): cpuData[reg] = cpuData.get(reg, 0) | feature[reg] @@ -66,15 +55,11 @@ def parseQemu(path, features): def parseCPUData(path): cpuData = dict() for f in xml.etree.ElementTree.parse(path).getroot(): - if f.tag == "cpuid": - reg_list = ["eax_in", "ecx_in", "eax", "ebx", "ecx", "edx"] - elif f.tag == "msr": - reg_list = ["index", "eax", "edx"] - else: + if f.tag not in ("cpuid", "msr"): continue feature = {"type": f.tag} - for reg in reg_list: + for reg in _KEYS[f.tag] + _REGS[f.tag]: feature[reg] = int(f.attrib.get(reg, "0"), 0) addFeature(cpuData, feature) return cpuData @@ -86,15 +71,11 @@ def parseMap(): cpuMap = dict() for f in xml.etree.ElementTree.parse(path).getroot().iter("feature"): - if f[0].tag == "cpuid": - reg_list = ["eax_in", "ecx_in", "eax", "ebx", "ecx", "edx"] - elif f[0].tag == "msr": - reg_list = ["index", "eax", "edx"] - else: + if f[0].tag not in ("cpuid", "msr"): continue feature = {"type": f[0].tag} - for reg in reg_list: + for reg in _KEYS[f[0].tag] + _REGS[f[0].tag]: feature[reg] = int(f[0].attrib.get(reg, "0"), 0) cpuMap[f.attrib["name"]] = feature return cpuMap -- 2.26.2
This allows for the functionality of cpu-cpuid.py script to be integrated more naturally in a later patch. Changes the way this script should be called: cpu-gather.py -> cpu-gather.py cpu-gather.py --gather -> cpu-gather.py gather cpu-gather.py --parse -> cpu-gather.py parse cpu-gather.py --gather --parse -> cpu-gather.py full Signed-off-by: Tim Wiederhake <twiederh@redhat.com> --- tests/cputestdata/cpu-gather.py | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/tests/cputestdata/cpu-gather.py b/tests/cputestdata/cpu-gather.py index a01c615504..5ca19e5f2b 100755 --- a/tests/cputestdata/cpu-gather.py +++ b/tests/cputestdata/cpu-gather.py @@ -324,21 +324,22 @@ def main(): help="Path to qemu. " "If unset, will try '/usr/bin/qemu-system-x86_64', " "'/usr/bin/qemu-kvm', and '/usr/libexec/qemu-kvm'.") - parser.add_argument( - "--gather", - action="store_true", - help="Acquire data on target system. This is the default. " - "If '--parse' is not set, outputs data on stdout.") - parser.add_argument( - "--parse", - action="store_true", - help="Parse data for libvirt use. " - "If '--gather' is not set, expects input on stdin.") + subparsers = parser.add_subparsers(dest="action") + subparsers.add_parser( + "gather", + help="Acquire data on target system and outputs to stdout. " + "This is the default. ") + subparsers.add_parser( + "parse", + help="Reads data from stdin and parses data for libvirt use.") + subparsers.add_parser( + "full", + help="Equivalent to `cpu-gather gather | cpu-gather parse`.") args = parser.parse_args() - if not args.gather and not args.parse: - args.gather = True + if not args.action: + args.action = "gather" if not args.path_to_qemu: args.path_to_qemu = "qemu-system-x86_64" @@ -350,13 +351,13 @@ def main(): if os.path.isfile(f): args.path_to_qemu = f - if args.gather: + if args.action in ["gather", "full"]: data = gather(args) - if not args.parse: + if args.action == "gather": json.dump(data, sys.stdout, indent=2) - if args.parse: - if not args.gather: + if args.action in ["parse", "full"]: + if args.action == "parse": data = json.load(sys.stdin) parse(data) -- 2.26.2
This is a preparatory step to merge cpu-cpuid.py. Signed-off-by: Tim Wiederhake <twiederh@redhat.com> --- tests/cputestdata/cpu-gather.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/tests/cputestdata/cpu-gather.py b/tests/cputestdata/cpu-gather.py index 5ca19e5f2b..a4a4050e42 100755 --- a/tests/cputestdata/cpu-gather.py +++ b/tests/cputestdata/cpu-gather.py @@ -288,7 +288,7 @@ def output_json(data, filename): f.write("\n") -def parse(data): +def parse(args, data): filename = parse_filename(data) filename_xml = "{}.xml".format(filename) filename_json = "{}.json".format(filename) @@ -301,10 +301,7 @@ def parse(data): if os.path.getsize(filename_json) == 0: return - output = subprocess.check_output( - ["./cpu-cpuid.py", "diff", filename_json], - universal_newlines=True) - print(output) + args.json_files = getattr(args, "json_files", list()) + [filename_json] def main(): @@ -359,7 +356,12 @@ def main(): if args.action in ["parse", "full"]: if args.action == "parse": data = json.load(sys.stdin) - parse(data) + parse(args, data) + + if "json_files" in args: + cmd = ["./cpu-cpuid.py", "diff"] + cmd.extend(args.json_files) + subprocess.check_call(cmd, universal_newlines=True) if __name__ == "__main__": -- 2.26.2
Old usage: cpu-cpuid.py diff FILE... New usage: cpu-gather.py diff FILE... Signed-off-by: Tim Wiederhake <twiederh@redhat.com> --- tests/cputestdata/cpu-cpuid.py | 152 -------------------------------- tests/cputestdata/cpu-gather.py | 137 +++++++++++++++++++++++++++- 2 files changed, 134 insertions(+), 155 deletions(-) delete mode 100755 tests/cputestdata/cpu-cpuid.py diff --git a/tests/cputestdata/cpu-cpuid.py b/tests/cputestdata/cpu-cpuid.py deleted file mode 100755 index 3e852d5d55..0000000000 --- a/tests/cputestdata/cpu-cpuid.py +++ /dev/null @@ -1,152 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import os -import sys -import json -import xml.etree.ElementTree - - -_KEYS = { - "cpuid": ["eax_in", "ecx_in"], - "msr": ["index"], -} - -_REGS = { - "cpuid": ["eax", "ebx", "ecx", "edx"], - "msr": ["eax", "edx"], -} - - -def checkFeature(cpuData, feature): - for key in ["type"] + _KEYS.get(feature["type"], list()): - if feature[key] not in cpuData: - return False - cpuData = cpuData[feature[key]] - - for reg in _REGS.get(feature["type"], list()): - if feature[reg] > 0 and feature[reg] == feature[reg] & cpuData[reg]: - return True - return False - - -def addFeature(cpuData, feature): - for key in ["type"] + _KEYS.get(feature["type"], list()): - if feature[key] not in cpuData: - cpuData[feature[key]] = dict() - cpuData = cpuData[feature[key]] - - for reg in _REGS.get(feature["type"], list()): - cpuData[reg] = cpuData.get(reg, 0) | feature[reg] - - -def parseQemu(path, features): - cpuData = {} - with open(path, "r") as f: - data, pos = json.JSONDecoder().raw_decode(f.read()) - - for (prop, val) in data["return"]["model"]["props"].items(): - if val and prop in features: - addFeature(cpuData, features[prop]) - - return cpuData - - -def parseCPUData(path): - cpuData = dict() - for f in xml.etree.ElementTree.parse(path).getroot(): - if f.tag not in ("cpuid", "msr"): - continue - - feature = {"type": f.tag} - for reg in _KEYS[f.tag] + _REGS[f.tag]: - feature[reg] = int(f.attrib.get(reg, "0"), 0) - addFeature(cpuData, feature) - return cpuData - - -def parseMap(): - path = os.path.dirname(sys.argv[0]) - path = os.path.join(path, "..", "..", "src", "cpu_map", "x86_features.xml") - - cpuMap = dict() - for f in xml.etree.ElementTree.parse(path).getroot().iter("feature"): - if f[0].tag not in ("cpuid", "msr"): - continue - - feature = {"type": f[0].tag} - for reg in _KEYS[f[0].tag] + _REGS[f[0].tag]: - feature[reg] = int(f[0].attrib.get(reg, "0"), 0) - cpuMap[f.attrib["name"]] = feature - return cpuMap - - -def formatCPUData(cpuData, path, comment): - print(path) - with open(path, "w") as f: - f.write("<!-- " + comment + " -->\n") - f.write("<cpudata arch='x86'>\n") - - cpuid = cpuData["cpuid"] - for eax_in in sorted(cpuid.keys()): - for ecx_in in sorted(cpuid[eax_in].keys()): - leaf = cpuid[eax_in][ecx_in] - line = (" <cpuid eax_in='0x%08x' ecx_in='0x%02x' " - "eax='0x%08x' ebx='0x%08x' " - "ecx='0x%08x' edx='0x%08x'/>\n") - f.write(line % ( - eax_in, ecx_in, - leaf["eax"], leaf["ebx"], leaf["ecx"], leaf["edx"])) - - if "msr" in cpuData: - msr = cpuData["msr"] - for index in sorted(msr.keys()): - f.write(" <msr index='0x%x' edx='0x%08x' eax='0x%08x'/>\n" % - (index, msr[index]['edx'], msr[index]['eax'])) - - f.write("</cpudata>\n") - - -def diff(args): - cpuMap = parseMap() - - for jsonFile in args.json_files: - cpuDataFile = jsonFile.replace(".json", ".xml") - enabledFile = jsonFile.replace(".json", "-enabled.xml") - disabledFile = jsonFile.replace(".json", "-disabled.xml") - - cpuData = parseCPUData(cpuDataFile) - qemu = parseQemu(jsonFile, cpuMap) - - enabled = dict() - disabled = dict() - for feature in cpuMap.values(): - if checkFeature(qemu, feature): - addFeature(enabled, feature) - elif checkFeature(cpuData, feature): - addFeature(disabled, feature) - - formatCPUData(enabled, enabledFile, "Features enabled by QEMU") - formatCPUData(disabled, disabledFile, "Features disabled by QEMU") - - -def main(): - parser = argparse.ArgumentParser(description="Diff cpuid results") - subparsers = parser.add_subparsers(dest="action", required=True) - diffparser = subparsers.add_parser( - "diff", - help="Diff json description of CPU model against known features.") - diffparser.add_argument( - "json_files", - nargs="+", - metavar="FILE", - type=os.path.realpath, - help="Path to one or more json CPU model descriptions.") - args = parser.parse_args() - - diff(args) - exit(0) - - -if __name__ == "__main__": - main() diff --git a/tests/cputestdata/cpu-gather.py b/tests/cputestdata/cpu-gather.py index a4a4050e42..f679fb9066 100755 --- a/tests/cputestdata/cpu-gather.py +++ b/tests/cputestdata/cpu-gather.py @@ -8,6 +8,18 @@ import re import struct import subprocess import sys +import xml.etree.ElementTree + + +_KEYS = { + "cpuid": ["eax_in", "ecx_in"], + "msr": ["index"], +} + +_REGS = { + "cpuid": ["eax", "ebx", "ecx", "edx"], + "msr": ["eax", "edx"], +} def gather_name(args): @@ -304,6 +316,118 @@ def parse(args, data): args.json_files = getattr(args, "json_files", list()) + [filename_json] +def checkFeature(cpuData, feature): + for key in ["type"] + _KEYS.get(feature["type"], list()): + if feature[key] not in cpuData: + return False + cpuData = cpuData[feature[key]] + + for reg in _REGS.get(feature["type"], list()): + if feature[reg] > 0 and feature[reg] == feature[reg] & cpuData[reg]: + return True + return False + + +def addFeature(cpuData, feature): + for key in ["type"] + _KEYS.get(feature["type"], list()): + if feature[key] not in cpuData: + cpuData[feature[key]] = dict() + cpuData = cpuData[feature[key]] + + for reg in _REGS.get(feature["type"], list()): + cpuData[reg] = cpuData.get(reg, 0) | feature[reg] + + +def parseQemu(path, features): + cpuData = {} + with open(path, "r") as f: + data, pos = json.JSONDecoder().raw_decode(f.read()) + + for (prop, val) in data["return"]["model"]["props"].items(): + if val and prop in features: + addFeature(cpuData, features[prop]) + + return cpuData + + +def parseCPUData(path): + cpuData = dict() + for f in xml.etree.ElementTree.parse(path).getroot(): + if f.tag not in ("cpuid", "msr"): + continue + + feature = {"type": f.tag} + for reg in _KEYS[f.tag] + _REGS[f.tag]: + feature[reg] = int(f.attrib.get(reg, "0"), 0) + addFeature(cpuData, feature) + return cpuData + + +def parseMap(): + path = os.path.dirname(sys.argv[0]) + path = os.path.join(path, "..", "..", "src", "cpu_map", "x86_features.xml") + + cpuMap = dict() + for f in xml.etree.ElementTree.parse(path).getroot().iter("feature"): + if f[0].tag not in ("cpuid", "msr"): + continue + + feature = {"type": f[0].tag} + for reg in _KEYS[f[0].tag] + _REGS[f[0].tag]: + feature[reg] = int(f[0].attrib.get(reg, "0"), 0) + cpuMap[f.attrib["name"]] = feature + return cpuMap + + +def formatCPUData(cpuData, path, comment): + print(path) + with open(path, "w") as f: + f.write("<!-- " + comment + " -->\n") + f.write("<cpudata arch='x86'>\n") + + cpuid = cpuData["cpuid"] + for eax_in in sorted(cpuid.keys()): + for ecx_in in sorted(cpuid[eax_in].keys()): + leaf = cpuid[eax_in][ecx_in] + line = (" <cpuid eax_in='0x%08x' ecx_in='0x%02x' " + "eax='0x%08x' ebx='0x%08x' " + "ecx='0x%08x' edx='0x%08x'/>\n") + f.write(line % ( + eax_in, ecx_in, + leaf["eax"], leaf["ebx"], leaf["ecx"], leaf["edx"])) + + if "msr" in cpuData: + msr = cpuData["msr"] + for index in sorted(msr.keys()): + f.write(" <msr index='0x%x' edx='0x%08x' eax='0x%08x'/>\n" % + (index, msr[index]['edx'], msr[index]['eax'])) + + f.write("</cpudata>\n") + + +def diff(args): + cpuMap = parseMap() + + for jsonFile in args.json_files: + cpuDataFile = jsonFile.replace(".json", ".xml") + enabledFile = jsonFile.replace(".json", "-enabled.xml") + disabledFile = jsonFile.replace(".json", "-disabled.xml") + + cpuData = parseCPUData(cpuDataFile) + qemu = parseQemu(jsonFile, cpuMap) + + enabled = dict() + disabled = dict() + for feature in cpuMap.values(): + if checkFeature(qemu, feature): + addFeature(enabled, feature) + elif checkFeature(cpuData, feature): + addFeature(disabled, feature) + + formatCPUData(enabled, enabledFile, "Features enabled by QEMU") + formatCPUData(disabled, disabledFile, "Features disabled by QEMU") + + def main(): parser = argparse.ArgumentParser(description="Gather cpu test data") parser.add_argument( @@ -332,6 +456,15 @@ def main(): subparsers.add_parser( "full", help="Equivalent to `cpu-gather gather | cpu-gather parse`.") + diffparser = subparsers.add_parser( + "diff", + help="Diff json description of CPU model against known features.") + diffparser.add_argument( + "json_files", + nargs="+", + metavar="FILE", + type=os.path.realpath, + help="Path to one or more json CPU model descriptions.") args = parser.parse_args() @@ -359,9 +492,7 @@ def main(): parse(args, data) if "json_files" in args: - cmd = ["./cpu-cpuid.py", "diff"] - cmd.extend(args.json_files) - subprocess.check_call(cmd, universal_newlines=True) + diff(args) if __name__ == "__main__": -- 2.26.2
On Mon, Jan 04, 2021 at 12:30:10 +0100, Tim Wiederhake wrote:
This series merges the functionality of cpu-cpuid.py into cpu-gather.py and completes the transition.
For background, see https://www.redhat.com/archives/libvir-list/2020-December/msg00706.html
Tim Wiederhake (9): cpu-cpuid: Use argparse to parse arguments cpu-cpuid: Remove xmltodict usage in parseMap cpu-cpuid: Remove xmltodict usage in parseCPU cpu-cpuid: Merge addFeature functions cpu-cpuid: Merge checkFeature functions cpu-cpuid: Deduplicate register list cpu-gather: Use actions instead of flags for action argument cpu-gather: Factor out call to cpu-cpuid.py cpu-gather: Merge cpu-cpuid.py
tests/cputestdata/cpu-cpuid.py | 229 -------------------------------- tests/cputestdata/cpu-gather.py | 178 ++++++++++++++++++++++--- 2 files changed, 156 insertions(+), 251 deletions(-) delete mode 100755 tests/cputestdata/cpu-cpuid.py
Reviewed-by: Jiri Denemark <jdenemar@redhat.com> and pushed, thanks.
participants (2)
-
Jiri Denemark -
Tim Wiederhake