'xmltodict' is a Python module that is not installed by default. Replace it, so the dependencies of cpu-gather.py do not change when both scripts are merged. Signed-off-by: Tim Wiederhake <twiederh(a)redhat.com&gt; --- tests/cputestdata/cpu-cpuid.py | 35 +++++++++++----------------------- 1 file changed, 11 insertions(+), 24 deletions(-) diff --git a/tests/cputestdata/cpu-cpuid.py b/tests/cputestdata/cpu-cpuid.py index 8e06baea85..d570884db6 100755 --- a/tests/cputestdata/cpu-cpuid.py +++ b/tests/cputestdata/cpu-cpuid.py @@ -4,7 +4,6 @@ import argparse import os import sys import json -import xmltodict import xml.etree.ElementTree @@ -105,31 +104,19 @@ def parseQemu(path, features): def parseCPUData(path): - cpuData = {} - with open(path, "rb") as f: - data = xmltodict.parse(f) - - for leaf in data["cpudata"]["cpuid"]: - feature = {"type": "cpuid"} - feature["eax_in"] = int(leaf["@eax_in"], 0) - feature["ecx_in"] = int(leaf["@ecx_in"], 0) - for reg in ["eax", "ebx", "ecx", "edx"]: - feature[reg] = int(leaf["@" + reg], 0) + cpuData = dict() + for f in xml.etree.ElementTree.parse(path).getroot(): + if f.tag == "cpuid": + reg_list = ["eax_in", "ecx_in", "eax", "ebx", "ecx", "edx"] + elif f.tag == "msr": + reg_list = ["index", "eax", "edx"] + else: + continue + feature = {"type": f.tag} + for reg in reg_list: + feature[reg] = int(f.attrib.get(reg, "0"), 0) addFeature(cpuData, feature) - - if "msr" in data["cpudata"]: - if not isinstance(data["cpudata"]["msr"], list): - data["cpudata"]["msr"] = [data["cpudata"]["msr"]] - - for msr in data["cpudata"]["msr"]: - feature = {"type": "msr"} - feature["index"] = int(msr["@index"], 0) - feature["edx"] = int(msr["@edx"], 0) - feature["eax"] = int(msr["@eax"], 0) - - addFeature(cpuData, feature) - return cpuData -- 2.26.2

Prepare to deduplicate the list of relevant registers for cpuid and msr information. Signed-off-by: Tim Wiederhake <twiederh(a)redhat.com&gt; --- tests/cputestdata/cpu-cpuid.py | 60 ++++++++++------------------------ 1 file changed, 18 insertions(+), 42 deletions(-) diff --git a/tests/cputestdata/cpu-cpuid.py b/tests/cputestdata/cpu-cpuid.py index c9948da6f8..e90165e563 100755 --- a/tests/cputestdata/cpu-cpuid.py +++ b/tests/cputestdata/cpu-cpuid.py @@ -7,51 +7,27 @@ import json import xml.etree.ElementTree -def checkCPUIDFeature(cpuData, feature): - eax_in = feature["eax_in"] - ecx_in = feature["ecx_in"] - eax = feature["eax"] - ebx = feature["ebx"] - ecx = feature["ecx"] - edx = feature["edx"] - - if "cpuid" not in cpuData: - return False - - cpuid = cpuData["cpuid"] - if eax_in not in cpuid or ecx_in not in cpuid[eax_in]: - return False - - leaf = cpuid[eax_in][ecx_in] - return ((eax > 0 and leaf["eax"] & eax == eax) or - (ebx > 0 and leaf["ebx"] & ebx == ebx) or - (ecx > 0 and leaf["ecx"] & ecx == ecx) or - (edx > 0 and leaf["edx"] & edx == edx)) - - -def checkMSRFeature(cpuData, feature): - index = feature["index"] - edx = feature["edx"] - eax = feature["eax"] - - if "msr" not in cpuData: - return False - - msr = cpuData["msr"] - if index not in msr: - return False - - msr = msr[index] - return ((edx > 0 and msr["edx"] & edx == edx) or - (eax > 0 and msr["eax"] & eax == eax)) - - def checkFeature(cpuData, feature): if feature["type"] == "cpuid": - return checkCPUIDFeature(cpuData, feature) + # cpuData["cpuid"][eax_in][ecx_in] = {eax:, ebx:, ecx:, edx:} + keyList = ["type", "eax_in", "ecx_in"] + regList = ["eax", "ebx", "ecx", "edx"] + elif feature["type"] == "msr": + # cpuData["msr"][index] = {eax:, edx:} + keyList = ["type", "index"] + regList = ["eax", "edx"] + else: + return False + + for key in keyList: + if feature[key] not in cpuData: + return False + cpuData = cpuData[feature[key]] - if feature["type"] == "msr": - return checkMSRFeature(cpuData, feature) + for reg in regList: + if feature[reg] > 0 and feature[reg] == feature[reg] & cpuData[reg]: + return True + return False def addFeature(cpuData, feature): -- 2.26.2

This allows for the functionality of cpu-cpuid.py script to be integrated more naturally in a later patch. Changes the way this script should be called: cpu-gather.py -> cpu-gather.py cpu-gather.py --gather -> cpu-gather.py gather cpu-gather.py --parse -> cpu-gather.py parse cpu-gather.py --gather --parse -> cpu-gather.py full Signed-off-by: Tim Wiederhake <twiederh(a)redhat.com&gt; --- tests/cputestdata/cpu-gather.py | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/tests/cputestdata/cpu-gather.py b/tests/cputestdata/cpu-gather.py index a01c615504..5ca19e5f2b 100755 --- a/tests/cputestdata/cpu-gather.py +++ b/tests/cputestdata/cpu-gather.py @@ -324,21 +324,22 @@ def main(): help="Path to qemu. " "If unset, will try '/usr/bin/qemu-system-x86_64', " "'/usr/bin/qemu-kvm', and '/usr/libexec/qemu-kvm'.") - parser.add_argument( - "--gather", - action="store_true", - help="Acquire data on target system. This is the default. " - "If '--parse' is not set, outputs data on stdout.") - parser.add_argument( - "--parse", - action="store_true", - help="Parse data for libvirt use. " - "If '--gather' is not set, expects input on stdin.") + subparsers = parser.add_subparsers(dest="action") + subparsers.add_parser( + "gather", + help="Acquire data on target system and outputs to stdout. " + "This is the default. ") + subparsers.add_parser( + "parse", + help="Reads data from stdin and parses data for libvirt use.") + subparsers.add_parser( + "full", + help="Equivalent to `cpu-gather gather | cpu-gather parse`.") args = parser.parse_args() - if not args.gather and not args.parse: - args.gather = True + if not args.action: + args.action = "gather" if not args.path_to_qemu: args.path_to_qemu = "qemu-system-x86_64" @@ -350,13 +351,13 @@ def main(): if os.path.isfile(f): args.path_to_qemu = f - if args.gather: + if args.action in ["gather", "full"]: data = gather(args) - if not args.parse: + if args.action == "gather": json.dump(data, sys.stdout, indent=2) - if args.parse: - if not args.gather: + if args.action in ["parse", "full"]: + if args.action == "parse": data = json.load(sys.stdin) parse(data) -- 2.26.2

Old usage: cpu-cpuid.py diff FILE... New usage: cpu-gather.py diff FILE... Signed-off-by: Tim Wiederhake <twiederh(a)redhat.com&gt; --- tests/cputestdata/cpu-cpuid.py | 152 -------------------------------- tests/cputestdata/cpu-gather.py | 137 +++++++++++++++++++++++++++- 2 files changed, 134 insertions(+), 155 deletions(-) delete mode 100755 tests/cputestdata/cpu-cpuid.py diff --git a/tests/cputestdata/cpu-cpuid.py b/tests/cputestdata/cpu-cpuid.py deleted file mode 100755 index 3e852d5d55..0000000000 --- a/tests/cputestdata/cpu-cpuid.py +++ /dev/null @@ -1,152 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import os -import sys -import json -import xml.etree.ElementTree - - -_KEYS = { - "cpuid": ["eax_in", "ecx_in"], - "msr": ["index"], -} - -_REGS = { - "cpuid": ["eax", "ebx", "ecx", "edx"], - "msr": ["eax", "edx"], -} - - -def checkFeature(cpuData, feature): - for key in ["type"] + _KEYS.get(feature["type"], list()): - if feature[key] not in cpuData: - return False - cpuData = cpuData[feature[key]] - - for reg in _REGS.get(feature["type"], list()): - if feature[reg] > 0 and feature[reg] == feature[reg] & cpuData[reg]: - return True - return False - - -def addFeature(cpuData, feature): - for key in ["type"] + _KEYS.get(feature["type"], list()): - if feature[key] not in cpuData: - cpuData[feature[key]] = dict() - cpuData = cpuData[feature[key]] - - for reg in _REGS.get(feature["type"], list()): - cpuData[reg] = cpuData.get(reg, 0) | feature[reg] - - -def parseQemu(path, features): - cpuData = {} - with open(path, "r") as f: - data, pos = json.JSONDecoder().raw_decode(f.read()) - - for (prop, val) in data["return"]["model"]["props"].items(): - if val and prop in features: - addFeature(cpuData, features[prop]) - - return cpuData - - -def parseCPUData(path): - cpuData = dict() - for f in xml.etree.ElementTree.parse(path).getroot(): - if f.tag not in ("cpuid", "msr"): - continue - - feature = {"type": f.tag} - for reg in _KEYS[f.tag] + _REGS[f.tag]: - feature[reg] = int(f.attrib.get(reg, "0"), 0) - addFeature(cpuData, feature) - return cpuData - - -def parseMap(): - path = os.path.dirname(sys.argv[0]) - path = os.path.join(path, "..", "..", "src", "cpu_map", "x86_features.xml") - - cpuMap = dict() - for f in xml.etree.ElementTree.parse(path).getroot().iter("feature"): - if f[0].tag not in ("cpuid", "msr"): - continue - - feature = {"type": f[0].tag} - for reg in _KEYS[f[0].tag] + _REGS[f[0].tag]: - feature[reg] = int(f[0].attrib.get(reg, "0"), 0) - cpuMap[f.attrib["name"]] = feature - return cpuMap - - -def formatCPUData(cpuData, path, comment): - print(path) - with open(path, "w") as f: - f.write("<!-- " + comment + " -->\n") - f.write("<cpudata arch='x86'>\n") - - cpuid = cpuData["cpuid"] - for eax_in in sorted(cpuid.keys()): - for ecx_in in sorted(cpuid[eax_in].keys()): - leaf = cpuid[eax_in][ecx_in] - line = (" <cpuid eax_in='0x%08x' ecx_in='0x%02x' " - "eax='0x%08x' ebx='0x%08x' " - "ecx='0x%08x' edx='0x%08x'/>\n") - f.write(line % ( - eax_in, ecx_in, - leaf["eax"], leaf["ebx"], leaf["ecx"], leaf["edx"])) - - if "msr" in cpuData: - msr = cpuData["msr"] - for index in sorted(msr.keys()): - f.write(" <msr index='0x%x' edx='0x%08x' eax='0x%08x'/>\n" % - (index, msr[index]['edx'], msr[index]['eax'])) - - f.write("</cpudata>\n") - - -def diff(args): - cpuMap = parseMap() - - for jsonFile in args.json_files: - cpuDataFile = jsonFile.replace(".json", ".xml") - enabledFile = jsonFile.replace(".json", "-enabled.xml") - disabledFile = jsonFile.replace(".json", "-disabled.xml") - - cpuData = parseCPUData(cpuDataFile) - qemu = parseQemu(jsonFile, cpuMap) - - enabled = dict() - disabled = dict() - for feature in cpuMap.values(): - if checkFeature(qemu, feature): - addFeature(enabled, feature) - elif checkFeature(cpuData, feature): - addFeature(disabled, feature) - - formatCPUData(enabled, enabledFile, "Features enabled by QEMU") - formatCPUData(disabled, disabledFile, "Features disabled by QEMU") - - -def main(): - parser = argparse.ArgumentParser(description="Diff cpuid results") - subparsers = parser.add_subparsers(dest="action", required=True) - diffparser = subparsers.add_parser( - "diff", - help="Diff json description of CPU model against known features.") - diffparser.add_argument( - "json_files", - nargs="+", - metavar="FILE", - type=os.path.realpath, - help="Path to one or more json CPU model descriptions.") - args = parser.parse_args() - - diff(args) - exit(0) - - -if __name__ == "__main__": - main() diff --git a/tests/cputestdata/cpu-gather.py b/tests/cputestdata/cpu-gather.py index a4a4050e42..f679fb9066 100755 --- a/tests/cputestdata/cpu-gather.py +++ b/tests/cputestdata/cpu-gather.py @@ -8,6 +8,18 @@ import re import struct import subprocess import sys +import xml.etree.ElementTree + + +_KEYS = { + "cpuid": ["eax_in", "ecx_in"], + "msr": ["index"], +} + +_REGS = { + "cpuid": ["eax", "ebx", "ecx", "edx"], + "msr": ["eax", "edx"], +} def gather_name(args): @@ -304,6 +316,118 @@ def parse(args, data): args.json_files = getattr(args, "json_files", list()) + [filename_json] +def checkFeature(cpuData, feature): + for key in ["type"] + _KEYS.get(feature["type"], list()): + if feature[key] not in cpuData: + return False + cpuData = cpuData[feature[key]] + + for reg in _REGS.get(feature["type"], list()): + if feature[reg] > 0 and feature[reg] == feature[reg] & cpuData[reg]: + return True + return False + + +def addFeature(cpuData, feature): + for key in ["type"] + _KEYS.get(feature["type"], list()): + if feature[key] not in cpuData: + cpuData[feature[key]] = dict() + cpuData = cpuData[feature[key]] + + for reg in _REGS.get(feature["type"], list()): + cpuData[reg] = cpuData.get(reg, 0) | feature[reg] + + +def parseQemu(path, features): + cpuData = {} + with open(path, "r") as f: + data, pos = json.JSONDecoder().raw_decode(f.read()) + + for (prop, val) in data["return"]["model"]["props"].items(): + if val and prop in features: + addFeature(cpuData, features[prop]) + + return cpuData + + +def parseCPUData(path): + cpuData = dict() + for f in xml.etree.ElementTree.parse(path).getroot(): + if f.tag not in ("cpuid", "msr"): + continue + + feature = {"type": f.tag} + for reg in _KEYS[f.tag] + _REGS[f.tag]: + feature[reg] = int(f.attrib.get(reg, "0"), 0) + addFeature(cpuData, feature) + return cpuData + + +def parseMap(): + path = os.path.dirname(sys.argv[0]) + path = os.path.join(path, "..", "..", "src", "cpu_map", "x86_features.xml") + + cpuMap = dict() + for f in xml.etree.ElementTree.parse(path).getroot().iter("feature"): + if f[0].tag not in ("cpuid", "msr"): + continue + + feature = {"type": f[0].tag} + for reg in _KEYS[f[0].tag] + _REGS[f[0].tag]: + feature[reg] = int(f[0].attrib.get(reg, "0"), 0) + cpuMap[f.attrib["name"]] = feature + return cpuMap + + +def formatCPUData(cpuData, path, comment): + print(path) + with open(path, "w") as f: + f.write("<!-- " + comment + " -->\n") + f.write("<cpudata arch='x86'>\n") + + cpuid = cpuData["cpuid"] + for eax_in in sorted(cpuid.keys()): + for ecx_in in sorted(cpuid[eax_in].keys()): + leaf = cpuid[eax_in][ecx_in] + line = (" <cpuid eax_in='0x%08x' ecx_in='0x%02x' " + "eax='0x%08x' ebx='0x%08x' " + "ecx='0x%08x' edx='0x%08x'/>\n") + f.write(line % ( + eax_in, ecx_in, + leaf["eax"], leaf["ebx"], leaf["ecx"], leaf["edx"])) + + if "msr" in cpuData: + msr = cpuData["msr"] + for index in sorted(msr.keys()): + f.write(" <msr index='0x%x' edx='0x%08x' eax='0x%08x'/>\n" % + (index, msr[index]['edx'], msr[index]['eax'])) + + f.write("</cpudata>\n") + + +def diff(args): + cpuMap = parseMap() + + for jsonFile in args.json_files: + cpuDataFile = jsonFile.replace(".json", ".xml") + enabledFile = jsonFile.replace(".json", "-enabled.xml") + disabledFile = jsonFile.replace(".json", "-disabled.xml") + + cpuData = parseCPUData(cpuDataFile) + qemu = parseQemu(jsonFile, cpuMap) + + enabled = dict() + disabled = dict() + for feature in cpuMap.values(): + if checkFeature(qemu, feature): + addFeature(enabled, feature) + elif checkFeature(cpuData, feature): + addFeature(disabled, feature) + + formatCPUData(enabled, enabledFile, "Features enabled by QEMU") + formatCPUData(disabled, disabledFile, "Features disabled by QEMU") + + def main(): parser = argparse.ArgumentParser(description="Gather cpu test data") parser.add_argument( @@ -332,6 +456,15 @@ def main(): subparsers.add_parser( "full", help="Equivalent to `cpu-gather gather | cpu-gather parse`.") + diffparser = subparsers.add_parser( + "diff", + help="Diff json description of CPU model against known features.") + diffparser.add_argument( + "json_files", + nargs="+", + metavar="FILE", + type=os.path.realpath, + help="Path to one or more json CPU model descriptions.") args = parser.parse_args() @@ -359,9 +492,7 @@ def main(): parse(args, data) if "json_files" in args: - cmd = ["./cpu-cpuid.py", "diff"] - cmd.extend(args.json_files) - subprocess.check_call(cmd, universal_newlines=True) + diff(args) if __name__ == "__main__": -- 2.26.2