[RFCv3 01/25] scripts: Add a tool to generate xml parse/format functions

This tool is used to generate parsexml/formatbuf/clear functions. It is based on libclang and its python-binding. Some directives (such as genparse, xmlattr, etc.) need to be added on the declarations of structs to direct the tool. Signed-off-by: Shi Lei <shi_lei(a)massclouds.com&gt; --- po/POTFILES.in | 1 + scripts/xmlgen/directive.py | 1192 +++++++++++++++++++++++++++++++++++ scripts/xmlgen/go | 29 + scripts/xmlgen/main.py | 534 ++++++++++++++++ scripts/xmlgen/utils.py | 121 ++++ 5 files changed, 1877 insertions(+) create mode 100644 scripts/xmlgen/directive.py create mode 100755 scripts/xmlgen/go create mode 100755 scripts/xmlgen/main.py create mode 100644 scripts/xmlgen/utils.py diff --git a/po/POTFILES.in b/po/POTFILES.in index 413783ee..9740bb2b 100644 --- a/po/POTFILES.in +++ b/po/POTFILES.in @@ -5,6 +5,7 @@ @BUILDDIR(a)src/admin/admin_server_dispatch_stubs.h @BUILDDIR(a)src/remote/remote_client_bodies.h @BUILDDIR(a)src/remote/remote_daemon_dispatch_stubs.h +@SRCDIR(a)scripts/xmlgen/directive.py @SRCDIR(a)src/access/viraccessdriverpolkit.c @SRCDIR(a)src/access/viraccessmanager.c @SRCDIR(a)src/admin/admin_server.c diff --git a/scripts/xmlgen/directive.py b/scripts/xmlgen/directive.py new file mode 100644 index 00000000..cc8fb5aa --- /dev/null +++ b/scripts/xmlgen/directive.py @@ -0,0 +1,1192 @@ +# +# Copyright (C) 2021 Shandong Massclouds Co.,Ltd. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see +# <http://www.gnu.org/licenses/>;. +# + +import json +from collections import OrderedDict +from utils import singleton, dedup, Block, Terms, render + +BUILTIN_TYPES = { + 'String': {}, + 'Bool': { + 'conv': 'virStrToBoolYesNo(${mdvar}, &def->${name})' + }, + 'BoolYesNo': { + 'conv': 'virStrToBoolYesNo(${mdvar}, &def->${name})' + }, + 'BoolOnOff': { + 'conv': 'virStrToBoolOnOff(${mdvar}, &def->${name})' + }, + 'BoolTrueFalse': { + 'conv': 'virStrToBoolTrueFalse(${mdvar}, &def->${name})' + }, + 'Chars': { + 'conv': 'virStrcpyStatic(def->${name}, ${name}Str)' + }, + 'UChars': { + 'conv': 'virStrcpyStatic((char *)def->${name}, ${mdvar})' + }, + 'Int': { + 'fmt': '%d', + 'conv': 'virStrToLong_i(${mdvar}, NULL, 0, &def->${name})' + }, + 'UInt': { + 'fmt': '%u', + 'conv': 'virStrToLong_uip(${mdvar}, NULL, 0, &def->${name})' + }, + 'ULong': { + 'fmt': '%lu', + 'conv': 'virStrToLong_ulp(${mdvar}, NULL, 0, &def->${name})' + }, + 'ULongLong': { + 'fmt': '%llu', + 'conv': 'virStrToLong_ullp(${mdvar}, NULL, 0, &def->${name})' + }, + 'U8': { + 'fmt': '%u', + 'conv': 'virStrToLong_u8p(${mdvar}, NULL, 0, &def->${name})' + }, + 'U32': { + 'fmt': '%u', + 'conv': 'virStrToLong_uip(${mdvar}, NULL, 0, &def->${name})' + }, + 'Time': { + 'conv': 'virStrToTime(${mdvar}, &def->${name})' + }, +} + + +@singleton +class TypeTable(OrderedDict): + def __init__(self): + OrderedDict.__init__(self) + for name, kvs in BUILTIN_TYPES.items(): + kvs['name'] = name + kvs['meta'] = 'Builtin' + self[name] = kvs + + def register(self, kvs): + name = kvs['name'] + if name not in self: + self[name] = kvs + return name + + def get(self, name): + if name in self: + return self[name] + return {'meta': 'Struct', 'name': name, 'external': True} + + def check(self, name): + return name in self + + +T_NAMESPACE_PARSE = [ + 'if (xmlopt)', + ' def->ns = xmlopt->ns;', + 'if (def->ns.parse) {', + ' if (virXMLNamespaceRegister(ctxt, &def->ns) < 0)', + ' goto error;', + ' if ((def->ns.parse)(ctxt, &def->namespaceData) < 0)', + ' goto error;', + '}' +] + +T_NAMESPACE_FORMAT_BEGIN = [ + 'if (def->namespaceData && def->ns.format)', + ' virXMLNamespaceFormatNS(buf, &def->ns);' +] + +T_NAMESPACE_FORMAT_END = [ + 'if (def->namespaceData && def->ns.format) {', + ' if ((def->ns.format)(buf, def->namespaceData) < 0)', + ' return -1;', + '}' +] + + +def funcSignature(rtype, name, args, semicolon=''): + alignment = ' ' * (len(name) + 1) + connector = ',\n' + alignment + + ret = [] + ret.append(rtype) + ret.append('%s(%s)%s' % (name, connector.join(args), semicolon)) + return Block(ret) + + +def counterName(member): + if isinstance(member['array'], bool): + return 'n' + member['name'] + return member['array'] + + +def loop(count, block, condition=None): + assert isinstance(block, list) and len(block) > 0 + lp = ' {' if len(block) > 1 else '' + condition = condition if condition else count + + ret = Block() + ret.format('if (${condition}) {', condition=condition) + ret.format(' size_t i;') + ret.format(' for (i = 0; i < ${count}; i++)${lp}', count=count, lp=lp) + ret.mapfmt(' ${_each_line_}', block) + ret.format(' }' if lp else None) + ret.format('}') + return ret + + +def ispointer(member): + mtype = TypeTable().get(member['type']) + return member['pointer'] and not mtype.get('external') + + +def clearMember(member, pre_name=None): + if not member.get('xmlattr') and not member.get('xmlelem') \ + and not member.get('xmlgroup') and not member.get('xmlswitch'): + return None + + # Callback for make_switch + def _switch_clear_cb(child, switch, _): + return clearMember(child, switch['name']) + + if member.get('xmlswitch'): + return makeSwitch(member, _switch_clear_cb, None) + + mtype = TypeTable().get(member['type']) + + if pre_name: + ref = 'def->%s.%s' % (pre_name, member['name']) + else: + ref = 'def->%s' % member['name'] + + if member.get('array'): + ref += '[i]' + + ret = Block() + if mtype['meta'] == 'Struct': + if ispointer(member): + ret.format('${tname}Clear(${ref});', tname=mtype['name'], ref=ref) + ret.format('g_free(${ref});', ref=ref) + ret.format('${ref} = NULL;', ref=ref) + else: + ret.format('${tname}Clear(&${ref});', tname=mtype['name'], ref=ref) + elif mtype['name'] == 'String': + ret.format('g_free(${ref});', ref=ref) + ret.format('${ref} = NULL;', ref=ref) + elif mtype['name'] in ['Chars', 'UChars']: + ret.format('memset(${ref}, 0, sizeof(${ref}));', ref=ref) + elif not member.get('array'): + ret.format('${ref} = 0;', ref=ref) + + if member.get('specified'): + assert not member.get('array'), "'specified' can't come with 'array'." + sname, managed = member['specified'] + if managed: + ret.format('def->${sname} = false;', sname=sname) + + if member.get('array') and len(ret) > 0: + count = 'def->' + counterName(member) + ret = loop(count, ret) + ret.format('g_free(def->${name});', name=member['name']) + ret.format('def->${name} = NULL;', name=member['name']) + ret.format('${count} = 0;', count=count) + + return ret + + +def makeClearFunc(writer, atype): + if 'genparse' not in atype: + return + + args = ['%s *def' % atype['name']] + signature = funcSignature('void', atype['name'] + 'Clear', args) + writer.write(atype, 'clearfunc', '.h', signature.output() + ';') + + ret = Block() + ret.extend(signature) + ret.format('{') + ret.format(' if (!def)') + ret.format(' return;') + ret.newline() + + delay_members = [] + for member in atype['members']: + if member.get('delay_clear'): + delay_members.append(member) + continue + + code = clearMember(member) + if code: + ret.mapfmt(' ${_each_line_}', code) + ret.newline() + + for member in delay_members: + code = clearMember(member) + if code: + ret.mapfmt(' ${_each_line_}', code) + ret.newline() + + if 'namespace' in atype: + ret.format(' if (def->namespaceData && def->ns.free)') + ret.format(' (def->ns.free)(def->namespaceData);') + else: + ret.pop() # Remove last newline + + ret.format('}') + writer.write(atype, 'clearfunc', '.c', ret.output()) + + +def reportInvalid(tname, mdvar): + ret = Block() + ret.append('virReportError(VIR_ERR_XML_ERROR,') + ret.append(" _(\"Invalid '%s' setting '%s' in '%s'\"),") + ret.format(' "${tname}", ${mdvar}, instname);', + tname=tname, mdvar=mdvar) + return ret + + +def reportMissing(tname): + ret = Block() + ret.append('virReportError(VIR_ERR_XML_ERROR,') + ret.append(" _(\"Missing '%s' setting in '%s'\"),") + ret.format(' "${tname}", instname);', tname=tname) + return ret + + +def checkError(kind, cond, tname, mdvar): + if kind == 'Invalid': + report = reportInvalid(tname, mdvar) + elif kind == 'Missing': + report = reportMissing(tname) + else: + assert False, '%s is unsupported.' % kind + + ret = Block() + ret.format('if (${cond}) {', cond=cond) + ret.mapfmt(' ${_each_line_}', report) + ret.append(' goto error;') + ret.append('}') + return ret + + +def parseMember(member, parent, tmpvars, pre_name=None): + mtype = TypeTable().get(member['type']) + + # + # Helper functions + # + def _middle_var(member): + ret = member['name'].replace('.', '_') + if member.get('xmlgroup'): + ret = 'node' + elif mtype['name'] == 'String': + ret = 'def->' + ret + elif member.get('xmlattr') or \ + member.get('xmltext') or mtype['meta'] != 'Struct': + ret += 'Str' + else: + ret += 'Node' + return ret + + def _read_xml_func(mdvar, tname): + tagname = tname + if member.get('xmlattr'): + if '/' in tname: + funcname = 'virXMLChildPropString' + else: + funcname = 'virXMLPropString' + elif member.get('xmlelem'): + if not member.get('xmltext') and mtype['meta'] == 'Struct': + funcname = 'virXMLChildNode' + else: + funcname = 'virXMLChildNodeContent' + else: + return None + + return render('${mdvar} = ${funcname}(node, "${tname}");', + mdvar=mdvar, funcname=funcname, tname=tagname) + + def _assign_struct(name, member, mdvar): + ret = Block() + if ispointer(member): + ret.format('def->${name} = g_new0(typeof(*def->${name}), 1);', + name=name) + + func = mtype['name'] + 'ParseXML' + amp = '' if ispointer(member) else '&' + tmpl = '${func}(${mdvar}, ${amp}def->${name}, instname, ' \ + 'arg_parent, arg_opaque)' + fn = render(tmpl, func=func, mdvar=mdvar, amp=amp, name=name) + ret.extend(if_cond(fn + ' < 0', ['goto error;'])) + return ret + + def _assign_non_struct(name, member, mdvar): + if mtype['meta'] == 'Enum': + typename = mtype['name'] + if not typename.endswith('Type'): + typename += 'Type' + expr = render('(def->${name} = ${typename}FromString(${mdvar}))', + name=name, typename=typename, mdvar=mdvar) + if mtype['with_default']: + expr += ' <= 0' + else: + expr += ' < 0' + else: + builtin = BUILTIN_TYPES.get(mtype['name']) + assert builtin, mtype['name'] + tmpl = builtin.get('conv', None) + if tmpl: + expr = render(tmpl, name=name, mdvar=mdvar, tname=tname) + expr += ' < 0' + else: + return None + + return checkError('Invalid', expr, tname, mdvar) + + def _parse_array(name, member, tname): + num = 'n%sNodes' % Terms.upperInitial(tname) + tmpvars.append(num) + tmpvars.append('nodes') + count = counterName(member) + + if mtype['meta'] == 'Struct' and not member.get('xmltext'): + item = _assign_struct(name + '[i]', member, 'tnode') + else: + item = ['def->%s[i] = virXMLNodeContentString(tnode);' % name] + + ret = Block() + ret.format('${num} = virXMLChildNodeSet(node, "${tname}", &nodes);', + num=num, tname=tname) + ret.format('if (${num} > 0) {', num=num) + ret.format(' size_t i;') + ret.newline() + ret.format(' def->${name} = g_new0(typeof(*def->${name}), ${num});', + name=name, num=num) + ret.format(' for (i = 0; i < ${num}; i++) {', num=num) + ret.format(' xmlNodePtr tnode = nodes[i];') + ret.mapfmt(' ${_each_line_}', item) + ret.format(' }') + ret.format(' def->${count} = ${num};', count=count, num=num) + ret.format(' g_free(nodes);') + ret.format(' nodes = NULL;') + ret.format('} else if (${num} < 0) {', num=num) + ret.format(' virReportError(VIR_ERR_XML_ERROR,') + ret.format(' _("Invalid %s element found."),') + ret.format(' "${tname}");', tname=tname) + ret.format(' goto error;') + + if member.get('required'): + ret.append('} else {') + ret.mapfmt(' ${_each_line_}', reportMissing(tname)) + ret.append(' goto error;') + + ret.append('}') + return ret + + def getTag(member): + if member.get('xmlattr'): + return member['xmlattr'] + elif member.get('xmlelem'): + return member['xmlelem'] + else: + return None + + # + # Main routine + # + + if member.get('skipparse'): + return None + + tname = getTag(member) + if not tname and not member.get('xmlgroup'): + return None + + if pre_name: + name = pre_name + '.' + member['name'] + else: + name = member['name'] + + # For array member + if member.get('array'): + return _parse_array(name, member, tname) + + # For common member + mdvar = _middle_var(member) + if mdvar.endswith('Str') or mdvar.endswith('Node'): + tmpvars.append(mdvar) + + block = Block() + if tname: + block.append(_read_xml_func(mdvar, tname)) + if member.get('required'): + cond = render('${mdvar} == NULL', mdvar=mdvar) + block.extend(checkError('Missing', cond, tname, mdvar)) + + if mtype['meta'] == 'Struct': + assignment = _assign_struct(name, member, mdvar) + else: + assignment = _assign_non_struct(name, member, mdvar) + if not assignment: + return block + + if member.get('specified'): + sname, managed = member['specified'] + if managed: + assignment.format('def->${sname} = true;', sname=sname) + + if tname: + block.extend(if_cond(mdvar, assignment)) + else: + block.extend(assignment) + + return block + + +def makeParseFunc(writer, atype): + + # + # Helper functions + # + def _switch_parse_cb(child, switch, tmpvars): + return parseMember(child, atype, tmpvars, switch['name']) + + def _members_block(tmpvars): + block = Block() + for member in atype['members']: + if member.get('xmlswitch'): + code = makeSwitch(member, _switch_parse_cb, tmpvars) + else: + code = parseMember(member, atype, tmpvars) + + if code: + block.extend(code) + block.newline() + return block + + def _post_hook(tmpvars): + args = ['node', 'def', 'instname', 'parent', 'opaque'] + if 'namespace' in atype: + args.append('ctxt') + args.append('xmlopt') + + args.extend(tmpvars) + if 'nodes' in args: + args.remove('nodes') + + funcname = atype['name'] + 'ParseHook' + cond = '%s(%s) < 0' % (funcname, ', '.join(args)) + return if_cond(cond, ['goto error;']) + + def _handle_tmpvars(tmpvars): + args, heads, tails = [], [], [] + for var in tmpvars: + if var == 'nodes': + heads.append('xmlNodePtr *nodes = NULL;') + tails.append('g_free(nodes);') + tails.append('nodes = NULL;') + elif var.endswith('Str'): + heads.append('g_autofree char *%s = NULL;' % var) + args.append('const char *%s' % var) + elif var.endswith('Node'): + heads.append('xmlNodePtr %s = NULL;' % var) + args.append('xmlNodePtr %s' % var) + else: + assert var.endswith('Nodes') and var.startswith('n') + heads.append('int %s = 0;' % var) + args.append('int %s' % var) + + heads.append('void *arg_parent G_GNUC_UNUSED = def;') + heads.append('void *arg_opaque G_GNUC_UNUSED = opaque;') + heads.append('') + + heads.append('if (!def)') + heads.append(' goto error;') + tails.append('%sClear(def);' % atype['name']) + return args, heads, tails + + # + # Composite + # + if 'genparse' not in atype: + return + + typename = atype['name'] + funcname = typename + 'ParseXML' + + # Declare virXXXParseXML + args = Block([ + 'xmlNodePtr node', + '%s *def' % typename, + 'const char *instname', + 'void *parent', + 'void *opaque' + ]) + + if 'namespace' in atype: + args.append('xmlXPathContextPtr ctxt') + args.append('virNetworkXMLOption *xmlopt') + + signature = funcSignature('int', funcname, args) + writer.write(atype, 'parsefunc', '.h', signature.output() + ';') + + # Prepare for implementation + tmpvars = [] + parseblock = _members_block(tmpvars) + tmpvars = dedup(tmpvars) + tmpargs, headlines, cleanup = _handle_tmpvars(tmpvars) + posthook = _post_hook(tmpvars) + + # Declare virXXXParseXMLHook + macro = 'ENABLE_' + Terms.allcaps(typename) + '_PARSE_HOOK' + signature = funcSignature('int', typename + 'ParseHook', + args + tmpargs, ';') + + hook_decl = Block() + hook_decl.format('#ifdef ${macro}', macro=macro) + hook_decl.newline() + hook_decl.extend(signature) + hook_decl.newline() + hook_decl.format('#endif') + writer.write(atype, 'parsefunc', '.h', hook_decl.output()) + + setargs_args = ['xmlNodePtr node', 'void *parent', + 'void **pparent', 'void **popaque'] + setargs_signature = funcSignature('void', funcname + 'SetArgs', + setargs_args, ';') + + setargs_decl = Block() + setargs_decl.format('#ifdef ${macro}_SET_ARGS', macro=macro) + setargs_decl.newline() + setargs_decl.extend(setargs_signature) + setargs_decl.newline() + setargs_decl.format('#endif') + writer.write(atype, 'parsefunc', '.h', setargs_decl.output()) + + # Implement virXXXParseXML + args[2] += ' G_GNUC_UNUSED' + args[3] += ' G_GNUC_UNUSED' + args[4] += ' G_GNUC_UNUSED' + + impl = funcSignature('int', funcname, args) + impl.format('{') + impl.mapfmt(' ${_each_line_}', headlines) + + impl.newline() + impl.format('#ifdef ${macro}_SET_ARGS', macro=macro) + impl.format(' ${funcname}(node, parent, &arg_parent, &arg_opaque);', + funcname=(funcname + 'SetArgs')) + impl.format('#endif') + impl.newline() + + impl.mapfmt(' ${_each_line_}', parseblock) + + impl.format('#ifdef ${macro}', macro=macro) + impl.mapfmt(' ${_each_line_}', posthook) + impl.format('#endif') + impl.newline() + + if 'namespace' in atype: + impl.mapfmt(' ${_each_line_}', T_NAMESPACE_PARSE) + + impl.format(' return 0;') + impl.newline() + impl.format(' error:') + impl.mapfmt(' ${_each_line_}', cleanup) + impl.format(' return -1;') + impl.format('}') + + writer.write(atype, 'parsefunc', '.c', impl.output()) + + +def if_cond(condition, block): + assert isinstance(block, list) and len(block) > 0 + lp = ' {' if len(block) > 1 else '' + + ret = Block() + ret.format('if (${condition})${lp}', condition=condition, lp=lp) + ret.mapfmt(' ${_each_line_}', block) + ret.format('}' if lp else None) + return ret + + +def formatMember(member, parent): + mtype = TypeTable().get(member['type']) + + # + # Helper functions. + # + def _checkOnCondition(var): + ret = None + if ispointer(member): + ret = var + elif member.get('specified'): + sname, _ = member['specified'] + ret = 'def->' + sname + if ret.startswith('&'): + ret = ret[1:] + elif mtype['meta'] == 'Struct': + ret = '%sCheck(&%s, def, opaque)' % (mtype['name'], var) + elif mtype['meta'] == 'Enum': + if mtype['with_default']: + ret = var + ' > 0' + else: + ret = var + ' >= 0' + elif mtype['meta'] == 'Builtin': + if mtype['name'] in ['Chars', 'UChars']: + ret = var + '[0]' + else: + ret = var + + return ret + + def _format(layout, var): + if member.get('array'): + buf = 'buf' + elif member.get('wrap'): + buf = '&%s_%sBuf' % (member['wrap'], member['xmlattr']) + else: + buf = '&' + member['name'].replace('.', '_') + 'Buf' + + if mtype['meta'] == 'Struct': + if not ispointer(member): + var = '&' + var + + fname = mtype['name'] + 'FormatBuf' + cond = '%s(%s, "%s", %s, def, opaque) < 0' % \ + (fname, buf, layout, var) + return if_cond(cond, ['return -1;']) + elif mtype['meta'] == 'Enum': + name = mtype['name'] + if not name.endswith('Type'): + name += 'Type' + + ret = Block() + ret.format('const char *str = ${name}ToString(${var});', + name=name, var=var) + ret.format('if (!str) {') + ret.format(' virReportError(VIR_ERR_INTERNAL_ERROR,') + ret.format(' _("Unknown %s type %d"),') + ret.format(' "${tname}", ${var});', + tname=member['xmlattr'], var=var) + ret.format(' return -1;') + ret.format('}') + ret.format('virBufferAsprintf(${buf}, "${layout}", str);', + buf=buf, layout=layout) + return ret + elif mtype['name'] in ['Bool', 'BoolYesNo']: + var = '%s ? "yes" : "no"' % var + return ['virBufferEscapeString(%s, "%s", %s);' % (buf, layout, var)] + elif mtype['name'] == 'BoolOnOff': + var = '%s ? "on" : "off"' % var + return ['virBufferEscapeString(%s, "%s", %s);' % (buf, layout, var)] + elif mtype['name'] == 'BoolTrueFalse': + var = '%s ? "true" : "false"' % var + return ['virBufferEscapeString(%s, "%s", %s);' % (buf, layout, var)] + elif mtype['name'] == 'String': + return ['virBufferEscapeString(%s, "%s", %s);' % (buf, layout, var)] + elif mtype['name'] == 'Time': + return ['virTimeFormatBuf(%s, "%s", %s);' % (buf, layout, var)] + else: + return ['virBufferAsprintf(%s, "%s", %s);' % (buf, layout, var)] + + def _handleAttr(tagname, var): + if 'xmlattr' not in member: + return None + + fmt = '%s' + if mtype['meta'] == 'Builtin': + fmt = BUILTIN_TYPES[mtype['name']].get('fmt', '%s') + + if tagname: + layout = " %s='%s'" % (tagname, fmt) + else: + layout = "%s" + return _format(layout, var) + + def _handleElem(tagname, var): + if 'xmlattr' in member: + return None + + if member.get('xmltext') or mtype['meta'] != 'Struct': + layout = '<%s>%%s</%s>\\n' % (tagname, tagname) + else: + layout = tagname + + code = _format(layout, var) + return code + + # + # Main routine + # + name = member['name'] + if member.get('array'): + name = name + '[i]' + var = 'def->' + name + + ret = None + if 'xmlattr' in member: + tagname = member['xmlattr'] + ret = _handleAttr(tagname, var) + else: + tagname = member['xmlelem'] + ret = _handleElem(tagname, var) + + if not ret: + return None, None + + if member.get('wrap'): + hookbuf = '%s_%sBuf' % (member['wrap'], member['xmlattr']) + else: + hookbuf = member['name'].replace('.', '_') + 'Buf' + + checks = '!virBufferTouched(&%s)' % (hookbuf) + + # For array + if member.get('array'): + counter = 'def->' + counterName(member) + cond = checks + ' && ' + counter + code = Block() + code.format('virBufferInheritIndent(&${hbuf}, buf);', hbuf=hookbuf) + code.extend(loop(counter, ret, cond)) + code.format('virBufferAddBuffer(buf, &${hbuf});', hbuf=hookbuf) + return code, counter + + # Not array + check = _checkOnCondition(var) + if check and not member.get('required'): + if '&&' in check or '||' in check: + check = '(%s)' % check + checks = checks + ' && ' + check + + code = Block() + code.format('virBufferInheritIndent(&${hbuf}, buf);', hbuf=hookbuf) + code.extend(if_cond(checks, ret)) + code.format('virBufferAddBuffer(buf, &${hbuf});', hbuf=hookbuf) + return code, check + + +def makeSwitch(switch, callback, opaque=None): + assert switch.get('xmlswitch', None) and switch.get('switch_type', None) + + captype = Terms.allcaps(switch['switch_type']) + block = Block() + block.format('switch (def->${etype}) {', etype=switch['xmlswitch']) + block.newline() + for child in switch['members']: + value = captype + '_' + Terms.allcaps(child['name']) + block.format('case ${value}:', value=value) + block.mapfmt(' ${_each_line_}', callback(child, switch, opaque)) + block.format(' break;') + block.newline() + + enum = TypeTable().get(switch['switch_type']) + if enum['with_default']: + block.format('case ${captype}_NONE:', captype=captype) + + block.format('case ${captype}_LAST:', captype=captype) + block.format(' break;') + block.format('}') + return block + + +def makeFormatFunc(writer, atype): + if 'genformat' not in atype: + return + + # + # Helper functions and classes. + # + def _handleHeads(tmpvars): + tmpvars = dedup(tmpvars) + heads = Block() + heads.format('virTristateBool empty G_GNUC_UNUSED = ' + 'VIR_TRISTATE_BOOL_ABSENT;') + heads.format('virTristateBool shortcut G_GNUC_UNUSED = ' + 'VIR_TRISTATE_BOOL_ABSENT;') + heads.mapfmt( + 'g_auto(virBuffer) ${_each_line_} = VIR_BUFFER_INITIALIZER;', + tmpvars + ) + heads.newline() + + heads.format('if (!def || !buf)') + heads.format(' return 0;') + return '\n'.join(heads) + + def _handleHook(tmpvars, typename, kind=''): + funcname = typename + 'Format' + kind + 'Hook' + macro = 'ENABLE_' + Terms.allcaps(funcname) + + tmpvars = dedup(tmpvars) + args = Block(['def', 'parent', 'opaque']) + if not kind: + args.append('&empty') + args.append('&shortcut') + args.mapfmt('&${_each_line_}', tmpvars) + + heads = Block() + heads.format('if (%s(%s) < 0)' % (funcname, ', '.join(args))) + heads.format(' return -1;') + + args = Block([ + 'const %s *def' % typename, + 'const void *parent', + 'const void *opaque' + ]) + if not kind: + args.append('virTristateBool *empty') + args.append('virTristateBool *shortcut') + args.mapfmt('virBuffer *${_each_line_}', tmpvars) + + fmt_signature = funcSignature('int', funcname, args, ';') + + hook_decl = Block() + hook_decl.format('#ifdef ${macro}', macro=macro) + hook_decl.newline() + hook_decl.extend(fmt_signature) + hook_decl.newline() + hook_decl.format('#endif /* ${macro} */', macro=macro) + writer.write(atype, 'formatfunc', '.h', hook_decl.output()) + return '\n'.join(heads) + + def _handle_check_func(checks, kind=''): + # Declare virXXXCheck[Attr|Elem] + typename = atype['name'] + funcname = typename + 'Check' + kind + args = [ + 'const %s *def' % typename, + 'const void *parent', + 'void *opaque' + ] + signature = funcSignature('bool', funcname, args) + writer.write(atype, 'formatfunc', '.h', signature.output() + ';') + + # Implement virXXXFormat[Attr|Elem] + check = ' || '.join(checks) if checks else 'false' + + args[1] += ' G_GNUC_UNUSED' + args[2] += ' G_GNUC_UNUSED' + + macro = 'RESET_' + Terms.allcaps(funcname) + + impl = Block() + impl.format('#ifndef ${macro}', macro=macro) + impl.newline() + impl.extend(funcSignature('bool', funcname, args)) + impl.format('{') + impl.format(' if (!def)') + impl.format(' return false;') + impl.newline() + impl.format(' return ${check};', check=check) + impl.format('}') + impl.newline() + impl.format('#endif /* ${macro} */', macro=macro) + writer.write(atype, 'formatfunc', '.c', impl.output()) + + def _format_group(child, switch, kind): + kind = kind if kind else '' + prefix = (switch['name'] + '.') if switch else '' + mtype = TypeTable().get(child['type']) + funcname = '%sFormat%s' % (mtype['name'], kind) + var = 'def->%s%s' % (prefix, child['name']) + if not ispointer(child): + var = '&' + var + r_check = '%sCheck%s(%s, def, opaque)' % (mtype['name'], kind, var) + cond = '%s(buf, %s, def, opaque) < 0' % (funcname, var) + return if_cond(cond, ['return -1;']), r_check + + def _switch_format_cb(child, switch, kind): + return _format_group(child, switch, kind)[0] + + def _handle_wrap_attr(member): + member['wrap'], member['xmlattr'] = member['xmlattr'].split('/') + if member['xmlattr'] == '.': + member['xmlattr'] = '' + ret, check = formatMember(member, atype) + return ret, check + + def _switch_check_cb(child, switch, kind): + return ['ret = %s;' % _format_group(child, switch, kind)[1]] + + def _prepare_member(member, atype): + attr, attr_chk, elem, elem_chk = None, None, None, None + if member.get('xmlswitch'): + attr = makeSwitch(member, _switch_format_cb, 'Attr') + elem = makeSwitch(member, _switch_format_cb, 'Elem') + basename = atype['name'] + Terms.upperInitial(member['name']) + attr_chk = '%sCheckAttr(def, opaque)' % basename + elem_chk = '%sCheckElem(def, opaque)' % basename + + # Declare virXXX<UnionName>Check[Attr|Elem] for switch. + for kind in ['Attr', 'Elem']: + args = ['const %s *def' % atype['name'], 'void *opaque'] + funcname = basename + 'Check' + kind + decl = funcSignature('bool', funcname, args) + writer.write(atype, 'formatfunc', '.h', decl.output() + ';') + + # Implement virXXX<UnionName>Check[Attr|Elem] for switch. + checks = makeSwitch(member, _switch_check_cb, kind) + + args[1] += ' G_GNUC_UNUSED' + macro = 'RESET_' + Terms.allcaps(funcname) + + impl = Block() + impl.format('#ifndef ${macro}', macro=macro) + impl.newline() + impl.extend(funcSignature('bool', funcname, args)) + impl.format('{') + impl.format(' bool ret = false;') + impl.format(' if (!def)') + impl.format(' return false;') + impl.newline() + impl.mapfmt(' ${_each_line_}', checks) + impl.newline() + impl.format(' return ret;') + impl.format('}') + impl.newline() + impl.format('#endif /* ${macro} */', macro=macro) + writer.write(atype, 'formatfunc', '.c', impl.output()) + + elif member.get('xmlattr'): + if '/' in member['xmlattr']: + attr, attr_chk = _handle_wrap_attr(member) + else: + attr, attr_chk = formatMember(member, atype) + elif member.get('xmlelem'): + elem, elem_chk = formatMember(member, atype) + elif member.get('xmlgroup'): + attr, attr_chk = _format_group(member, None, 'Attr') + elem, elem_chk = _format_group(member, None, 'Elem') + return attr, attr_chk, elem, elem_chk + + class _WrapItem: + def __init__(self): + self.attrs = Block() + self.checks = [] + self.hchecks = Block() + self.optional = True + self.pos = 0 + + def _prepare(): + attrs = Block() + elems = Block() + attr_checks = [] + elem_checks = [] + attr_hook_vars = [] + elem_hook_vars = [] + wraps = OrderedDict() + for member in atype['members']: + attr, attr_chk, elem, elem_chk = _prepare_member(member, atype) + if member.get('wrap'): + item = wraps.setdefault(member['wrap'], _WrapItem()) + item.pos = len(elems) + if member['xmlattr']: + item.attrs.extend(attr) + item.attrs.newline() + item.checks.append(attr_chk) + if member.get('required'): + item.optional = False + wrap_var = '%s_%sBuf' % (member['wrap'], member['xmlattr']) + elem_hook_vars.append(wrap_var) + item.hchecks.format('virBufferUse(&${buf})', buf=wrap_var) + continue + + attrs.extend(attr) + attrs.newline(attr) + elems.extend(elem) + elems.newline(elem) + if attr_chk: + attr_checks.append(attr_chk) + if elem_chk: + elem_checks.append(elem_chk) + if member.get('xmlattr'): + attr_hook_vars.append(member['name'].replace('.', '_') + 'Buf') + elif member.get('xmlelem'): + elem_hook_vars.append(member['name'].replace('.', '_') + 'Buf') + + while wraps: + wrap, item = wraps.popitem() + lines = Block() + lines.format('virBufferAddLit(buf, "<${name}");', name=wrap) + if item.attrs: + lines.newline() + lines.extend(item.attrs) + lines.format('virBufferAddLit(buf, "/>\\n");') + + checks = item.hchecks + if item.optional: + checks.extend(item.checks) + elem_checks.extend(item.checks) + + if checks: + lines = if_cond(' || '.join(checks), lines) + lines.newline() + + for line in reversed(lines): + elems.insert(item.pos, line) + + attr_checks = dedup(attr_checks) + elem_checks = dedup(elem_checks) + return (attrs, attr_checks, attr_hook_vars, + elems, elem_checks, elem_hook_vars) + + def _compose_full(attrs, attr_checks, attr_hook_vars, + elems, elem_checks, elem_hook_vars): + typename = atype['name'] + + # Declare virXXXFormatBuf + args = [ + 'virBuffer *buf', + 'const char *name', + 'const %s *def' % typename, + 'const void *parent', + 'void *opaque' + ] + signature = funcSignature('int', typename + 'FormatBuf', args) + writer.write(atype, 'formatfunc', '.h', signature.output() + ';') + + # Implement virXXXFormatBuf + headlines = _handleHeads(attr_hook_vars + elem_hook_vars) + hooklines = _handleHook(attr_hook_vars + elem_hook_vars, typename) + + args[3] += ' G_GNUC_UNUSED' + args[4] += ' G_GNUC_UNUSED' + + macro = 'ENABLE_' + Terms.allcaps(typename) + '_FORMAT_HOOK' + + impl = funcSignature('int', typename + 'FormatBuf', args) + impl.format('{') + impl.mapfmt(' ${_each_line_}', headlines.split('\n')) + impl.newline() + impl.format('#ifdef ${macro}', macro=macro) + impl.mapfmt(' ${_each_line_}', hooklines.split('\n')) + impl.format('#endif /* ${macro} */', macro=macro) + impl.newline() + + empty_checks = '' + if attr_checks or elem_checks: + empty_checks = ' || !(%s)' % ' || '.join(attr_checks + elem_checks) + + impl.format(' if (empty != VIR_TRISTATE_BOOL_NO)') + impl.format(' if (empty${checks})', checks=empty_checks) + impl.format(' return 0;') + impl.newline() + impl.format(' virBufferAsprintf(buf, "<%s", name);') + impl.newline() + + if 'namespace' in atype: + impl.mapfmt(' ${_each_line_}', T_NAMESPACE_FORMAT_BEGIN) + + impl.mapfmt(' ${_each_line_}', attrs) + + if elems: + if attrs: + + shortcut_checks = '' + if elem_checks: + shortcut_checks = ' || !(%s)' % ' || '.join(elem_checks) + + impl.format(' if (shortcut != VIR_TRISTATE_BOOL_NO) {') + impl.format(' if (shortcut${checks}) {', + checks=shortcut_checks) + impl.format(' virBufferAddLit(buf, "/>\\n");') + impl.format(' return 0;') + impl.format(' }') + impl.format(' }') + impl.newline() + + impl.format(' virBufferAddLit(buf, ">\\n");') + impl.newline() + impl.format(' virBufferAdjustIndent(buf, 2);') + impl.newline() + impl.mapfmt(' ${_each_line_}', elems) + + if 'namespace' in atype: + impl.mapfmt(' ${_each_line_}', T_NAMESPACE_FORMAT_END) + impl.newline() + + impl.format(' virBufferAdjustIndent(buf, -2);') + impl.format(' virBufferAsprintf(buf, "</%s>\\n", name);') + else: + impl.format(' virBufferAddLit(buf, "/>\\n");') + + impl.newline() + impl.format(' return 0;') + impl.format('}') + writer.write(atype, 'formatfunc', '.c', impl.output()) + + # Construct check function + _handle_check_func(attr_checks + elem_checks) + + def _compose_part(kind, block, checks, hook_vars): + typename = atype['name'] + funcname = typename + 'Format' + kind + headlines = _handleHeads(hook_vars) + hooklines = _handleHook(hook_vars, atype['name'], kind) + if not block: + block = ['/* no code */', ''] + + # Declare virXXXFormat[Attr|Elem] + args = [ + 'virBuffer *buf', + 'const %s *def' % typename, + 'const void *parent', + 'void *opaque' + ] + signature = funcSignature('int', funcname, args) + writer.write(atype, 'formatfunc', '.h', signature.output() + ';') + + # Implement virXXXFormat[Attr|Elem] + args[2] += ' G_GNUC_UNUSED' + args[3] += ' G_GNUC_UNUSED' + + macro = 'ENABLE_' + Terms.allcaps(typename + 'Format' + kind) + '_HOOK' + + impl = funcSignature('int', funcname, args) + impl.format('{') + impl.mapfmt(' ${_each_line_}', headlines.split('\n')) + impl.newline() + impl.format('#ifdef ${macro}', macro=macro) + impl.mapfmt(' ${_each_line_}', hooklines.split('\n')) + impl.format('#endif /* ${macro} */', macro=macro) + impl.newline() + impl.mapfmt(' ${_each_line_}', block) + impl.format(' return 0;') + impl.format('}') + writer.write(atype, 'formatfunc', '.c', impl.output()) + + # Construct check function + _handle_check_func(checks, kind) + + # + # Main routine of formating. + # + (attrs, attr_checks, attr_hook_vars, + elems, elem_checks, elem_hook_vars) = _prepare() + + if atype['genformat'] in ['separate']: + _compose_part('Attr', attrs, attr_checks, attr_hook_vars) + _compose_part('Elem', elems, elem_checks, elem_hook_vars) + else: + _compose_full(attrs, attr_checks, attr_hook_vars, + elems, elem_checks, elem_hook_vars,) + + +def showDirective(atype): + print('\n[Directive]\n') + print(json.dumps(atype, indent=4) + '\n') diff --git a/scripts/xmlgen/go b/scripts/xmlgen/go new file mode 100755 index 00000000..6c4abe7a --- /dev/null +++ b/scripts/xmlgen/go @@ -0,0 +1,29 @@ +# This is a command-line tool +# +# Copyright (C) 2021 Shandong Massclouds Co.,Ltd. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see +# <http://www.gnu.org/licenses/>;. +# + +DIR="src" +if [ ${FOR_TEST} ]; then + DIR="tests/xmlgenin" +fi + +export PYTHONDONTWRITEBYTECODE=1 +WORK_DIR=$(cd $(dirname $0); pwd) +SRCDIR="${WORK_DIR}/../../${DIR}" +BUILDDIR="${WORK_DIR}/../../build/${DIR}" +${WORK_DIR}/main.py -s ${SRCDIR} -b ${BUILDDIR} $@ diff --git a/scripts/xmlgen/main.py b/scripts/xmlgen/main.py new file mode 100755 index 00000000..48c94984 --- /dev/null +++ b/scripts/xmlgen/main.py @@ -0,0 +1,534 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2021 Shandong Massclouds Co.,Ltd. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see +# <http://www.gnu.org/licenses/>;. +# + +import os +import re +import sys +import argparse +from clang.cindex import Index, CursorKind +from clang.cindex import SourceLocation, SourceRange, TokenKind +from datetime import datetime +from directive import TypeTable, showDirective +from directive import makeClearFunc, makeParseFunc, makeFormatFunc +from utils import Terms, Block + +TOOL_DESC = ''' +Generate xml parse/format functions based on directives. + +Subcommand:\n + list: List types. By default, only list structs tagged by + 'genparse'/'genformat'. When the option '-a' is specified, + list all types discovered by this tool.\n + show: Show the target type's directives and its code for preview. + Specify target type by its name. + The option '-k' can be used to filter output.\n + generate: Generate code. To be called by Makefile. + The option '-k' can be used to filter output. + +Option:\n + -k: Specify kinds to filter code output. More than one kind can be + specified, 'p' for parsefunc and 'f' for formatfunc. + When it is omitted, all functions will be outputed.\n + The option '-k' is only valid for show and generate. +''' + +DIRECTIVES = [ + 'genparse', 'genformat', 'xmlattr', 'xmlelem', 'xmltext', 'xmlgroup', + 'required', 'array', 'specify', 'xmlswitch', 'skipparse', +] + +BUILTIN_MAP = { + 'bool': 'Bool', + 'virBoolYesNo': 'BoolYesNo', + 'virBoolOnOff': 'BoolOnOff', + 'virBoolTrueFalse': 'BoolTrueFalse', + 'char': 'Char', + 'unsigned char': 'UChar', + 'int': 'Int', + 'unsigned int': 'UInt', + 'long': 'Long', + 'unsigned long': 'ULong', + 'long long': 'LongLong', + 'unsigned long long': 'ULongLong', + 'uint8_t': 'U8', + 'uint32_t': 'U32', + 'time_t': 'Time', +} + + +# Three builtin types need to be handled specially: +# 'char *' => String +# 'char XXXX[...]' => Chars +# 'unsigned char XXXX[...]' => UChars +def getBuiltinType(ctype, ptr=False, size=None): + if ctype == 'char': + if ptr: + return 'String' + elif size: + return 'Chars' + + if ctype == 'unsigned char' and size: + return 'UChars' + + return BUILTIN_MAP.get(ctype, None) + + +def cursorLineExtent(cursor, tu): + loc = cursor.location + start = SourceLocation.from_position(tu, loc.file, loc.line, 1) + end = SourceLocation.from_position(tu, loc.file, loc.line, -1) + return SourceRange.from_locations(start, end) + + +def getTokens(cursor, tu): + return tu.get_tokens(extent=cursorLineExtent(cursor, tu)) + + +def createDirectives(text): + tlist = re.findall(r'/\*(.*)\*/', text) + if len(tlist) != 1: + return None + + tlist = tlist[0].split(',') + if len(tlist) == 0: + return None + + directives = {} + for item in tlist: + item = item.strip() + if ':' in item: + key, value = item.split(':') + else: + key, value = item, None + + if key in DIRECTIVES: + directives[key] = value + return directives + + +def getDirectives(tokens, cursor): + for token in tokens: + if token.location.column <= cursor.location.column: + continue + if token.kind == TokenKind.COMMENT: + directive = createDirectives(token.spelling) + if directive: + return directive + return None + + +def determinType(kvs, tokens, cursor): + prefix = [] + kind = None + for token in tokens: + if token.location.column >= cursor.location.column: + break + if not kind: + kind = token.kind + prefix.append(token.spelling) + + suffix = [] + for token in tokens: + if token.spelling == ';': + break + suffix.append(token.spelling) + + size = None + if len(suffix) == 3 and suffix[0] == '[' and suffix[2] == ']': + size = suffix[1] + + assert kind in [TokenKind.IDENTIFIER, TokenKind.KEYWORD], \ + 'Bad field "%s" (%s).' % (cursor.spelling, kind) + + assert prefix + typename = ' '.join(prefix) + + # For array, remove the most-outer pointer + if kvs.get('array'): + if typename.endswith('*'): + typename = typename[:-1].strip() + + ptr = False + if typename.endswith('*'): + typename = typename[:-1].strip() + ptr = True + + ret = getBuiltinType(typename, ptr, size) + if ret: + typename = ret + + kvs.update({'type': typename, 'pointer': ptr}) + if size: + kvs['size'] = size + return kvs + + +def analyseMember(cursor, tu, struct): + if cursor.spelling == 'namespaceData': + struct['namespace'] = True + + dvs = getDirectives(getTokens(cursor, tu), cursor) + if not dvs: + return None + + kvs = {'name': cursor.spelling} + kvs.update(dvs) + + # Formalize member + for key in ['array', 'required', 'skipparse', 'xmltext']: + if key in kvs and not kvs[key]: + kvs[key] = True + + if kvs.get('specify'): + assert isinstance(kvs['specify'], str) and len(kvs['specify']) > 0 + + for tag in ['xmlattr', 'xmlelem', 'xmlgroup']: + if tag in kvs: + if not kvs[tag]: + if kvs.get('array'): + kvs[tag] = Terms.singularize(kvs['name']) + else: + kvs[tag] = kvs['name'] + + return determinType(kvs, getTokens(cursor, tu), cursor) + + +def findMember(struct, name): + for member in struct['members']: + if member['name'] == name: + return member + + return None + + +def analyseStruct(struct, cursor, tu): + tokens = getTokens(cursor, tu) + kvs = getDirectives(tokens, cursor) + if not kvs: + return struct + + path, _ = os.path.splitext(cursor.location.file.name) + path, filename = os.path.split(path) + _, dirname = os.path.split(path) + kvs['output'] = dirname + '/' + filename + struct.update(kvs) + + last_kind = None + inner_members = [] + for child in cursor.get_children(): + if inner_members: + if last_kind == CursorKind.STRUCT_DECL: + # Flatten the members of embedded struct + for member in inner_members: + member['name'] = child.spelling + '.' + member['name'] + struct['members'].append(member) + else: + # Embedded union + union = getDirectives(getTokens(child, tu), child) + if union and 'xmlswitch' in union: + switch = findMember(struct, union['xmlswitch']) + switch['delay_clear'] = True + union['switch_type'] = switch['type'] + union['name'] = child.spelling + union['members'] = inner_members + struct['members'].append(union) + struct['xmlswitch'] = union['xmlswitch'] + + inner_members = [] + last_kind = None + continue + + if child.kind in [CursorKind.STRUCT_DECL, CursorKind.UNION_DECL]: + last_kind = child.kind + for ichild in child.get_children(): + member = analyseMember(ichild, tu, struct) + if member: + inner_members.append(member) + continue + + member = analyseMember(child, tu, struct) + if member: + struct['members'].append(member) + + return struct + + +# Check whether the first item of enum is default. +# (i.e. it ends with '_NONE' or '_DEFAULT') +def checkFirstEnumItem(cursor): + find_paren = False + for token in cursor.get_tokens(): + # skip all parts in front of '{' + if not find_paren: + if token.spelling != '{': + continue + find_paren = True + + if token.kind == TokenKind.IDENTIFIER: + return token.spelling.endswith(('_NONE', '_DEFAULT', '_ABSENT')) + + return False + + +def rescanStruct(struct): + for member in struct['members']: + if member.get('specify'): + tname = member['specify'] + target = findMember(struct, tname) + assert target, "Can't find '%s' in '%s'" % (tname, struct['name']) + managed = not (member.get('xmlattr') or member.get('xmlelem')) + target['specified'] = (member['name'], managed) + + +def discoverStructures(tu): + for cursor in tu.cursor.get_children(): + if cursor.kind == CursorKind.STRUCT_DECL and cursor.is_definition(): + # Detect structs + name = cursor.spelling + if not name: + continue + if name.startswith('_'): + name = name[1:] + if TypeTable().check(name): + continue + struct = {'name': name, 'meta': 'Struct', 'members': []} + analyseStruct(struct, cursor, tu) + rescanStruct(struct) + TypeTable().register(struct) + elif cursor.kind == CursorKind.TYPEDEF_DECL: + # Detect enums + # We can't seek out enums by CursorKind.ENUM_DECL, + # since almost all enums are anonymous. + token = cursor.get_tokens() + try: + next(token) # skip 'typedef' + if next(token).spelling == 'enum': + if not TypeTable().check(cursor.spelling): + enum = {'name': cursor.spelling, 'meta': 'Enum', + 'with_default': checkFirstEnumItem(cursor)} + TypeTable().register(enum) + except StopIteration: + pass + + +class CodeWriter(object): + def __init__(self, args): + self._buildtop = args.buildtop + self._curdir = args.curdir + self._cmd = args.cmd + self._files = {} + self._filters = {} + self._filters['clearfunc'] = args.kinds and 'p' in args.kinds + self._filters['parsefunc'] = args.kinds and 'p' in args.kinds + self._filters['formatfunc'] = args.kinds and 'f' in args.kinds + if args.cmd == 'show': + self._filters['target'] = args.target + + def _getFile(self, path, ext): + assert ext in ['.h', '.c'] + _, basename = os.path.split(path) + path = '%s.generated%s' % (path, ext) + f = self._files.get(path) + if f is None: + f = open(path, 'w') + f.write('/* Generated by scripts/xmlgen */\n\n') + if ext in ['.c']: + f.write('#include <config.h>\n') + f.write('#include "%s.h"\n' % basename) + f.write('#include "viralloc.h"\n') + f.write('#include "virerror.h"\n') + f.write('#include "virstring.h"\n\n') + f.write('#include "virenum.h"\n\n') + f.write('#define VIR_FROM_THIS VIR_FROM_NONE\n') + else: + f.write('#pragma once\n\n') + f.write('#include "internal.h"\n') + f.write('#include "virxml.h"\n') + self._files[path] = f + return f + + def write(self, atype, kind, extname, content): + if not self._filters[kind]: + return + + if self._cmd == 'show': + target = self._filters['target'] + if not target or target == atype['name']: + outputname = atype.get('output', '') + '.generated' + if extname == '.h': + print('\n[%s.h]' % outputname) + else: + print('\n[%s.c]' % outputname) + print('\n' + content + '\n') + return + + assert self._cmd == 'generate' + + if atype.get('output') and atype['output'].startswith(self._curdir): + lfs = '\n' if extname == '.h' else '\n\n' + path = self._buildtop + '/' + atype['output'] + f = self._getFile(path, extname) + f.write(lfs + content + '\n') + + def complete(self): + for name in self._files: + self._files[name].close() + self._files.clear() + + +def getHFiles(path): + retlist = [] + for fname in os.listdir(path): + if fname.endswith('.h'): + retlist.append(os.path.join(path, fname)) + return retlist + + +def showTips(target, kinds): + atype = TypeTable().get(target) + if not atype or atype.get('external'): + sys.exit(0) + + kind_parse = kinds and 'p' in kinds and 'genparse' in atype + kind_format = kinds and 'f' in kinds and 'genformat' in atype + if not kind_parse and not kind_format: + sys.exit(0) + + captype = Terms.allcaps(target) + pathname = atype.get('output', '') + subdir, basename = os.path.split(pathname) + + tips = Block() + tips.newline() + tips.append('[Tips]') + tips.newline() + tips.format('/* Put these lines at the bottom of "${path}.h" */', + path=pathname) + tips.format('/* Makesure "${base}.h" to be appended into ' + '${cur}_xmlgen_input in src/${cur}/meson.build */', + base=basename, cur=subdir) + + if kind_parse or kind_format: + tips.newline() + tips.append('/* Define macro to enable hook or redefine check ' + 'when necessary */') + + if kind_parse: + tips.format('/* #define ENABLE_${cap}_PARSE_HOOK */', cap=captype) + tips.format('/* #define ENABLE_${cap}_PARSE_HOOK_SET_ARGS */', + cap=captype) + + if kind_format: + if atype.get('genformat') in ['separate']: + tips.format('/* #define ENABLE_${cap}_FORMAT_ATTR_HOOK */', + cap=captype) + tips.format('/* #define ENABLE_${cap}_FORMAT_ELEM_HOOK */', + cap=captype) + tips.newline() + tips.format('/* #define RESET_${cap}_CHECK_ATTR */', + cap=captype) + tips.format('/* #define RESET_${cap}_CHECK_ELEM */', + cap=captype) + else: + tips.format('/* #define ENABLE_${cap}_FORMAT_HOOK */', + cap=captype) + tips.newline() + tips.format('/* #define RESET_${cap}_CHECK */', cap=captype) + + tips.newline() + tips.append('/* Makesure below is the bottom line! */') + tips.format('#include "${base}.generated.h"', base=basename) + print(tips.output()) + + +HELP_LIST = 'list structs tagged by "genparse"/"genformat"' +HELP_LIST_ALL = 'list all discovered types' + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + formatter_class=argparse.RawDescriptionHelpFormatter, + description=TOOL_DESC) + parser.add_argument('-s', dest='srctop', help='top source directory') + parser.add_argument('-b', dest='buildtop', help='top build directory') + parser.add_argument('-d', dest='curdir', + help='directory to be dealt with. (util or conf)') + subparsers = parser.add_subparsers(dest='cmd') + parser_list = subparsers.add_parser('list', help=HELP_LIST) + parser_list.add_argument('-a', dest='list_all', action='store_true', + default=False, help=HELP_LIST_ALL) + parser_show = subparsers.add_parser('show', help='show target code') + parser_show.add_argument('target', help='target for being previewed') + parser_show.add_argument('-k', dest='kinds', default='pf', + help='kinds of code to be previewed') + parser_show.add_argument('-V', dest='verbose', action='store_true', + help='internal information for debug') + parser_generate = subparsers.add_parser('generate', help='generate code') + parser_generate.add_argument('-k', dest='kinds', default='pf', + help='kinds of code to be generated') + args = parser.parse_args() + + if not args.cmd or not args.srctop or not args.buildtop: + parser.print_help() + sys.exit(1) + + if args.cmd == 'generate': + print('###### xmlgen: work in src/%s ... ######' % args.curdir) + + timestamp = datetime.now() + + # Examine all *.h in "$(srctop)/[util|conf]" + index = Index.create() + srctop = args.srctop + hfiles = getHFiles(srctop + '/util') + getHFiles(srctop + '/conf') + for hfile in hfiles: + tu = index.parse(hfile) + discoverStructures(tu) # find all structs and enums + + if args.cmd == 'list': + print('%-64s %s' % ('TYPENAME', 'META')) + for name, kvs in TypeTable().items(): + if not args.list_all: + if not ('genparse' in kvs or 'genformat' in kvs): + continue + print('%-64s %s' % (name, kvs['meta'])) + sys.exit(0) + elif args.cmd == 'show': + assert args.target, args + atype = TypeTable().get(args.target) + if not atype: + sys.exit(0) + if args.verbose: + showDirective(atype) + + writer = CodeWriter(args) + + for atype in TypeTable().values(): + makeClearFunc(writer, atype) + makeParseFunc(writer, atype) + makeFormatFunc(writer, atype) + + writer.complete() + + if args.cmd == 'generate': + elapse = (datetime.now() - timestamp).microseconds + print('\n###### xmlgen: elapse %d(us) ######\n' % elapse) + elif args.cmd == 'show': + showTips(args.target, args.kinds) + + sys.exit(0) diff --git a/scripts/xmlgen/utils.py b/scripts/xmlgen/utils.py new file mode 100644 index 00000000..922cd6d9 --- /dev/null +++ b/scripts/xmlgen/utils.py @@ -0,0 +1,121 @@ +# +# Copyright (C) 2021 Shandong Massclouds Co.,Ltd. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see +# <http://www.gnu.org/licenses/>;. +# + +from string import Template + + +def singleton(cls): + _instances = {} + + def inner(): + if cls not in _instances: + _instances[cls] = cls() + return _instances[cls] + return inner + + +class Terms(object): + abbrs = ['uuid', 'pci', 'zpci', 'ptr', 'mac', 'mtu', 'dns', 'ip', 'dhcp'] + plurals = {'addresses': 'address'} + caps = {'NET_DEV': 'NETDEV', 'MACTABLE': 'MAC_TABLE'} + + @classmethod + def _split(cls, word): + ret = [] + if not word: + return ret + head = 0 + for pos in range(1, len(word)): + if word[pos].isupper() and not word[pos - 1].isupper(): + ret.append(word[head:pos]) + head = pos + ret.append(word[head:]) + return ret + + @classmethod + def singularize(cls, name): + ret = cls.plurals.get(name, None) + if ret: + return ret + assert name.endswith('s') + return name[:-1] + + # Don't use str.capitalize() which force other letters to be lowercase. + @classmethod + def upperInitial(cls, word): + if not word: + return '' + if word in cls.abbrs: + return word.upper() + if len(word) > 0 and word[0].isupper(): + return word + return word[0].upper() + word[1:] + + @classmethod + def allcaps(cls, word): + if len(word) == 0: + return word + parts = cls._split(word) + ret = '_'.join([part.upper() for part in parts]) + for key, value in cls.caps.items(): + ret = ret.replace('_%s_' % key, '_%s_' % value) + return ret + + +def render(template, **kwargs): + return Template(template).substitute(kwargs) + + +class Block(list): + def format(self, template, **args): + if template: + self.append(Template(template).substitute(**args)) + + def extend(self, block): + if isinstance(block, list): + super(Block, self).extend(block) + + # ${_each_line_} is the only legal key for template + # and represents each line of the block. + def mapfmt(self, template, block): + if not block or not template: + return + + assert isinstance(block, list), block + for line in block: + if line: + self.append(Template(template).substitute(_each_line_=line)) + else: + self.append('') + + def newline(self, condition=True): + if condition: + super(Block, self).append('') + + def output(self, connector='\n'): + return connector.join(self) + + +def dedup(alist): + assert isinstance(alist, list) + ret = [] + for e in alist: + if e not in ret: + ret.append(e) + + return ret -- 2.25.1