As part of an goal to eliminate Perl from libvirt build tools, rewrite the mock-noinline.pl tool in Python. This was a straight conversion, manually going line-by-line to change the syntax from Perl to Python. Thus the overall structure of the file and approach is the same. Signed-off-by: Daniel P. Berrangé <berrange@redhat.com> --- Makefile.am | 2 +- build-aux/mock-noinline.pl | 75 --------------------------------- build-aux/syntax-check.mk | 4 +- scripts/mock-noinline.py | 85 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 88 insertions(+), 78 deletions(-) delete mode 100644 build-aux/mock-noinline.pl create mode 100644 scripts/mock-noinline.py diff --git a/Makefile.am b/Makefile.am index 5187ca6cc2..d9369c9197 100644 --- a/Makefile.am +++ b/Makefile.am @@ -49,7 +49,7 @@ EXTRA_DIST = \ scripts/check-spacing.py \ build-aux/header-ifdef.pl \ scripts/minimize-po.py \ - build-aux/mock-noinline.pl \ + scripts/mock-noinline.py \ scripts/prohibit-duplicate-header.py \ build-aux/syntax-check.mk \ build-aux/useless-if-before-free \ diff --git a/build-aux/mock-noinline.pl b/build-aux/mock-noinline.pl deleted file mode 100644 index b005b8d95e..0000000000 --- a/build-aux/mock-noinline.pl +++ /dev/null @@ -1,75 +0,0 @@ -#!/usr/bin/env perl - -my %noninlined; -my %mocked; - -# Functions in public header don't get the noinline annotation -# so whitelist them here -$noninlined{"virEventAddTimeout"} = 1; -# This one confuses the script as its defined in the mock file -# but is actually just a local helper -$noninlined{"virMockStatRedirect"} = 1; - -foreach my $arg (@ARGV) { - if ($arg =~ /\.h$/) { - #print "Scan header $arg\n"; - &scan_annotations($arg); - } elsif ($arg =~ /mock\.c$/) { - #print "Scan mock $arg\n"; - &scan_overrides($arg); - } -} - -my $warned = 0; -foreach my $func (keys %mocked) { - next if exists $noninlined{$func}; - - $warned++; - print STDERR "$func is mocked at $mocked{$func} but missing noinline annotation\n"; -} - -exit $warned ? 1 : 0; - - -sub scan_annotations { - my $file = shift; - - open FH, $file or die "cannot read $file: $!"; - - my $func; - while (<FH>) { - if (/^\s*(\w+)\(/ || /^(?:\w+\*?\s+)+(?:\*\s*)?(\w+)\(/) { - my $name = $1; - if ($name !~ /(?:G_GNUC|ATTRIBUTE)/) { - $func = $name; - } - } elsif (/^\s*$/) { - $func = undef; - } - if (/G_GNUC_NO_INLINE/) { - if (defined $func) { - $noninlined{$func} = 1; - } - } - } - - close FH -} - -sub scan_overrides { - my $file = shift; - - open FH, $file or die "cannot read $file: $!"; - - my $func; - while (<FH>) { - if (/^(\w+)\(/ || /^\w+\s*(?:\*\s*)?(\w+)\(/) { - my $name = $1; - if ($name =~ /^vir/) { - $mocked{$name} = "$file:$."; - } - } - } - - close FH -} diff --git a/build-aux/syntax-check.mk b/build-aux/syntax-check.mk index d308896b26..0cfd181224 100644 --- a/build-aux/syntax-check.mk +++ b/build-aux/syntax-check.mk @@ -2162,8 +2162,8 @@ spacing-check: { echo '$(ME): incorrect formatting' 1>&2; exit 1; } mock-noinline: - $(AM_V_GEN)$(VC_LIST) | $(GREP) '\.[ch]$$' | xargs \ - $(PERL) $(top_srcdir)/build-aux/mock-noinline.pl + $(AM_V_GEN)$(VC_LIST) | $(GREP) '\.[ch]$$' | $(RUNUTF8) xargs \ + $(PYTHON) $(top_srcdir)/scripts/mock-noinline.py header-ifdef: $(AM_V_GEN)$(VC_LIST) | $(GREP) '\.[h]$$' | xargs \ diff --git a/scripts/mock-noinline.py b/scripts/mock-noinline.py new file mode 100644 index 0000000000..2770ea1238 --- /dev/null +++ b/scripts/mock-noinline.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python +# +# Copyright (C) 2017-2019 Red Hat, Inc. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see +# <http://www.gnu.org/licenses/>. + +from __future__ import print_function + +import re +import sys + +noninlined = {} +mocked = {} + +# Functions in public header don't get the noinline annotation +# so whitelist them here +noninlined["virEventAddTimeout"] = True +# This one confuses the script as its defined in the mock file +# but is actually just a local helper +noninlined["virMockStatRedirect"] = True + + +def scan_annotations(filename): + with open(filename, "r") as fh: + func = None + for line in fh: + line = line.strip() + m = re.search(r'''^\s*(\w+)\(''', line) + if m is None: + m = re.search(r'''^(?:\w+\*?\s+)+(?:\*\s*)?(\w+)\(''', line) + if m is not None: + name = m.group(1) + if name.find("ATTRIBUTE") == -1 and name.find("G_GNUC_") == -1: + func = name + elif line == "": + func = None + + if line.find("G_GNUC_NO_INLINE") != -1: + if func is not None: + noninlined[func] = True + + +def scan_overrides(filename): + with open(filename, "r") as fh: + lineno = 0 + for line in fh: + lineno = lineno + 1 + + m = re.search(r'''^(\w+)\(''', line) + if m is None: + m = re.search(r'''^\w+\s*(?:\*\s*)?(\w+)\(''', line) + if m is not None: + name = m.group(1) + if name.startswith("vir"): + mocked[name] = "%s:%d" % (filename, lineno) + + +for filename in sys.argv[1:]: + if filename.endswith(".h"): + scan_annotations(filename) + elif filename.endswith("mock.c"): + scan_overrides(filename) + +warned = False +for func in mocked.keys(): + if func not in noninlined: + warned = True + print("%s is mocked at %s but missing noinline annotation" % + (func, mocked[func]), file=sys.stderr) + +if warned: + sys.exit(1) +sys.exit(0) -- 2.21.0

As part of an goal to eliminate Perl from libvirt build tools, rewrite the check-aclperms.pl tool in Python. This was a straight conversion, manually going line-by-line to change the syntax from Perl to Python. Thus the overall structure of the file and approach is the same. Signed-off-by: Daniel P. Berrangé <berrange@redhat.com> --- Makefile.am | 1 + scripts/check-aclperms.py | 75 +++++++++++++++++++++++++++++++++++++++ src/Makefile.am | 4 +-- src/check-aclperms.pl | 73 ------------------------------------- 4 files changed, 78 insertions(+), 75 deletions(-) create mode 100755 scripts/check-aclperms.py delete mode 100755 src/check-aclperms.pl diff --git a/Makefile.am b/Makefile.am index 6cccbf38da..ab9d09fcd4 100644 --- a/Makefile.am +++ b/Makefile.am @@ -46,6 +46,7 @@ EXTRA_DIST = \ README.md \ AUTHORS.in \ scripts/augeas-gentest.py \ + scripts/check-aclperms.py \ scripts/check-spacing.py \ scripts/header-ifdef.py \ scripts/minimize-po.py \ diff --git a/scripts/check-aclperms.py b/scripts/check-aclperms.py new file mode 100755 index 0000000000..b1084a3758 --- /dev/null +++ b/scripts/check-aclperms.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python +# +# Copyright (C) 2013-2019 Red Hat, Inc. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see +# <http://www.gnu.org/licenses/>. +# +# This script just validates that the stringified version of +# a virAccessPerm enum matches the enum constant name. We do +# a lot of auto-generation of code, so when these don't match +# problems occur, preventing auth from succeeding at all. + +from __future__ import print_function + +import re +import sys + +if len(sys.argv) != 3: + print("syntax: %s HEADER IMPL" % (sys.argv[0]), file=sys.stderr) + sys.exit(1) + +hdr = sys.argv[1] +impl = sys.argv[2] + +perms = {} + +with open(hdr) as fh: + for line in fh: + symmatch = re.search(r"^\s+VIR_ACCESS_PERM_([_A-Z]+)(,?|\s|$)", line) + if symmatch is not None: + perm = symmatch.group(1) + + if not perm.endswith("_LAST"): + perms[perm] = 1 + +warned = False + +with open(impl) as fh: + group = None + + for line in fh: + symlastmatch = re.search(r"VIR_ACCESS_PERM_([_A-Z]+)_LAST", line) + if symlastmatch is not None: + group = symlastmatch.group(1) + elif re.search(r'''"[_a-z]+"''', line) is not None: + bits = line.split(",") + for bit in bits: + m = re.search(r'''"([_a-z]+)"''', bit) + if m is not None: + perm = (group + "_" + m.group(1)).upper() + if perm not in perms: + print("Unknown perm string %s for group %s" % + (m.group(1), group), file=sys.stderr) + warned = True + + del perms[perm] + +for perm in perms.keys(): + print("Perm %s had not string form" % perm, file=sys.stderr) + warned = True + +if warned: + sys.exit(1) +sys.exit(0) diff --git a/src/Makefile.am b/src/Makefile.am index 9b0a46702b..318dd6c20f 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -353,11 +353,11 @@ check-aclrules: $(addprefix $(srcdir)/,$(filter-out /%,$(STATEFUL_DRIVER_SOURCE_FILES))) check-aclperms: - $(AM_V_GEN)$(PERL) $(srcdir)/check-aclperms.pl \ + $(AM_V_GEN)$(RUNUTF8) $(PYTHON) $(top_srcdir)/scripts/check-aclperms.py \ $(srcdir)/access/viraccessperm.h \ $(srcdir)/access/viraccessperm.c -EXTRA_DIST += check-driverimpls.pl check-aclrules.pl check-aclperms.pl +EXTRA_DIST += check-driverimpls.pl check-aclrules.pl check-local: check-protocol check-symfile check-symsorting \ check-drivername check-driverimpls check-aclrules \ diff --git a/src/check-aclperms.pl b/src/check-aclperms.pl deleted file mode 100755 index 55b6598313..0000000000 --- a/src/check-aclperms.pl +++ /dev/null @@ -1,73 +0,0 @@ -#!/usr/bin/env perl -# -# Copyright (C) 2013 Red Hat, Inc. -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2.1 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library. If not, see -# <http://www.gnu.org/licenses/>. -# -# This script just validates that the stringified version of -# a virAccessPerm enum matches the enum constant name. We do -# a lot of auto-generation of code, so when these don't match -# problems occur, preventing auth from succeeding at all. - -my $hdr = shift; -my $impl = shift; - -my %perms; - -my @perms; - -open HDR, $hdr or die "cannot read $hdr: $!"; - -while (<HDR>) { - if (/^\s+VIR_ACCESS_PERM_([_A-Z]+)(,?|\s|$)/) { - my $perm = $1; - - $perms{$perm} = 1 unless ($perm =~ /_LAST$/); - } -} - -close HDR; - - -open IMPL, $impl or die "cannot read $impl: $!"; - -my $group; -my $warned = 0; - -while (defined (my $line = <IMPL>)) { - if ($line =~ /VIR_ACCESS_PERM_([_A-Z]+)_LAST/) { - $group = $1; - } elsif ($line =~ /"[_a-z]+"/) { - my @bits = split /,/, $line; - foreach my $bit (@bits) { - if ($bit =~ /"([_a-z]+)"/) { - my $perm = uc($group . "_" . $1); - if (!exists $perms{$perm}) { - print STDERR "Unknown perm string $1 for group $group\n"; - $warned = 1; - } - delete $perms{$perm}; - } - } - } -} -close IMPL; - -foreach my $perm (keys %perms) { - print STDERR "Perm $perm had not string form\n"; - $warned = 1; -} - -exit $warned; -- 2.21.0

As part of an goal to eliminate Perl from libvirt build tools, rewrite the check-symfile.pl tool in Python. This was a straight conversion, manually going line-by-line to change the syntax from Perl to Python. Thus the overall structure of the file and approach is the same. Signed-off-by: Daniel P. Berrangé <berrange@redhat.com> --- Makefile.am | 1 + scripts/check-symfile.py | 80 +++++++++++++++++++++++++++++++++++++++ src/Makefile.am | 5 +-- src/admin/Makefile.inc.am | 4 +- src/check-symfile.pl | 70 ---------------------------------- 5 files changed, 85 insertions(+), 75 deletions(-) create mode 100755 scripts/check-symfile.py delete mode 100755 src/check-symfile.pl diff --git a/Makefile.am b/Makefile.am index b98c932ff7..1c7b8045c2 100644 --- a/Makefile.am +++ b/Makefile.am @@ -48,6 +48,7 @@ EXTRA_DIST = \ scripts/augeas-gentest.py \ scripts/check-aclperms.py \ scripts/check-spacing.py \ + scripts/check-symfile.py \ scripts/check-symsorting.py \ scripts/header-ifdef.py \ scripts/minimize-po.py \ diff --git a/scripts/check-symfile.py b/scripts/check-symfile.py new file mode 100755 index 0000000000..a543a0fbcd --- /dev/null +++ b/scripts/check-symfile.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python +# +# Copyright (C) 2012-2019 Red Hat, Inc. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see +# <http://www.gnu.org/licenses/>. + +from __future__ import print_function + +import re +import subprocess +import sys + +if len(sys.argv) < 3: + print("syntax: %s SYMFILE ELFLIB(S)" % sys.argv[0], file=sys.stderr) + +symfile = sys.argv[1] +elflibs = sys.argv[2:] + +wantsyms = {} +gotsyms = {} + +ret = 0 + +with open(symfile, "r") as fh: + for line in fh: + line = line.strip() + if line.find("{") != -1: + continue + if line.find("}") != -1: + continue + if line in ["global:", "local:"]: + continue + if line == "": + continue + if line[0] == '#': + continue + if line.find("*") != -1: + continue + + line = line.strip(";") + + if line in wantsyms: + print("Symbol $1 is listed twice", file=sys.stderr) + ret = 1 + else: + wantsyms[line] = True + +for elflib in elflibs: + nm = subprocess.Popen(["nm", elflib], shell=False, + stdout=subprocess.PIPE).stdout + + for line in nm: + line = line.decode("utf-8") + symmatch = re.search(r'''^\S+\s(?:[TBD])\s(\S+)\s*$''', line) + if symmatch is None: + continue + + gotsyms[symmatch.group(1)] = True + + +for sym in wantsyms.keys(): + if sym in gotsyms: + continue + + print("Expected symbol '%s' is not in ELF library" % sym, file=sys.stderr) + ret = 1 + +sys.exit(ret) diff --git a/src/Makefile.am b/src/Makefile.am index 41cfd79ad3..929eef784c 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -277,15 +277,14 @@ PDWTAGS = \ # rule for libvirt.la. However, checking symbols relies on Linux ELF layout if WITH_LINUX check-symfile: libvirt.syms libvirt.la - $(AM_V_GEN)$(PERL) $(srcdir)/check-symfile.pl libvirt.syms \ - .libs/libvirt.so + $(AM_V_GEN)$(RUNUTF8) $(PYTHON) $(top_srcdir)/scripts/check-symfile.py \ + libvirt.syms .libs/libvirt.so else ! WITH_LINUX check-symfile: endif ! WITH_LINUX check-symsorting: $(AM_V_GEN)$(RUNUTF8) $(PYTHON) $(top_srcdir)/scripts/check-symsorting.py \ $(srcdir) $(SYM_FILES) -EXTRA_DIST += check-symfile.pl # Keep this list synced with RPC_PROBE_FILES PROTOCOL_STRUCTS = \ diff --git a/src/admin/Makefile.inc.am b/src/admin/Makefile.inc.am index 97048df6f3..a02991df68 100644 --- a/src/admin/Makefile.inc.am +++ b/src/admin/Makefile.inc.am @@ -105,8 +105,8 @@ libvirt_admin_la_CFLAGS = \ if WITH_LINUX check-admin-symfile: admin/libvirt_admin.syms libvirt-admin.la - $(AM_V_GEN)$(PERL) $(srcdir)/check-symfile.pl admin/libvirt_admin.syms \ - .libs/libvirt-admin.so + $(AM_V_GEN)$(RUNUTF8) $(PYTHON) $(top_srcdir)/scripts/check-symfile.py \ + admin/libvirt_admin.syms .libs/libvirt-admin.so else ! WITH_LINUX check-admin-symfile: endif ! WITH_LINUX diff --git a/src/check-symfile.pl b/src/check-symfile.pl deleted file mode 100755 index 4f88300864..0000000000 --- a/src/check-symfile.pl +++ /dev/null @@ -1,70 +0,0 @@ -#!/usr/bin/env perl - -# Copyright (C) 2012-2013 Red Hat, Inc. -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2.1 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library. If not, see -# <http://www.gnu.org/licenses/>. - -die "syntax: $0 SYMFILE ELFLIB(S)" unless int(@ARGV) >= 2; - -my $symfile = shift @ARGV; -my @elflibs = @ARGV; - -my %wantsyms; -my %gotsyms; - -my $ret = 0; - -open SYMFILE, $symfile or die "cannot read $symfile: $!"; - -while (<SYMFILE>) { - next if /{/; - next if /}/; - next if /global:/; - next if /local:/; - next if /^\s*$/; - next if /^\s*#/; - next if /\*/; - - die "malformed line $_" unless /^\s*(\S+);$/; - - if (exists $wantsyms{$1}) { - print STDERR "Symbol $1 is listed twice\n"; - $ret = 1; - } else { - $wantsyms{$1} = 1; - } -} -close SYMFILE; - -foreach my $elflib (@elflibs) { - open NM, "-|", "nm", $elflib or die "cannot run 'nm $elflib': $!"; - - while (<NM>) { - next unless /^\S+\s(?:[TBD])\s(\S+)\s*$/; - - $gotsyms{$1} = 1; - } - - close NM; -} - -foreach my $sym (keys(%wantsyms)) { - next if exists $gotsyms{$sym}; - - print STDERR "Expected symbol $sym is not in ELF library\n"; - $ret = 1; -} - -exit($ret); -- 2.21.0

As part of an goal to eliminate Perl from libvirt build tools, rewrite the gensystemtap.pl tool in Python. This was a straight conversion, manually going line-by-line to change the syntax from Perl to Python. Thus the overall structure of the file and approach is the same. Signed-off-by: Daniel P. Berrangé <berrange@redhat.com> --- Makefile.am | 1 + build-aux/syntax-check.mk | 4 +- scripts/gensystemtap.py | 184 ++++++++++++++++++++++++++++++++++++ src/Makefile.am | 5 +- src/rpc/Makefile.inc.am | 1 - src/rpc/gensystemtap.pl | 193 -------------------------------------- 6 files changed, 189 insertions(+), 199 deletions(-) create mode 100755 scripts/gensystemtap.py delete mode 100755 src/rpc/gensystemtap.pl diff --git a/Makefile.am b/Makefile.am index e50a29635b..afed409c94 100644 --- a/Makefile.am +++ b/Makefile.am @@ -51,6 +51,7 @@ EXTRA_DIST = \ scripts/check-symfile.py \ scripts/check-symsorting.py \ scripts/dtrace2systemtap.py \ + scripts/gensystemtap.py \ scripts/header-ifdef.py \ scripts/minimize-po.py \ scripts/mock-noinline.py \ diff --git a/build-aux/syntax-check.mk b/build-aux/syntax-check.mk index 016e05e7a2..26eb5b94d9 100644 --- a/build-aux/syntax-check.mk +++ b/build-aux/syntax-check.mk @@ -466,6 +466,7 @@ sc_prohibit_risky_id_promotion: # since gnulib has more guarantees for snprintf portability sc_prohibit_sprintf: @prohibit='\<[s]printf\>' \ + in_vc_files='\.[ch]$$' \ halt='use snprintf, not sprintf' \ $(_sc_search_regexp) @@ -2261,9 +2262,6 @@ exclude_file_name_regexp--sc_prohibit_readlink = \ exclude_file_name_regexp--sc_prohibit_setuid = ^src/util/virutil\.c|tools/virt-login-shell\.c$$ -exclude_file_name_regexp--sc_prohibit_sprintf = \ - ^(build-aux/syntax-check\.mk|docs/hacking\.html\.in|.*\.stp|.*\.pl)$$ - exclude_file_name_regexp--sc_prohibit_strncpy = ^src/util/virstring\.c$$ exclude_file_name_regexp--sc_prohibit_strtol = ^examples/.*$$ diff --git a/scripts/gensystemtap.py b/scripts/gensystemtap.py new file mode 100755 index 0000000000..c4bc1620d0 --- /dev/null +++ b/scripts/gensystemtap.py @@ -0,0 +1,184 @@ +#!/usr/bin/env python +# +# Copyright (C) 2011-2019 Red Hat, Inc. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see +# <http://www.gnu.org/licenses/>. +# +# Generate a set of systemtap functions for translating various +# RPC enum values into strings +# +# python gensystemtap.py */*.x > libvirt_functions.stp +# + +from __future__ import print_function + +import re +import sys + +funcs = {} + +types = {} +status = {} +auth = {} + + +def load_file(fh): + instatus = False + intype = False + inauth = False + + for line in fh: + if re.search(r'''enum\s+virNetMessageType''', line): + intype = True + elif re.search(r'''enum\s+virNetMessageStatus''', line): + instatus = True + elif re.search(r'''enum remote_auth_type''', line): + inauth = True + elif line.find("}") != -1: + intype = False + instatus = False + inauth = False + elif instatus: + m = re.search(r'''^\s+VIR_NET_(\w+)\s*=\s*(\d+),?$''', line) + if m is not None: + status[m.group(2)] = m.group(1).lower() + elif intype: + m = re.search(r'''^\s+VIR_NET_(\w+)\s*=\s*(\d+),?$''', line) + if m is not None: + types[m.group(2)] = m.group(1).lower() + elif inauth: + m = re.search(r'''^\s+REMOTE_AUTH_(\w+)\s*=\s*(\d+),?$''', line) + if m is not None: + auth[m.group(2)] = m.group(1).lower() + else: + m = re.search(r'''(?:VIR_)?(\w+?)(?:_PROTOCOL)?''' + + r'''_PROGRAM\s*=\s*0x([a-fA-F0-9]+)\s*;''', line) + if m is not None: + funcs[m.group(1).lower()] = { + "id": int(m.group(2), 16), + "version": None, + "progs": [] + } + continue + + m = re.search(r'''(?:VIR_)?(\w+?)(?:_PROTOCOL)?_''' + + r'''(?:PROGRAM|PROTOCOL)_VERSION\s*=\s*(\d+)\s*;''', + line) + if m is not None: + funcs[m.group(1).lower()]["version"] = m.group(2) + continue + + m = re.search(r'''(?:VIR_)?(\w+?)(?:_PROTOCOL)?''' + + r'''_PROC_(.*?)\s+=\s+(\d+)''', line) + if m is not None: + funcs[m.group(1).lower()]["progs"].insert( + int(m.group(3)), m.group(2).lower()) + + +for file in sys.argv[1:]: + with open(file, "r") as fh: + load_file(fh) + + +def genfunc(name, varname, types): + print("function %s(%s, verbose)" % (name, varname)) + print("{") + + first = True + for typename in sorted(types.keys()): + cond = "} else if" + if first: + cond = "if" + first = False + + print(" %s (%s == %s) {" % (cond, varname, typename)) + print(" %sstr = \"%s\"" % (varname, types[typename])) + + print(" } else {") + print(" %sstr = \"unknown\";" % varname) + print(" verbose = 1;") + print(" }") + print(" if (verbose) {") + print(" %sstr = %sstr . sprintf(\":%%d\", %s)" % + (varname, varname, varname)) + print(" }") + print(" return %sstr;" % varname) + print("}") + + +genfunc("libvirt_rpc_auth_name", "type", auth) +genfunc("libvirt_rpc_type_name", "type", types) +genfunc("libvirt_rpc_status_name", "status", status) + +print("function libvirt_rpc_program_name(program, verbose)") +print("{") + +first = True +for funcname in sorted(funcs.keys()): + cond = "} else if" + if first: + cond = "if" + first = False + + print(" %s (program == %s) {" % (cond, funcs[funcname]["id"])) + print(" programstr = \"%s\"" % funcname) + +print(" } else {") +print(" programstr = \"unknown\";") +print(" verbose = 1;") +print(" }") +print(" if (verbose) {") +print(" programstr = programstr . sprintf(\":%d\", program)") +print(" }") +print(" return programstr;") +print("}") + +print("function libvirt_rpc_procedure_name(program, version, proc, verbose)") +print("{") + +first = True +for prog in sorted(funcs.keys()): + cond = "} else if" + if first: + cond = "if" + first = False + + print(" %s (program == %s && version == %s) {" % + (cond, funcs[prog]["id"], funcs[prog]["version"])) + + pfirst = True + for id in range(len(funcs[prog]["progs"])): + pcond = "} else if" + if pfirst: + pcond = "if" + pfirst = False + + print(" %s (proc == %s) {" % (pcond, id + 1)) + print(" procstr = \"%s\";" % funcs[prog]["progs"][id]) + + print(" } else {") + print(" procstr = \"unknown\";") + print(" verbose = 1;") + print(" }") + +print(" } else {") +print(" procstr = \"unknown\";") +print(" verbose = 1;") +print(" }") +print(" if (verbose) {") +print(" procstr = procstr . sprintf(\":%d\", proc)") +print(" }") +print(" return procstr;") +print("}") diff --git a/src/Makefile.am b/src/Makefile.am index 085c193013..b0ca9d3442 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -561,8 +561,9 @@ RPC_PROBE_FILES += $(srcdir)/rpc/virnetprotocol.x \ $(srcdir)/remote/qemu_protocol.x \ $(srcdir)/admin/admin_protocol.x -libvirt_functions.stp: $(RPC_PROBE_FILES) $(srcdir)/rpc/gensystemtap.pl - $(AM_V_GEN)$(PERL) -w $(srcdir)/rpc/gensystemtap.pl $(RPC_PROBE_FILES) > $@ +libvirt_functions.stp: $(RPC_PROBE_FILES) $(top_srcdir)/scripts/gensystemtap.py + $(AM_V_GEN)$(RUNUTF8) $(PYTHON) $(top_srcdir)/scripts/gensystemtap.py \ + $(RPC_PROBE_FILES) > $@ %_probes.stp: %_probes.d $(top_srcdir)/scripts/dtrace2systemtap.py \ $(top_builddir)/config.status diff --git a/src/rpc/Makefile.inc.am b/src/rpc/Makefile.inc.am index a5bde92c9f..11920b3270 100644 --- a/src/rpc/Makefile.inc.am +++ b/src/rpc/Makefile.inc.am @@ -3,7 +3,6 @@ EXTRA_DIST += \ rpc/gendispatch.pl \ rpc/genprotocol.pl \ - rpc/gensystemtap.pl \ rpc/virnetprotocol.x \ rpc/virkeepaliveprotocol.x \ $(NULL) diff --git a/src/rpc/gensystemtap.pl b/src/rpc/gensystemtap.pl deleted file mode 100755 index 6693d4d6f5..0000000000 --- a/src/rpc/gensystemtap.pl +++ /dev/null @@ -1,193 +0,0 @@ -#!/usr/bin/env perl -# -# Copyright (C) 2011-2012 Red Hat, Inc. -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2.1 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library. If not, see -# <http://www.gnu.org/licenses/>. -# -# Generate a set of systemtap functions for translating various -# RPC enum values into strings -# -# perl gensystemtap.pl */*.x > libvirt_functions.stp -# - -use strict; - -my %funcs; - -my %type; -my %status; -my %auth; - -my $instatus = 0; -my $intype = 0; -my $inauth = 0; -while (<>) { - if (/enum\s+virNetMessageType/) { - $intype = 1; - } elsif (/enum\s+virNetMessageStatus/) { - $instatus = 1; - } elsif (/enum remote_auth_type/) { - $inauth = 1; - } elsif (/}/) { - $instatus = $intype = $inauth = 0; - } elsif ($instatus) { - if (/^\s+VIR_NET_(\w+)\s*=\s*(\d+),?$/) { - $status{$2} = lc $1; - } - } elsif ($intype) { - if (/^\s+VIR_NET_(\w+)\s*=\s*(\d+),?$/) { - $type{$2} = lc $1; - } - } elsif ($inauth) { - if (/^\s+REMOTE_AUTH_(\w+)\s*=\s*(\d+),?$/) { - $auth{$2} = lc $1; - } - } else { - if (/(?:VIR_)?(\w+?)(?:_PROTOCOL)?_PROGRAM\s*=\s*0x([a-fA-F0-9]+)\s*;/) { - $funcs{lc $1} = { id => hex($2), version => undef, progs => [] }; - } elsif (/(?:VIR_)?(\w+?)(?:_PROTOCOL)?_(?:PROGRAM|PROTOCOL)_VERSION\s*=\s*(\d+)\s*;/) { - $funcs{lc $1}->{version} = $2; - } elsif (/(?:VIR_)?(\w+?)(?:_PROTOCOL)?_PROC_(.*?)\s+=\s+(\d+)/) { - $funcs{lc $1}->{progs}->[$3] = lc $2; - } - } -} - -print <<EOF; -function libvirt_rpc_auth_name(type, verbose) -{ -EOF -my $first = 1; -foreach my $type (sort(keys %auth)) { - my $cond = $first ? "if" : "} else if"; - $first = 0; - print " $cond (type == ", $type, ") {\n"; - print " typestr = \"", $auth{$type}, "\"\n"; -} -print <<EOF; - } else { - typestr = "unknown"; - verbose = 1; - } - if (verbose) { - typestr = typestr . sprintf(":%d", type) - } - return typestr; -} -EOF - -print <<EOF; -function libvirt_rpc_type_name(type, verbose) -{ -EOF -$first = 1; -foreach my $type (sort(keys %type)) { - my $cond = $first ? "if" : "} else if"; - $first = 0; - print " $cond (type == ", $type, ") {\n"; - print " typestr = \"", $type{$type}, "\"\n"; -} -print <<EOF; - } else { - typestr = "unknown"; - verbose = 1; - } - if (verbose) { - typestr = typestr . sprintf(":%d", type) - } - return typestr; -} -EOF - -print <<EOF; -function libvirt_rpc_status_name(status, verbose) -{ -EOF -$first = 1; -foreach my $status (sort(keys %status)) { - my $cond = $first ? "if" : "} else if"; - $first = 0; - print " $cond (status == ", $status, ") {\n"; - print " statusstr = \"", $status{$status}, "\"\n"; -} -print <<EOF; - } else { - statusstr = "unknown"; - verbose = 1; - } - if (verbose) { - statusstr = statusstr . sprintf(":%d", status) - } - return statusstr; -} -EOF - -print <<EOF; -function libvirt_rpc_program_name(program, verbose) -{ -EOF -$first = 1; -foreach my $prog (sort(keys %funcs)) { - my $cond = $first ? "if" : "} else if"; - $first = 0; - print " $cond (program == ", $funcs{$prog}->{id}, ") {\n"; - print " programstr = \"", $prog, "\"\n"; -} -print <<EOF; - } else { - programstr = "unknown"; - verbose = 1; - } - if (verbose) { - programstr = programstr . sprintf(":%d", program) - } - return programstr; -} -EOF - - -print <<EOF; -function libvirt_rpc_procedure_name(program, version, proc, verbose) -{ -EOF -$first = 1; -foreach my $prog (sort(keys %funcs)) { - my $cond = $first ? "if" : "} else if"; - $first = 0; - print " $cond (program == ", $funcs{$prog}->{id}, " && version == ", $funcs{$prog}->{version}, ") {\n"; - - my $pfirst = 1; - for (my $id = 1 ; $id <= $#{$funcs{$prog}->{progs}} ; $id++) { - my $cond = $pfirst ? "if" : "} else if"; - $pfirst = 0; - print " $cond (proc == $id) {\n"; - print " procstr = \"", $funcs{$prog}->{progs}->[$id], "\";\n"; - } - print " } else {\n"; - print " procstr = \"unknown\";\n"; - print " verbose = 1;\n"; - print " }\n"; -} -print <<EOF; - } else { - procstr = "unknown"; - verbose = 1; - } - if (verbose) { - procstr = procstr . sprintf(":%d", proc) - } - return procstr; -} -EOF -- 2.21.0

As part of an goal to eliminate Perl from libvirt build tools, rewrite the check-driverimpls.pl tool in Python. This was a straight conversion, manually going line-by-line to change the syntax from Perl to Python. Thus the overall structure of the file and approach is the same. Signed-off-by: Daniel P. Berrangé <berrange@redhat.com> --- Makefile.am | 1 + scripts/check-driverimpls.py | 102 +++++++++++++++++++++++++++++++++++ src/Makefile.am | 4 +- src/check-driverimpls.pl | 80 --------------------------- 4 files changed, 105 insertions(+), 82 deletions(-) create mode 100755 scripts/check-driverimpls.py delete mode 100755 src/check-driverimpls.pl diff --git a/Makefile.am b/Makefile.am index 7155ab6ce9..407a664626 100644 --- a/Makefile.am +++ b/Makefile.am @@ -48,6 +48,7 @@ EXTRA_DIST = \ scripts/augeas-gentest.py \ scripts/check-aclperms.py \ scripts/check-drivername.py \ + scripts/check-driverimpls.py \ scripts/check-spacing.py \ scripts/check-symfile.py \ scripts/check-symsorting.py \ diff --git a/scripts/check-driverimpls.py b/scripts/check-driverimpls.py new file mode 100755 index 0000000000..78d53e75a4 --- /dev/null +++ b/scripts/check-driverimpls.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python +# +# Copyright (C) 2013-2019 Red Hat, Inc. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see +# <http://www.gnu.org/licenses/>. +# + +from __future__ import print_function + +import re +import sys + + +def checkdriverimpls(filename): + intable = False + mainprefix = None + + errs = False + with open(filename, "r") as fh: + lineno = 0 + for line in fh: + lineno = lineno + 1 + if intable: + if line.find("}") != -1: + intable = False + mainprefix = None + continue + + m = re.search(r'''\.(\w+)\s*=\s*(\w+),?/''', line) + if m is not None: + api = m.group(1) + impl = m.group(2) + + if api in ["no", "name"]: + continue + if impl in ["NULL"]: + continue + + suffix = impl + prefix = re.sub(r'''^([a-z]+)(.*?)$''', r'''\1''', impl) + + if mainprefix is not None: + if mainprefix != prefix: + print("%s:%d Bad prefix '%s' for API '%s', " + + "expecting '%s'" % + (filename, lineno, prefix, api, mainprefix), + file=sys.stderr) + errs = True + else: + mainprefix = prefix + + if not api.startswith(mainprefix): + suffix = re.sub(r'''^[a-z]+''', "", suffix) + suffix = re.sub(r'''^([A-Z]+)''', + lambda m: m.group(1).lower(), suffix) + + if api != suffix: + want = api + if want.startswith("nwf"): + want = "NWF" + want[3:] + + if not api.startswith(mainprefix): + want = re.sub(r'''^([a-z])''', + lambda m: m.group(1).upper(), want) + want = mainprefix + want + + print("%s:%d Bad impl name '%s' for API " + + "'%s', expecting '%s'" % + (filename, lineno, impl, api, want), + file=sys.stderr) + errs = True + else: + m = re.search(r'''^(?:static\s+)?(vir(?:\w+)?Driver)''' + + r'''\s+(?!.*;)''', line) + if m is not None: + drv = m.group(1) + if drv in ["virNWFilterCallbackDriver", + "virNWFilterTechDriver", + "virConnectDriver"]: + continue + intable = True + + return errs + + +status = 0 +for filename in sys.argv[1:]: + if checkdriverimpls(filename): + status = 1 +sys.exit(status) diff --git a/src/Makefile.am b/src/Makefile.am index 55ad51abf1..62f1d55402 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -350,7 +350,7 @@ check-drivername: $(srcdir)/libvirt_lxc.syms check-driverimpls: - $(AM_V_GEN)$(PERL) $(srcdir)/check-driverimpls.pl \ + $(AM_V_GEN)$(RUNUTF8) $(PYTHON) $(top_srcdir)/scripts/check-driverimpls.py \ $(filter /%,$(DRIVER_SOURCE_FILES)) \ $(filter $(srcdir)/%,$(DRIVER_SOURCE_FILES)) \ $(addprefix $(srcdir)/,$(filter-out $(srcdir)/%, \ @@ -366,7 +366,7 @@ check-aclperms: $(srcdir)/access/viraccessperm.h \ $(srcdir)/access/viraccessperm.c -EXTRA_DIST += check-driverimpls.pl check-aclrules.pl +EXTRA_DIST += check-aclrules.pl check-local: check-protocol check-symfile check-symsorting \ check-drivername check-driverimpls check-aclrules \ diff --git a/src/check-driverimpls.pl b/src/check-driverimpls.pl deleted file mode 100755 index 3c0d54724c..0000000000 --- a/src/check-driverimpls.pl +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env perl -# -# Copyright (C) 2013 Red Hat, Inc. -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2.1 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library. If not, see -# <http://www.gnu.org/licenses/>. -# - -use strict; -use warnings; - -my $intable = 0; -my $table; -my $mainprefix; - -my $status = 0; -while (<>) { - if ($intable) { - if (/}/) { - $intable = 0; - $table = undef; - $mainprefix = undef; - } elsif (/\.(\w+)\s*=\s*(\w+),?/) { - my $api = $1; - my $impl = $2; - - next if $api eq "no"; - next if $api eq "name"; - next if $impl eq "NULL"; - - my $suffix = $impl; - my $prefix = $impl; - $prefix =~ s/^([a-z]+)(.*?)$/$1/; - - if (defined $mainprefix) { - if ($mainprefix ne $prefix) { - print "$ARGV:$. Bad prefix '$prefix' for API '$api', expecting '$mainprefix'\n"; - $status = 1; - } - } else { - $mainprefix = $prefix; - } - - if ($api !~ /^$mainprefix/) { - $suffix =~ s/^[a-z]+//; - $suffix =~ s/^([A-Z]+)/lc $1/e; - } - - if ($api ne $suffix) { - my $want = $api; - $want =~ s/^nwf/NWF/; - if ($api !~ /^$mainprefix/) { - $want =~ s/^([a-z])/uc $1/e; - $want = $mainprefix . $want; - } - print "$ARGV:$. Bad impl name '$impl' for API '$api', expecting '$want'\n"; - $status = 1; - } - } - } elsif (/^(?:static\s+)?(vir(?:\w+)?Driver)\s+(?!.*;)/) { - next if $1 eq "virNWFilterCallbackDriver" || - $1 eq "virNWFilterTechDriver" || - $1 eq "virConnectDriver"; - $intable = 1; - $table = $1; - } -} - -exit $status; -- 2.21.0

As part of an goal to eliminate Perl from libvirt build tools, rewrite the genpolkit.pl tool in Python. This was a straight conversion, manually going line-by-line to change the syntax from Perl to Python. Thus the overall structure of the file and approach is the same. Signed-off-by: Daniel P. Berrangé <berrange@redhat.com> --- Makefile.am | 1 + scripts/genpolkit.py | 122 +++++++++++++++++++++++++++++++++++++ src/access/Makefile.inc.am | 6 +- src/access/genpolkit.pl | 119 ------------------------------------ 4 files changed, 126 insertions(+), 122 deletions(-) create mode 100755 scripts/genpolkit.py delete mode 100755 src/access/genpolkit.pl diff --git a/Makefile.am b/Makefile.am index f28b07d814..e7ebe7281a 100644 --- a/Makefile.am +++ b/Makefile.am @@ -54,6 +54,7 @@ EXTRA_DIST = \ scripts/check-symfile.py \ scripts/check-symsorting.py \ scripts/dtrace2systemtap.py \ + scripts/genpolkit.py \ scripts/gensystemtap.py \ scripts/header-ifdef.py \ scripts/minimize-po.py \ diff --git a/scripts/genpolkit.py b/scripts/genpolkit.py new file mode 100755 index 0000000000..0cdba2bd3c --- /dev/null +++ b/scripts/genpolkit.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python +# +# Copyright (C) 2012-2019 Red Hat, Inc. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see +# <http://www.gnu.org/licenses/>. +# + +from __future__ import print_function + +import re +import sys + +objects = [ + "CONNECT", "DOMAIN", "INTERFACE", "NETWORK_PORT", + "NETWORK", "NODE_DEVICE", "NWFILTER_BINDING", + "NWFILTER", "SECRET", "STORAGE_POOL", "STORAGE_VOL", +] + +objectstr = "|".join(objects) + +# Data we're going to be generating looks like this +# +# <policyconfig> +# <action id="org.libvirt.unix.monitor"> +# <description>Monitor local virtualized systems</description> +# <message>System policy prevents monitoring of +# local virtualized systems</message> +# <defaults> +# <allow_any>yes</allow_any> +# <allow_inactive>yes</allow_inactive> +# <allow_active>yes</allow_active> +# </defaults> +# </action> +# ...more <action> rules... +# </policyconfig> + +opts = {} +in_opts = False + +perms = {} + +aclfile = sys.argv[1] +with open(aclfile, "r") as fh: + for line in fh: + if in_opts: + if line.find("*/") != -1: + in_opts = False + else: + m = re.search(r'''\*\s*\@(\w+):\s*(.*?)\s*$''', line) + if m is not None: + opts[m.group(1)] = m.group(2) + elif line.find("**") != -1: + in_opts = True + else: + m = re.search(r'''VIR_ACCESS_PERM_(%s)_((?:\w|_)+),''' % + objectstr, line) + if m is not None: + obj = m.group(1).lower() + perm = m.group(2).lower() + if perm == "last": + continue + + obj = obj.replace("_", "-") + perm = perm.replace("_", "-") + + if obj not in perms: + perms[obj] = {} + perms[obj][perm] = { + "desc": opts.get("desc", None), + "message": opts.get("message", None), + "anonymous": opts.get("anonymous", None), + } + opts = {} + +print('<?xml version="1.0" encoding="UTF-8"?>') +print('<!DOCTYPE policyconfig PUBLIC ' + + '"-//freedesktop//DTD polkit Policy Configuration 1.0//EN"') +print(' "http://www.freedesktop.org/software/polkit/policyconfig-1.dtd">') +print('<policyconfig>') +print(' <vendor>Libvirt Project</vendor>') +print(' <vendor_url>https://libvirt.org</vendor_url>') + +for obj in sorted(perms.keys()): + for perm in sorted(perms[obj].keys()): + description = perms[obj][perm]["desc"] + message = perms[obj][perm]["message"] + anonymous = perms[obj][perm]["anonymous"] + + if description is None: + raise Exception("missing description for %s.%s" % (obj, perm)) + if message is None: + raise Exception("missing message for %s.%s" % (obj, perm)) + + allow_any = "no" + if anonymous: + allow_any = "yes" + allow_inactive = allow_any + allow_active = allow_any + + print(' <action id="org.libvirt.api.%s.%s">' % (obj, perm)) + print(' <description>%s</description>' % description) + print(' <message>%s</message>' % message) + print(' <defaults>') + print(' <allow_any>%s</allow_any>' % allow_any) + print(' <allow_inactive>%s</allow_inactive>' % allow_inactive) + print(' <allow_active>%s</allow_active>' % allow_active) + print(' </defaults>') + print(' </action>') + +print('</policyconfig>') diff --git a/src/access/Makefile.inc.am b/src/access/Makefile.inc.am index fd0a5d8098..11f87c6aa7 100644 --- a/src/access/Makefile.inc.am +++ b/src/access/Makefile.inc.am @@ -43,7 +43,6 @@ ACCESS_DRIVER_POLKIT_POLICY = access/org.libvirt.api.policy GENERATED_SYM_FILES += $(ACCESS_DRIVER_SYM_FILES) EXTRA_DIST += \ - access/genpolkit.pl \ $(NULL) @@ -66,8 +65,9 @@ libvirt_driver_access_la_LIBADD = \ $(ACCESS_DRIVER_POLKIT_POLICY): $(srcdir)/access/viraccessperm.h \ - $(srcdir)/access/genpolkit.pl Makefile.am - $(AM_V_GEN)$(PERL) $(srcdir)/access/genpolkit.pl < $< > $@ || rm -f $@ + $(top_srcdir)/scripts/genpolkit.py Makefile.am + $(AM_V_GEN)$(RUNUTF8) $(PYTHON) \ + $(top_srcdir)/scripts/genpolkit.py $< > $@ || rm -f $@ if WITH_POLKIT libvirt_driver_access_la_SOURCES += $(ACCESS_DRIVER_POLKIT_SOURCES) diff --git a/src/access/genpolkit.pl b/src/access/genpolkit.pl deleted file mode 100755 index f8f20caf65..0000000000 --- a/src/access/genpolkit.pl +++ /dev/null @@ -1,119 +0,0 @@ -#!/usr/bin/env perl -# -# Copyright (C) 2012-2013 Red Hat, Inc. -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2.1 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library. If not, see -# <http://www.gnu.org/licenses/>. -# - -use strict; -use warnings; - -my @objects = ( - "CONNECT", "DOMAIN", "INTERFACE", "NETWORK_PORT", - "NETWORK","NODE_DEVICE", "NWFILTER_BINDING", "NWFILTER", - "SECRET", "STORAGE_POOL", "STORAGE_VOL", - ); - -my $objects = join ("|", @objects); - -# Data we're going to be generating looks like this -# -# <policyconfig> -# <action id="org.libvirt.unix.monitor"> -# <description>Monitor local virtualized systems</description> -# <message>System policy prevents monitoring of local virtualized systems</message> -# <defaults> -# <allow_any>yes</allow_any> -# <allow_inactive>yes</allow_inactive> -# <allow_active>yes</allow_active> -# </defaults> -# </action> -# ...more <action> rules... -# </policyconfig> - -my %opts; -my $in_opts = 0; - -my %perms; - -while (<>) { - if ($in_opts) { - if (m,\*/,) { - $in_opts = 0; - } elsif (/\*\s*\@(\w+):\s*(.*?)\s*$/) { - $opts{$1} = $2; - } - } elsif (m,/\*\*,) { - $in_opts = 1; - } elsif (/VIR_ACCESS_PERM_($objects)_((?:\w|_)+),/) { - my $object = lc $1; - my $perm = lc $2; - next if $perm eq "last"; - - $object =~ s/_/-/g; - $perm =~ s/_/-/g; - - $perms{$object} = {} unless exists $perms{$object}; - $perms{$object}->{$perm} = { - desc => $opts{desc}, - message => $opts{message}, - anonymous => $opts{anonymous} - }; - %opts = (); - } -} - -print <<EOF; -<?xml version="1.0" encoding="UTF-8"?> -<!DOCTYPE policyconfig PUBLIC "-//freedesktop//DTD polkit Policy Configuration 1.0//EN" - "http://www.freedesktop.org/software/polkit/policyconfig-1.dtd"> -<policyconfig> - <vendor>Libvirt Project</vendor> - <vendor_url>https://libvirt.org</vendor_url> -EOF - -foreach my $object (sort { $a cmp $b } keys %perms) { - foreach my $perm (sort { $a cmp $b } keys %{$perms{$object}}) { - my $description = $perms{$object}->{$perm}->{desc}; - my $message = $perms{$object}->{$perm}->{message}; - my $anonymous = $perms{$object}->{$perm}->{anonymous}; - - die "missing description for $object.$perm" unless - defined $description; - die "missing message for $object.$perm" unless - defined $message; - - my $allow_any = $anonymous ? "yes" : "no"; - my $allow_inactive = $allow_any; - my $allow_active = $allow_any; - - print <<EOF; - <action id="org.libvirt.api.$object.$perm"> - <description>$description</description> - <message>$message</message> - <defaults> - <allow_any>$allow_any</allow_any> - <allow_inactive>$allow_inactive</allow_inactive> - <allow_active>$allow_active</allow_active> - </defaults> - </action> -EOF - - } -} - -print <<EOF; -</policyconfig> -EOF -- 2.21.0

As part of an goal to eliminate Perl from libvirt build tools, rewrite the group-qemu-caps.pl tool in Python. This was a straight conversion, manually going line-by-line to change the syntax from Perl to Python. Thus the overall structure of the file and approach is the same. Signed-off-by: Daniel P. Berrangé <berrange@redhat.com> --- Makefile.am | 1 + build-aux/syntax-check.mk | 3 +- scripts/group-qemu-caps.py | 123 ++++++++++++++++++++++++++++++++++++ tests/group-qemu-caps.pl | 124 ------------------------------------- 4 files changed, 126 insertions(+), 125 deletions(-) create mode 100755 scripts/group-qemu-caps.py delete mode 100755 tests/group-qemu-caps.pl diff --git a/Makefile.am b/Makefile.am index 6f6cead526..769cd4ce64 100644 --- a/Makefile.am +++ b/Makefile.am @@ -57,6 +57,7 @@ EXTRA_DIST = \ scripts/dtrace2systemtap.py \ scripts/genpolkit.py \ scripts/gensystemtap.py \ + scripts/group-qemu-caps.py \ scripts/header-ifdef.py \ scripts/minimize-po.py \ scripts/mock-noinline.py \ diff --git a/build-aux/syntax-check.mk b/build-aux/syntax-check.mk index 7d54df182a..44639f499e 100644 --- a/build-aux/syntax-check.mk +++ b/build-aux/syntax-check.mk @@ -2176,7 +2176,8 @@ test-wrap-argv: $(PYTHON) $(top_srcdir)/scripts/test-wrap-argv.py --check group-qemu-caps: - $(AM_V_GEN)$(PERL) $(top_srcdir)/tests/group-qemu-caps.pl --check $(top_srcdir)/ + $(AM_V_GEN)$(RUNUTF8) $(PYTHON) $(top_srcdir)/scripts/group-qemu-caps.py \ + --check --prefix $(top_srcdir)/ # List all syntax-check exemptions: exclude_file_name_regexp--sc_avoid_strcase = ^tools/vsh\.h$$ diff --git a/scripts/group-qemu-caps.py b/scripts/group-qemu-caps.py new file mode 100755 index 0000000000..3edaf5d09f --- /dev/null +++ b/scripts/group-qemu-caps.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see +# <http://www.gnu.org/licenses/>. +# +# +# Regroup array values into smaller groups separated by numbered comments. +# +# If --check is the first parameter, the script will return +# a non-zero value if a file is not grouped correctly. +# Otherwise the files are regrouped in place. + +from __future__ import print_function + +import argparse +import re +import subprocess +import sys + + +def regroup_caps(check, filename, start_regex, end_regex, + trailing_newline, counter_prefix): + step = 5 + + original = [] + with open(filename, "r") as fh: + for line in fh: + original.append(line) + + fixed = [] + game_on = False + counter = 0 + for line in original: + line = line.rstrip("\n") + if game_on: + if re.search(r'''.*/\* [0-9]+ \*/.*''', line): + continue + if re.search(r'''^\s*$''', line): + continue + if counter % step == 0: + if counter != 0: + fixed.append("\n") + fixed.append("%s/* %d */\n" % (counter_prefix, counter)) + + if not (line.find("/*") != -1 and line.find("*/") == -1): + # count two-line comments as one line + counter = counter + 1 + + if re.search(start_regex, line): + game_on = True + elif game_on and re.search(end_regex, line): + if (counter - 1) % step == 0: + fixed = fixed[:-1] # /* $counter */ + if counter != 1: + fixed = fixed[:-1] # \n + + if trailing_newline: + fixed.append("\n") + + game_on = False + + fixed.append(line + "\n") + + if check: + orig = "".join(original) + new = "".join(fixed) + if new != orig: + diff = subprocess.Popen(["diff", "-u", filename, "-"], + stdin=subprocess.PIPE) + diff.communicate(input=new.encode('utf-8')) + + print("Incorrect line wrapping in $file", + file=sys.stderr) + print("Use test-wrap-argv.py to wrap test data files", + file=sys.stderr) + return False + else: + with open(filename, "w") as fh: + for line in fixed: + print(line, file=fh, end='') + + return True + + +parser = argparse.ArgumentParser(description='Test arg line wrapper') +parser.add_argument('--check', action="store_true", + help='check existing files only') +parser.add_argument('--prefix', default='', + help='source code tree prefix') +args = parser.parse_args() + +errs = False + +if not regroup_caps(args.check, + args.prefix + 'src/qemu/qemu_capabilities.c', + r'virQEMUCaps grouping marker', + r'\);', + 0, + " "): + errs = True + +if not regroup_caps(args.check, + args.prefix + 'src/qemu/qemu_capabilities.h', + r'virQEMUCapsFlags grouping marker', + r'QEMU_CAPS_LAST \/\* this must', + 1, + " "): + errs = True + +if errs: + sys.exit(1) +sys.exit(0) diff --git a/tests/group-qemu-caps.pl b/tests/group-qemu-caps.pl deleted file mode 100755 index 829e63a562..0000000000 --- a/tests/group-qemu-caps.pl +++ /dev/null @@ -1,124 +0,0 @@ -#!/usr/bin/env perl -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2.1 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library. If not, see -# <http://www.gnu.org/licenses/>. -# -# -# Regroup array values into smaller groups separated by numbered comments. -# -# If --check is the first parameter, the script will return -# a non-zero value if a file is not grouped correctly. -# Otherwise the files are regrouped in place. - -use strict; -use warnings; - -my $check = 0; - -if (defined $ARGV[0] && $ARGV[0] eq "--check") { - $check = 1; - shift @ARGV; -} - -my $prefix = ''; -if (defined $ARGV[0]) { - $prefix = $ARGV[0]; - shift @ARGV; -} - -my $ret = 0; -if (®roup_caps($prefix . 'src/qemu/qemu_capabilities.c', - 'virQEMUCaps grouping marker', - '\);', - 0, - " ") < 0) { - $ret = 1; -} -if (®roup_caps($prefix . 'src/qemu/qemu_capabilities.h', - 'virQEMUCapsFlags grouping marker', - 'QEMU_CAPS_LAST \/\* this must', - 1, - " ") < 0) { - $ret = 1; -} - -exit $ret; - -sub regroup_caps { - my $filename = shift; - my $start_regex = shift; - my $end_regex = shift; - my $trailing_newline = shift; - my $counter_prefix = shift; - my $step = 5; - - open FILE, '<', $filename or die "cannot open $filename: $!"; - my @original = <FILE>; - close FILE; - - my @fixed; - my $game_on = 0; - my $counter = 0; - foreach (@original) { - if ($game_on) { - next if ($_ =~ '/\* [0-9]+ \*/'); - next if (/^\s+$/); - if ($counter % $step == 0) { - if ($counter != 0) { - push @fixed, "\n"; - } - push @fixed, "$counter_prefix/* $counter */\n"; - } - if (!($_ =~ '/\*' && !($_ =~ '\*/'))) { - # count two-line comments as one line - $counter++; - } - } - if (/$start_regex/) { - $game_on = 1; - } elsif ($game_on && $_ =~ /$end_regex/) { - if (($counter -1) % $step == 0) { - pop @fixed; # /* $counter */ - if ($counter != 1) { - pop @fixed; # \n - } - } - if ($trailing_newline) { - push @fixed, "\n"; - } - $game_on = 0; - } - push @fixed, $_; - } - - if ($check) { - my $nl = join('', @fixed); - my $ol = join('', @original); - unless ($nl eq $ol) { - open DIFF, "| diff -u $filename -" or die "cannot run diff: $!"; - print DIFF $nl; - close DIFF; - - print STDERR "Incorrect array grouping in $filename\n"; - print STDERR "Use group-qemu-caps.pl to group long array members\n"; - return -1; - } - } else { - open FILE, '>', $filename or die "cannot open $filename: $!"; - foreach my $line (@fixed) { - print FILE $line; - } - close FILE; - } -} -- 2.21.0

As part of an goal to eliminate Perl from libvirt build tools, rewrite the check-file-access.pl tool in Python. This was a straight conversion, manually going line-by-line to change the syntax from Perl to Python. Thus the overall structure of the file and approach is the same. Signed-off-by: Daniel P. Berrangé <berrange@redhat.com> --- Makefile.am | 1 + scripts/check-file-access.py | 123 +++++++++++++++++++++++++++++++ tests/Makefile.am | 3 +- tests/check-file-access.pl | 126 -------------------------------- tests/file_access_whitelist.txt | 2 +- 5 files changed, 126 insertions(+), 129 deletions(-) create mode 100755 scripts/check-file-access.py delete mode 100755 tests/check-file-access.pl diff --git a/Makefile.am b/Makefile.am index 769cd4ce64..19114069e3 100644 --- a/Makefile.am +++ b/Makefile.am @@ -50,6 +50,7 @@ EXTRA_DIST = \ scripts/check-aclrules.py \ scripts/check-drivername.py \ scripts/check-driverimpls.py \ + scripts/check-file-access.py \ scripts/check-remote-protocol.py \ scripts/check-spacing.py \ scripts/check-symfile.py \ diff --git a/scripts/check-file-access.py b/scripts/check-file-access.py new file mode 100755 index 0000000000..cdcbf2666f --- /dev/null +++ b/scripts/check-file-access.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python +# +# Copyright (C) 2016-2019 Red Hat, Inc. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see +# <http://www.gnu.org/licenses/>. +# +# This script is supposed to check test_file_access.txt file and +# warn about file accesses outside our working tree. +# +# + +from __future__ import print_function + +import re +import sys + +access_file = "test_file_access.txt" +whitelist_file = "file_access_whitelist.txt" + +known_actions = ["open", "fopen", "access", "stat", "lstat", "connect"] + +files = [] +whitelist = [] + +with open(access_file, "r") as fh: + for line in fh: + line = line.rstrip("\n") + + m = re.search(r'''^(\S*):\s*(\S*):\s*(\S*)(\s*:\s*(.*))?$''', line) + if m is not None: + rec = { + "path": m.group(1), + "action": m.group(2), + "progname": m.group(3), + "testname": m.group(5), + } + files.append(rec) + else: + raise Exception("Malformed line %s" % line) + +with open(whitelist_file, "r") as fh: + for line in fh: + line = line.rstrip("\n") + + if re.search(r'''^\s*#.*$''', line): + continue # comment + if line == "": + continue + + m = re.search(r'''^(\S*):\s*(\S*)(:\s*(\S*)(\s*:\s*(.*))?)?$''', line) + if m is not None and m.group(2) in known_actions: + # $path: $action: $progname: $testname + rec = { + "path": m.group(1), + "action": m.group(3), + "progname": m.group(4), + "testname": m.group(6), + } + whitelist.append(rec) + else: + m = re.search(r'''^(\S*)(:\s*(\S*)(\s*:\s*(.*))?)?$''', line) + if m is not None: + # $path: $progname: $testname + rec = { + "path": m.group(1), + "action": None, + "progname": m.group(3), + "testname": m.group(5), + } + whitelist.append(rec) + else: + raise Exception("Malformed line %s" % line) + + +# Now we should check if %traces is included in $whitelist. For +# now checking just keys is sufficient +err = False +for file in files: + match = False + + for rule in whitelist: + if not re.search("^" + rule["path"], file["path"]): + continue + + if (rule["action"] is not None and + not re.search("^" + rule["action"], file["action"])): + continue + + if (rule["progname"] is not None and + not re.search("^" + rule["progname"], file["progname"])): + continue + + if (rule["testname"] is not None and + file["testname"] is not None and + not re.search("^" + rule["testname"], file["testname"])): + continue + + match = True + + if not match: + err = True + print("%s: %s: %s" % + (file["path"], file["action"], file["progname"]), + file=sys.stderr, end="") + if file["testname"] is not None: + print(": %s" % file["testname"], file=sys.stderr, end="") + print("", file=sys.stderr) + +if err: + sys.exit(1) +sys.exit(0) diff --git a/tests/Makefile.am b/tests/Makefile.am index 9d9c582e42..c3bca26019 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -453,14 +453,13 @@ EXTRA_DIST += $(test_scripts) if WITH_LINUX check-access: file-access-clean VIR_TEST_FILE_ACCESS=1 $(MAKE) $(AM_MAKEFLAGS) check - $(PERL) check-file-access.pl | sort -u + $(RUNUTF8) $(PYTHON) $(top_srcdir)/scripts/check-file-access.py | sort -u file-access-clean: > test_file_access.txt endif WITH_LINUX EXTRA_DIST += \ - check-file-access.pl \ file_access_whitelist.txt if WITH_TESTS diff --git a/tests/check-file-access.pl b/tests/check-file-access.pl deleted file mode 100755 index ea0b7a18a2..0000000000 --- a/tests/check-file-access.pl +++ /dev/null @@ -1,126 +0,0 @@ -#!/usr/bin/env perl -# -# Copyright (C) 2016 Red Hat, Inc. -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2.1 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library. If not, see -# <http://www.gnu.org/licenses/>. -# -# This script is supposed to check test_file_access.txt file and -# warn about file accesses outside our working tree. -# -# - -use strict; -use warnings; - -my $access_file = "test_file_access.txt"; -my $whitelist_file = "file_access_whitelist.txt"; - -my @known_actions = ("open", "fopen", "access", "stat", "lstat", "connect"); - -my @files; -my @whitelist; - -open FILE, "<", $access_file or die "Unable to open $access_file: $!"; -while (<FILE>) { - chomp; - if (/^(\S*):\s*(\S*):\s*(\S*)(\s*:\s*(.*))?$/) { - my %rec; - ${rec}{path} = $1; - ${rec}{action} = $2; - ${rec}{progname} = $3; - if (defined $5) { - ${rec}{testname} = $5; - } - push (@files, \%rec); - } else { - die "Malformed line $_"; - } -} -close FILE; - -open FILE, "<", $whitelist_file or die "Unable to open $whitelist_file: $!"; -while (<FILE>) { - chomp; - if (/^\s*#.*$/) { - # comment - } elsif (/^(\S*):\s*(\S*)(:\s*(\S*)(\s*:\s*(.*))?)?$/ and - grep /^$2$/, @known_actions) { - # $path: $action: $progname: $testname - my %rec; - ${rec}{path} = $1; - ${rec}{action} = $3; - if (defined $4) { - ${rec}{progname} = $4; - } - if (defined $6) { - ${rec}{testname} = $6; - } - push (@whitelist, \%rec); - } elsif (/^(\S*)(:\s*(\S*)(\s*:\s*(.*))?)?$/) { - # $path: $progname: $testname - my %rec; - ${rec}{path} = $1; - if (defined $3) { - ${rec}{progname} = $3; - } - if (defined $5) { - ${rec}{testname} = $5; - } - push (@whitelist, \%rec); - } else { - die "Malformed line $_"; - } -} -close FILE; - -# Now we should check if %traces is included in $whitelist. For -# now checking just keys is sufficient -my $error = 0; -for my $file (@files) { - my $match = 0; - - for my $rule (@whitelist) { - if (not %${file}{path} =~ m/^$rule->{path}$/) { - next; - } - - if (defined %${rule}{action} and - not %${file}{action} =~ m/^$rule->{action}$/) { - next; - } - - if (defined %${rule}{progname} and - not %${file}{progname} =~ m/^$rule->{progname}$/) { - next; - } - - if (defined %${rule}{testname} and - defined %${file}{testname} and - not %${file}{testname} =~ m/^$rule->{testname}$/) { - next; - } - - $match = 1; - } - - if (not $match) { - $error = 1; - print "$file->{path}: $file->{action}: $file->{progname}"; - print ": $file->{testname}" if defined %${file}{testname}; - print "\n"; - } -} - -exit $error; diff --git a/tests/file_access_whitelist.txt b/tests/file_access_whitelist.txt index 3fb318cbab..5ec7ee63bb 100644 --- a/tests/file_access_whitelist.txt +++ b/tests/file_access_whitelist.txt @@ -5,7 +5,7 @@ # $path: $progname: $testname # $path: $action: $progname: $testname # -# All these variables are evaluated as perl RE. So to allow +# All these variables are evaluated as python RE. So to allow # /dev/sda and /dev/sdb, you can just '/dev/sd[a-b]', or to allow # /proc/$pid/status you can '/proc/\d+/status' and so on. # Moreover, $action, $progname and $testname can be empty, in which -- 2.21.0

As part of an goal to eliminate Perl from libvirt build tools, rewrite the hvsupport.pl tool in Python. This was a straight conversion, manually going line-by-line to change the syntax from Perl to Python. Thus the overall structure of the file and approach is the same. The new impl generates byte-for-byte identical output to the old impl. Signed-off-by: Daniel P. Berrangé <berrange@redhat.com> --- Makefile.am | 1 + docs/Makefile.am | 7 +- docs/hvsupport.pl | 459 --------------------------------------- scripts/hvsupport.py | 504 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 508 insertions(+), 463 deletions(-) delete mode 100755 docs/hvsupport.pl create mode 100755 scripts/hvsupport.py diff --git a/Makefile.am b/Makefile.am index 19114069e3..bc8aecc4a2 100644 --- a/Makefile.am +++ b/Makefile.am @@ -60,6 +60,7 @@ EXTRA_DIST = \ scripts/gensystemtap.py \ scripts/group-qemu-caps.py \ scripts/header-ifdef.py \ + scripts/hvsupport.py \ scripts/minimize-po.py \ scripts/mock-noinline.py \ scripts/prohibit-duplicate-header.py \ diff --git a/docs/Makefile.am b/docs/Makefile.am index 0311bbedd8..cf83949168 100644 --- a/docs/Makefile.am +++ b/docs/Makefile.am @@ -172,7 +172,6 @@ EXTRA_DIST= \ $(internals_html_in) $(fonts) \ $(kbase_html_in) \ aclperms.htmlinc \ - hvsupport.pl \ $(schema_DATA) acl_generated = aclperms.htmlinc @@ -212,12 +211,12 @@ web: $(dot_html) $(internals_html) $(kbase_html) \ hvsupport.html: hvsupport.html.in -hvsupport.html.in: $(srcdir)/hvsupport.pl $(api_DATA) \ +hvsupport.html.in: $(top_srcdir)/scripts/hvsupport.py $(api_DATA) \ $(top_srcdir)/src/libvirt_public.syms \ $(top_srcdir)/src/libvirt_qemu.syms $(top_srcdir)/src/libvirt_lxc.syms \ $(top_srcdir)/src/driver.h - $(AM_V_GEN)$(PERL) $(srcdir)/hvsupport.pl $(top_srcdir) $(top_builddir) > $@ \ - || { rm $@ && exit 1; } + $(AM_V_GEN)$(RUNUTF8) $(PYTHON) $(top_srcdir)/scripts/hvsupport.py \ + $(top_srcdir) $(top_builddir) > $@ || { rm $@ && exit 1; } news.html.in: \ $(srcdir)/news.xml \ diff --git a/docs/hvsupport.pl b/docs/hvsupport.pl deleted file mode 100755 index 0977098eac..0000000000 --- a/docs/hvsupport.pl +++ /dev/null @@ -1,459 +0,0 @@ -#!/usr/bin/env perl - -use strict; -use warnings; - -use File::Find; - -die "syntax: $0 SRCDIR BUILDDIR\n" unless int(@ARGV) == 2; - -my $srcdir = shift @ARGV; -my $builddir = shift @ARGV; - -my $symslibvirt = "$srcdir/src/libvirt_public.syms"; -my $symsqemu = "$srcdir/src/libvirt_qemu.syms"; -my $symslxc = "$srcdir/src/libvirt_lxc.syms"; -my @drivertable = ( - "$srcdir/src/driver-hypervisor.h", - "$srcdir/src/driver-interface.h", - "$srcdir/src/driver-network.h", - "$srcdir/src/driver-nodedev.h", - "$srcdir/src/driver-nwfilter.h", - "$srcdir/src/driver-secret.h", - "$srcdir/src/driver-state.h", - "$srcdir/src/driver-storage.h", - "$srcdir/src/driver-stream.h", - ); - -my %groupheaders = ( - "virHypervisorDriver" => "Hypervisor APIs", - "virNetworkDriver" => "Virtual Network APIs", - "virInterfaceDriver" => "Host Interface APIs", - "virNodeDeviceDriver" => "Host Device APIs", - "virStorageDriver" => "Storage Pool APIs", - "virSecretDriver" => "Secret APIs", - "virNWFilterDriver" => "Network Filter APIs", - ); - - -my @srcs; -find({ - wanted => sub { - if (m!$srcdir/src/.*/\w+_(driver|common|tmpl|monitor|hal|udev)\.c$!) { - push @srcs, $_ if $_ !~ /vbox_driver\.c/; - } - }, no_chdir => 1}, "$srcdir/src"); - -# Map API functions to the header and documentation files they're in -# so that we can generate proper hyperlinks to their documentation. -# -# The function names are grep'd from the XML output of apibuild.py. -sub getAPIFilenames { - my $filename = shift; - - my %files; - my $line; - - open FILE, "<", $filename or die "cannot read $filename: $!"; - - while (defined($line = <FILE>)) { - if ($line =~ /function name='([^']+)' file='([^']+)'/) { - $files{$1} = $2; - } - } - - close FILE; - - if (keys %files == 0) { - die "No functions found in $filename. Has the apibuild.py output changed?"; - } - return \%files; -} - -sub parseSymsFile { - my $apisref = shift; - my $prefix = shift; - my $filename = shift; - my $xmlfilename = shift; - - my $line; - my $vers; - my $prevvers; - - my $filenames = getAPIFilenames($xmlfilename); - - open FILE, "<$filename" - or die "cannot read $filename: $!"; - - while (defined($line = <FILE>)) { - chomp $line; - next if $line =~ /^\s*#/; - next if $line =~ /^\s*$/; - next if $line =~ /^\s*(global|local):/; - if ($line =~ /^\s*${prefix}_(\d+\.\d+\.\d+)\s*{\s*$/) { - if (defined $vers) { - die "malformed syms file"; - } - $vers = $1; - } elsif ($line =~ /\s*}\s*;\s*$/) { - if (defined $prevvers) { - die "malformed syms file"; - } - $prevvers = $vers; - $vers = undef; - } elsif ($line =~ /\s*}\s*${prefix}_(\d+\.\d+\.\d+)\s*;\s*$/) { - if ($1 ne $prevvers) { - die "malformed syms file $1 != $vers"; - } - $prevvers = $vers; - $vers = undef; - } elsif ($line =~ /\s*(\w+)\s*;\s*$/) { - $$apisref{$1} = {}; - $$apisref{$1}->{vers} = $vers; - $$apisref{$1}->{file} = $$filenames{$1}; - } else { - die "unexpected data $line\n"; - } - } - - close FILE; -} - -my %apis; -# Get the list of all public APIs and their corresponding version -parseSymsFile(\%apis, "LIBVIRT", $symslibvirt, "$builddir/docs/libvirt-api.xml"); - -# And the same for the QEMU specific APIs -parseSymsFile(\%apis, "LIBVIRT_QEMU", $symsqemu, "$builddir/docs/libvirt-qemu-api.xml"); - -# And the same for the LXC specific APIs -parseSymsFile(\%apis, "LIBVIRT_LXC", $symslxc, "$builddir/docs/libvirt-lxc-api.xml"); - - -# Some special things which aren't public APIs, -# but we want to report -$apis{virConnectSupportsFeature}->{vers} = "0.3.2"; -$apis{virDomainMigratePrepare}->{vers} = "0.3.2"; -$apis{virDomainMigratePerform}->{vers} = "0.3.2"; -$apis{virDomainMigrateFinish}->{vers} = "0.3.2"; -$apis{virDomainMigratePrepare2}->{vers} = "0.5.0"; -$apis{virDomainMigrateFinish2}->{vers} = "0.5.0"; -$apis{virDomainMigratePrepareTunnel}->{vers} = "0.7.2"; - -$apis{virDomainMigrateBegin3}->{vers} = "0.9.2"; -$apis{virDomainMigratePrepare3}->{vers} = "0.9.2"; -$apis{virDomainMigratePrepareTunnel3}->{vers} = "0.9.2"; -$apis{virDomainMigratePerform3}->{vers} = "0.9.2"; -$apis{virDomainMigrateFinish3}->{vers} = "0.9.2"; -$apis{virDomainMigrateConfirm3}->{vers} = "0.9.2"; - -$apis{virDomainMigrateBegin3Params}->{vers} = "1.1.0"; -$apis{virDomainMigratePrepare3Params}->{vers} = "1.1.0"; -$apis{virDomainMigratePrepareTunnel3Params}->{vers} = "1.1.0"; -$apis{virDomainMigratePerform3Params}->{vers} = "1.1.0"; -$apis{virDomainMigrateFinish3Params}->{vers} = "1.1.0"; -$apis{virDomainMigrateConfirm3Params}->{vers} = "1.1.0"; - - - -# Now we want to get the mapping between public APIs -# and driver struct fields. This lets us later match -# update the driver impls with the public APis. - -my $line; - -# Group name -> hash of APIs { fields -> api name } -my %groups; -my $ingrp; -foreach my $drivertable (@drivertable) { - open FILE, "<$drivertable" - or die "cannot read $drivertable: $!"; - - while (defined($line = <FILE>)) { - if ($line =~ /struct _(vir\w*Driver)/) { - my $grp = $1; - if ($grp ne "virStateDriver" && - $grp ne "virStreamDriver") { - $ingrp = $grp; - $groups{$ingrp} = { apis => {}, drivers => {} }; - } - } elsif ($ingrp) { - if ($line =~ /^\s*vir(?:Drv)(\w+)\s+(\w+);\s*$/) { - my $field = $2; - my $name = $1; - - my $api; - if (exists $apis{"vir$name"}) { - $api = "vir$name"; - } elsif ($name =~ /\w+(Open|Close|URIProbe)/) { - next; - } else { - die "driver $name does not have a public API"; - } - $groups{$ingrp}->{apis}->{$field} = $api; - } elsif ($line =~ /};/) { - $ingrp = undef; - } - } - } - - close FILE; -} - - -# Finally, we read all the primary driver files and extract -# the driver API tables from each one. - -foreach my $src (@srcs) { - open FILE, "<$src" or - die "cannot read $src: $!"; - - my $groups_regex = join("|", keys %groups); - $ingrp = undef; - my $impl; - while (defined($line = <FILE>)) { - if (!$ingrp) { - # skip non-matching lines early to save time - next if not $line =~ /$groups_regex/; - - if ($line =~ /^\s*(?:static\s+)?($groups_regex)\s+(\w+)\s*=\s*{/ || - $line =~ /^\s*(?:static\s+)?($groups_regex)\s+NAME\(\w+\)\s*=\s*{/) { - $ingrp = $1; - $impl = $src; - - if ($impl =~ m,.*/node_device_(\w+)\.c,) { - $impl = $1; - } else { - $impl =~ s,.*/(\w+?)_((\w+)_)?(\w+)\.c,$1,; - } - - if ($groups{$ingrp}->{drivers}->{$impl}) { - die "Group $ingrp already contains $impl"; - } - - $groups{$ingrp}->{drivers}->{$impl} = {}; - } - - } else { - if ($line =~ m!\s*\.(\w+)\s*=\s*(\w+)\s*,?\s*(?:/\*\s*(\d+\.\d+\.\d+)\s*(?:-\s*(\d+\.\d+\.\d+))?\s*\*/\s*)?$!) { - my $api = $1; - my $meth = $2; - my $vers = $3; - my $deleted = $4; - - next if $api eq "no" || $api eq "name"; - - if ($meth eq "NULL" && !defined $deleted) { - die "Method impl for $api is NULL, but no deleted version is provided"; - } - if ($meth ne "NULL" && defined $deleted) { - die "Method impl for $api is non-NULL, but deleted version is provided"; - } - - die "Method $meth in $src is missing version" unless defined $vers || $api eq "connectURIProbe"; - - if (!exists($groups{$ingrp}->{apis}->{$api})) { - next if $api =~ /\w(Open|Close|URIProbe)/; - - die "Found unexpected method $api in $ingrp\n"; - } - - $groups{$ingrp}->{drivers}->{$impl}->{$api} = {}; - $groups{$ingrp}->{drivers}->{$impl}->{$api}->{vers} = $vers; - $groups{$ingrp}->{drivers}->{$impl}->{$api}->{deleted} = $deleted; - if ($api eq "domainMigratePrepare" || - $api eq "domainMigratePrepare2" || - $api eq "domainMigratePrepare3") { - if (!$groups{$ingrp}->{drivers}->{$impl}->{"domainMigrate"}) { - $groups{$ingrp}->{drivers}->{$impl}->{"domainMigrate"} = {}; - $groups{$ingrp}->{drivers}->{$impl}->{"domainMigrate"}->{vers} = $vers; - } - } - - } elsif ($line =~ /}/) { - $ingrp = undef; - } - } - } - - close FILE; -} - - -# The '.open' driver method is used for 3 public APIs, so we -# have a bit of manual fixup todo with the per-driver versioning -# and support matrix - -$groups{virHypervisorDriver}->{apis}->{"openAuth"} = "virConnectOpenAuth"; -$groups{virHypervisorDriver}->{apis}->{"openReadOnly"} = "virConnectOpenReadOnly"; -$groups{virHypervisorDriver}->{apis}->{"domainMigrate"} = "virDomainMigrate"; - -my $openAuthVers = (0 * 1000 * 1000) + (4 * 1000) + 0; - -foreach my $drv (keys %{$groups{"virHypervisorDriver"}->{drivers}}) { - my $openVersStr = $groups{"virHypervisorDriver"}->{drivers}->{$drv}->{"connectOpen"}->{vers}; - my $openVers; - if ($openVersStr =~ /(\d+)\.(\d+)\.(\d+)/) { - $openVers = ($1 * 1000 * 1000) + ($2 * 1000) + $3; - } - - # virConnectOpenReadOnly always matches virConnectOpen version - $groups{"virHypervisorDriver"}->{drivers}->{$drv}->{"connectOpenReadOnly"} = - $groups{"virHypervisorDriver"}->{drivers}->{$drv}->{"connectOpen"}; - - $groups{"virHypervisorDriver"}->{drivers}->{$drv}->{"connectOpenAuth"} = {}; - - # virConnectOpenAuth is always 0.4.0 if the driver existed - # before this time, otherwise it matches the version of - # the driver's virConnectOpen entry - if ($openVersStr eq "Y" || - $openVers >= $openAuthVers) { - $groups{"virHypervisorDriver"}->{drivers}->{$drv}->{"connectOpenAuth"}->{vers} = $openVersStr; - } else { - $groups{"virHypervisorDriver"}->{drivers}->{$drv}->{"connectOpenAuth"}->{vers} = "0.4.0"; - } -} - - -# Another special case for the virDomainCreateLinux which was replaced -# with virDomainCreateXML -$groups{virHypervisorDriver}->{apis}->{"domainCreateLinux"} = "virDomainCreateLinux"; - -my $createAPIVers = (0 * 1000 * 1000) + (0 * 1000) + 3; - -foreach my $drv (keys %{$groups{"virHypervisorDriver"}->{drivers}}) { - my $createVersStr = $groups{"virHypervisorDriver"}->{drivers}->{$drv}->{"domainCreateXML"}->{vers}; - next unless defined $createVersStr; - my $createVers; - if ($createVersStr =~ /(\d+)\.(\d+)\.(\d+)/) { - $createVers = ($1 * 1000 * 1000) + ($2 * 1000) + $3; - } - - $groups{"virHypervisorDriver"}->{drivers}->{$drv}->{"domainCreateLinux"} = {}; - - # virCreateLinux is always 0.0.3 if the driver existed - # before this time, otherwise it matches the version of - # the driver's virCreateXML entry - if ($createVersStr eq "Y" || - $createVers >= $createAPIVers) { - $groups{"virHypervisorDriver"}->{drivers}->{$drv}->{"domainCreateLinux"}->{vers} = $createVersStr; - } else { - $groups{"virHypervisorDriver"}->{drivers}->{$drv}->{"domainCreateLinux"}->{vers} = "0.0.3"; - } -} - - -# Finally we generate the HTML file with the tables - -print <<EOF; -<?xml version="1.0" encoding="UTF-8"?> -<!DOCTYPE html> -<html xmlns="http://www.w3.org/1999/xhtml"> -<body class="hvsupport"> -<h1>libvirt API support matrix</h1> - -<ul id="toc"></ul> - -<p> -This page documents which <a href="html/">libvirt calls</a> work on -which libvirt drivers / hypervisors, and which version the API appeared -in. If a hypervisor driver later dropped support for the API, the version -when it was removed is also mentioned (highlighted in -<span class="removedhv">dark red</span>). -</p> - -EOF - - foreach my $grp (sort { $a cmp $b } keys %groups) { - print "<h2><a id=\"$grp\">", $groupheaders{$grp}, "</a></h2>\n"; - print <<EOF; -<table class="top_table"> -<thead> -<tr> -<th>API</th> -<th>Version</th> -EOF - - foreach my $drv (sort { $a cmp $b } keys %{$groups{$grp}->{drivers}}) { - print " <th>$drv</th>\n"; - } - - print <<EOF; -</tr> -</thead> -<tbody> -EOF - - my $row = 0; - foreach my $field (sort { - $groups{$grp}->{apis}->{$a} - cmp - $groups{$grp}->{apis}->{$b} - } keys %{$groups{$grp}->{apis}}) { - my $api = $groups{$grp}->{apis}->{$field}; - my $vers = $apis{$api}->{vers}; - my $htmlgrp = $apis{$api}->{file}; - print <<EOF; -<tr> -<td> -EOF - - if (defined $htmlgrp) { - print <<EOF; -<a href=\"html/libvirt-$htmlgrp.html#$api\">$api</a> -EOF - - } else { - print $api; - } - print <<EOF; -</td> -<td>$vers</td> -EOF - - foreach my $drv (sort {$a cmp $b } keys %{$groups{$grp}->{drivers}}) { - print "<td>"; - if (exists $groups{$grp}->{drivers}->{$drv}->{$field}) { - if ($groups{$grp}->{drivers}->{$drv}->{$field}->{vers}) { - print $groups{$grp}->{drivers}->{$drv}->{$field}->{vers}; - } - if ($groups{$grp}->{drivers}->{$drv}->{$field}->{deleted}) { - print " - <span class=\"removedhv\">", $groups{$grp}->{drivers}->{$drv}->{$field}->{deleted}, "</span>"; - } - } - print "</td>\n"; - } - - print <<EOF; -</tr> -EOF - - $row++; - if (($row % 15) == 0) { - print <<EOF; -<tr> -<th>API</th> -<th>Version</th> -EOF - - foreach my $drv (sort { $a cmp $b } keys %{$groups{$grp}->{drivers}}) { - print " <th>$drv</th>\n"; - } - - print <<EOF; -</tr> -EOF - } - - } - - print <<EOF; -</tbody> -</table> -EOF -} - -print <<EOF; -</body> -</html> -EOF diff --git a/scripts/hvsupport.py b/scripts/hvsupport.py new file mode 100755 index 0000000000..7900f86f0f --- /dev/null +++ b/scripts/hvsupport.py @@ -0,0 +1,504 @@ +#!/usr/bin/env python +# +# Copyright (C) 2011-2019 Red Hat, Inc. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see +# <http://www.gnu.org/licenses/>. + +from __future__ import print_function + +import sys +import os.path +import re + +if len(sys.argv) != 3: + print("syntax: %s TOP-SRCDIR TOP-BUILDDIR\n" % sys.argv[0], file=sys.stderr) + +srcdir = sys.argv[1] +builddir = sys.argv[2] + +symslibvirt = os.path.join(srcdir, "src", "libvirt_public.syms") +symsqemu = os.path.join(srcdir, "src", "libvirt_qemu.syms") +symslxc = os.path.join(srcdir, "src", "libvirt_lxc.syms") +drivertablefiles = [ + os.path.join(srcdir, "src", "driver-hypervisor.h"), + os.path.join(srcdir, "src", "driver-interface.h"), + os.path.join(srcdir, "src", "driver-network.h"), + os.path.join(srcdir, "src", "driver-nodedev.h"), + os.path.join(srcdir, "src", "driver-nwfilter.h"), + os.path.join(srcdir, "src", "driver-secret.h"), + os.path.join(srcdir, "src", "driver-state.h"), + os.path.join(srcdir, "src", "driver-storage.h"), + os.path.join(srcdir, "src", "driver-stream.h"), +] + +groupheaders = { + "virHypervisorDriver": "Hypervisor APIs", + "virNetworkDriver": "Virtual Network APIs", + "virInterfaceDriver": "Host Interface APIs", + "virNodeDeviceDriver": "Host Device APIs", + "virStorageDriver": "Storage Pool APIs", + "virSecretDriver": "Secret APIs", + "virNWFilterDriver": "Network Filter APIs", +} + + +srcs = [] +for root, dirs, files in os.walk(os.path.join(srcdir, "src")): + for file in files: + if ((file.endswith("driver.c") and + not file.endswith("vbox_driver.c")) or + file.endswith("common.c") or + file.endswith("tmpl.c") or + file.endswith("monitor.c") or + file.endswith("hal.c") or + file.endswith("udev.c")): + srcs.append(os.path.join(root, file)) + + +# Map API functions to the header and documentation files they're in +# so that we can generate proper hyperlinks to their documentation. +# +# The function names are grep'd from the XML output of apibuild.py. +def getAPIFilenames(filename): + files = {} + + with open(filename) as fh: + for line in fh: + res = re.search(r"function name='([^']+)' file='([^']+)'", line) + if res is not None: + files[res.group(1)] = res.group(2) + + if len(files) == 0: + raise Exception(("No functions found in %s. " + + "Has the apibuild.py output changed?") % + filename) + + return files + + +def parseSymsFile(apisref, prefix, filename, xmlfilename): + vers = None + prevvers = None + + filenames = getAPIFilenames(xmlfilename) + + with open(filename) as fh: + for line in fh: + line = line.strip() + + if line == "": + continue + if line[0] == '#': + continue + if line.startswith("global:"): + continue + if line.startswith("local:"): + continue + + groupstartmatch = re.search(r"^\s*%s_(\d+\.\d+\.\d+)\s*{\s*$" % + prefix, line) + groupendmatch1 = re.search(r"^\s*}\s*;\s*$", line) + groupendmatch2 = re.search(r"^\s*}\s*%s_(\d+\.\d+\.\d+)\s*;\s*$" % + prefix, line) + symbolmatch = re.search(r"^\s*(\w+)\s*;\s*$", line) + if groupstartmatch is not None: + if vers is not None: + raise Exception("malformed syms file when starting group") + + vers = groupstartmatch.group(1) + elif groupendmatch1 is not None: + if prevvers is not None: + raise Exception("malformed syms file when ending group") + + prevvers = vers + vers = None + elif groupendmatch2 is not None: + if groupendmatch2.group(1) != prevvers: + raise Exception("malformed syms file %s != %s " + + "when ending group" % + (groupendmatch2.group(1), prevvers)) + + prevvers = vers + vers = None + elif symbolmatch is not None: + name = symbolmatch.group(1) + apisref[name] = { + "vers": vers, + "file": filenames.get(name), + } + else: + raise Exception("unexpected data %s" % line) + + +apis = {} +# Get the list of all public APIs and their corresponding version +parseSymsFile(apis, "LIBVIRT", symslibvirt, + os.path.join(builddir, "docs", "libvirt-api.xml")) + +# And the same for the QEMU specific APIs +parseSymsFile(apis, "LIBVIRT_QEMU", symsqemu, + os.path.join(builddir, "docs", "libvirt-qemu-api.xml")) + +# And the same for the LXC specific APIs +parseSymsFile(apis, "LIBVIRT_LXC", symslxc, + os.path.join(builddir, "docs", "libvirt-lxc-api.xml")) + + +# Some special things which aren't public APIs, +# but we want to report +apis["virConnectSupportsFeature"] = { + "vers": "0.3.2" +} +apis["virDomainMigratePrepare"] = { + "vers": "0.3.2" +} +apis["virDomainMigratePerform"] = { + "vers": "0.3.2" +} +apis["virDomainMigrateFinish"] = { + "vers": "0.3.2" +} +apis["virDomainMigratePrepare2"] = { + "vers": "0.5.0" +} +apis["virDomainMigrateFinish2"] = { + "vers": "0.5.0" +} +apis["virDomainMigratePrepareTunnel"] = { + "vers": "0.7.2" +} + +apis["virDomainMigrateBegin3"] = { + "vers": "0.9.2" +} +apis["virDomainMigratePrepare3"] = { + "vers": "0.9.2" +} +apis["virDomainMigratePrepareTunnel3"] = { + "vers": "0.9.2" +} +apis["virDomainMigratePerform3"] = { + "vers": "0.9.2" +} +apis["virDomainMigrateFinish3"] = { + "vers": "0.9.2" +} +apis["virDomainMigrateConfirm3"] = { + "vers": "0.9.2" +} + +apis["virDomainMigrateBegin3Params"] = { + "vers": "1.1.0" +} +apis["virDomainMigratePrepare3Params"] = { + "vers": "1.1.0" +} +apis["virDomainMigratePrepareTunnel3Params"] = { + "vers": "1.1.0" +} +apis["virDomainMigratePerform3Params"] = { + "vers": "1.1.0" +} +apis["virDomainMigrateFinish3Params"] = { + "vers": "1.1.0" +} +apis["virDomainMigrateConfirm3Params"] = { + "vers": "1.1.0" +} + + +# Now we want to get the mapping between public APIs +# and driver struct fields. This lets us later match +# update the driver impls with the public APis. + +# Group name -> hash of APIs { fields -> api name } +groups = {} +ingrp = None +for drivertablefile in drivertablefiles: + with open(drivertablefile) as fh: + for line in fh: + starttablematch = re.search(r"struct _(vir\w*Driver)", line) + if starttablematch is not None: + grp = starttablematch.group(1) + if grp != "virStateDriver" and grp != "virStreamDriver": + ingrp = grp + groups[ingrp] = { + "apis": {}, + "drivers": {} + } + elif ingrp is not None: + callbackmatch = re.search(r"^\s*vir(?:Drv)(\w+)\s+(\w+);\s*$", + line) + if callbackmatch is not None: + name = callbackmatch.group(1) + field = callbackmatch.group(2) + + api = "vir" + name + if api in apis: + groups[ingrp]["apis"][field] = api + elif re.search(r"\w+(Open|Close|URIProbe)", api) is not None: + continue + else: + raise Exception(("driver %s does not have " + + "a public API") % name) + elif re.search(r"};", line): + ingrp = None + + +# Finally, we read all the primary driver files and extract +# the driver API tables from each one. + +for src in srcs: + with open(src) as fh: + groupsre = "|".join(groups.keys()) + + ingrp = None + impl = None + for line in fh: + if ingrp is None: + m = re.search(r"^\s*(static\s+)?(" + + groupsre + + r")\s+(\w+)\s*=\s*{", line) + if m is None: + m = re.search(r"^\s*(static\s+)?(" + + groupsre + + r")\s+NAME\(\w+\)\s*=\s*{", line) + if m is not None: + ingrp = m.group(2) + impl = src + + implmatch = re.search(r".*/node_device_(\w+)\.c", impl) + if implmatch is None: + implmatch = re.search(r".*/(\w+?)_((\w+)_)?(\w+)\.c", impl) + if implmatch is None: + raise Exception("Unexpected impl format '%s'" % impl) + impl = implmatch.group(1) + + if impl in groups[ingrp]["drivers"]: + raise Exception( + "Group %s already contains %s" % (ingrp, impl)) + + groups[ingrp]["drivers"][impl] = {} + else: + callbackmatch = re.search(r"\s*\.(\w+)\s*=\s*(\w+)\s*,?\s*" + + r"(?:/\*\s*(\d+\.\d+\.\d+)\s*" + + r"(?:-\s*(\d+\.\d+\.\d+))?\s*\*/\s*)?$", + line) + if callbackmatch is not None: + api = callbackmatch.group(1) + meth = callbackmatch.group(2) + vers = callbackmatch.group(3) + deleted = callbackmatch.group(4) + + if api == "no" or api == "name": + continue + + if meth == "NULL" and deleted is None: + raise Exception( + ("Method impl for %s is NULL, but " + + "no deleted version is provided") % api) + + if meth != "NULL" and deleted is not None: + raise Exception( + ("Method impl for %s is non-NULL, but " + + "deleted version is provided") % api) + + if vers is None and api != "connectURIProbe": + raise Exception( + "Method %s in %s is missing version" % + (meth, src)) + + if api not in groups[ingrp]["apis"]: + if re.search(r"\w+(Open|Close|URIProbe)", api): + continue + + raise Exception("Found unexpected method " + + "%s in %s" % (api, ingrp)) + + groups[ingrp]["drivers"][impl][api] = { + "vers": vers, + "deleted": deleted, + } + + if (api == "domainMigratePrepare" or + api == "domainMigratePrepare2" or + api == "domainMigratePrepare3"): + if ("domainMigrate" not in + groups[ingrp]["drivers"][impl]): + groups[ingrp]["drivers"][impl]["domainMigrate"] = { + "vers": vers, + } + elif line.find("}") != -1: + ingrp = None + + +# The '.open' driver method is used for 3 public APIs, so we +# have a bit of manual fixup todo with the per-driver versioning +# and support matrix + +groups["virHypervisorDriver"]["apis"]["openAuth"] = \ + "virConnectOpenAuth" +groups["virHypervisorDriver"]["apis"]["openReadOnly"] = \ + "virConnectOpenReadOnly" +groups["virHypervisorDriver"]["apis"]["domainMigrate"] = \ + "virDomainMigrate" + +openAuthVers = (0 * 1000 * 1000) + (4 * 1000) + 0 + +drivers = groups["virHypervisorDriver"]["drivers"] +for drv in drivers.keys(): + openVersStr = drivers[drv]["connectOpen"]["vers"] + openVers = 0 + if openVersStr != "Y": + openVersBits = openVersStr.split(".") + if len(openVersBits) != 3: + raise Exception("Expected 3 digit version for %s" % openVersStr) + openVers = ((int(openVersBits[0]) * 1000 * 1000) + + (int(openVersBits[1]) * 1000) + + int(openVersBits[2])) + + # virConnectOpenReadOnly always matches virConnectOpen version + drivers[drv]["connectOpenReadOnly"] = \ + drivers[drv]["connectOpen"] + + # virConnectOpenAuth is always 0.4.0 if the driver existed + # before this time, otherwise it matches the version of + # the driver's virConnectOpen entry + if openVersStr == "Y" or openVers >= openAuthVers: + vers = openVersStr + else: + vers = "0.4.0" + drivers[drv]["connectOpenAuth"] = { + "vers": vers, + } + + +# Another special case for the virDomainCreateLinux which was replaced +# with virDomainCreateXML +groups["virHypervisorDriver"]["apis"]["domainCreateLinux"] = \ + "virDomainCreateLinux" + +createAPIVers = (0 * 1000 * 1000) + (0 * 1000) + 3 + +for drv in drivers.keys(): + if "domainCreateXML" not in drivers[drv]: + continue + createVersStr = drivers[drv]["domainCreateXML"]["vers"] + createVers = 0 + if createVersStr != "Y": + createVersBits = createVersStr.split(".") + if len(createVersBits) != 3: + raise Exception("Expected 3 digit version for %s" % createVersStr) + createVers = ((int(createVersBits[0]) * 1000 * 1000) + + (int(createVersBits[1]) * 1000) + + int(createVersBits[2])) + + # virCreateLinux is always 0.0.3 if the driver existed + # before this time, otherwise it matches the version of + # the driver's virCreateXML entry + if createVersStr == "Y" or createVers >= createAPIVers: + vers = createVersStr + else: + vers = "0.0.3" + + drivers[drv]["domainCreateLinux"] = { + "vers": vers, + } + + +# Finally we generate the HTML file with the tables + +print('''<?xml version="1.0" encoding="UTF-8"?> +<!DOCTYPE html> +<html xmlns="http://www.w3.org/1999/xhtml"> +<body class="hvsupport"> +<h1>libvirt API support matrix</h1> + +<ul id="toc"></ul> + +<p> +This page documents which <a href="html/">libvirt calls</a> work on +which libvirt drivers / hypervisors, and which version the API appeared +in. If a hypervisor driver later dropped support for the API, the version +when it was removed is also mentioned (highlighted in +<span class="removedhv">dark red</span>). +</p> +''') + +for grp in sorted(groups.keys()): + print("<h2><a id=\"%s\">%s</a></h2>" % (grp, groupheaders[grp])) + print('''<table class="top_table"> +<thead> +<tr> +<th>API</th> +<th>Version</th>''') + + for drv in sorted(groups[grp]["drivers"].keys()): + print(" <th>%s</th>" % drv) + + print('''</tr> +</thead> +<tbody>''') + + row = 0 + + def sortkey(field): + return groups[grp]["apis"][field] + + for field in sorted(groups[grp]["apis"].keys(), key=sortkey): + api = groups[grp]["apis"][field] + vers = apis[api]["vers"] + htmlgrp = apis[api].get("file") + print("<tr>") + + if htmlgrp is not None: + print(('''<td>\n<a href=\"html/libvirt-%s.html#%s\">''' + + '''%s</a>\n</td>''') % + (htmlgrp, api, api)) + else: + print("<td>\n%s</td>" % api) + + print("<td>%s</td>" % vers) + + for drv in sorted(groups[grp]["drivers"].keys()): + info = "" + if field in groups[grp]["drivers"][drv]: + vers = groups[grp]["drivers"][drv][field]["vers"] + if vers is not None: + info = info + vers + + deleted = groups[grp]["drivers"][drv][field].get("deleted") + if deleted is not None: + info = info + (''' - <span class="removedhv">''' + + '''%s</span>''' % deleted) + + print("<td>%s</td>" % info) + + print("</tr>") + + row = row + 1 + if (row % 15) == 0: + print('''<tr> +<th>API</th> +<th>Version</th>''') + + for drv in sorted(groups[grp]["drivers"].keys()): + print(" <th>%s</th>" % drv) + + print("</tr>") + + print("</tbody>\n</table>") + +print("</body>\n</html>") -- 2.21.0

As part of an goal to eliminate Perl from libvirt build tools, rewrite the genaclperms.pl tool in Python. This was a straight conversion, manually going line-by-line to change the syntax from Perl to Python. Thus the overall structure of the file and approach is the same. Signed-off-by: Daniel P. Berrangé <berrange@redhat.com> --- Makefile.am | 1 + docs/Makefile.am | 6 +- docs/genaclperms.pl | 125 ----------------------------------------- scripts/genaclperms.py | 123 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 127 insertions(+), 128 deletions(-) delete mode 100755 docs/genaclperms.pl create mode 100755 scripts/genaclperms.py diff --git a/Makefile.am b/Makefile.am index bc8aecc4a2..a14474f535 100644 --- a/Makefile.am +++ b/Makefile.am @@ -56,6 +56,7 @@ EXTRA_DIST = \ scripts/check-symfile.py \ scripts/check-symsorting.py \ scripts/dtrace2systemtap.py \ + scripts/genaclperms.py \ scripts/genpolkit.py \ scripts/gensystemtap.py \ scripts/group-qemu-caps.py \ diff --git a/docs/Makefile.am b/docs/Makefile.am index cf83949168..ca7a2ec260 100644 --- a/docs/Makefile.am +++ b/docs/Makefile.am @@ -163,7 +163,7 @@ schemadir = $(pkgdatadir)/schemas schema_DATA = $(wildcard $(srcdir)/schemas/*.rng) EXTRA_DIST= \ - apibuild.py genaclperms.pl \ + apibuild.py \ site.xsl subsite.xsl newapi.xsl page.xsl \ wrapstring.xsl \ $(dot_html_in) $(gif) $(apipng) \ @@ -177,8 +177,8 @@ EXTRA_DIST= \ acl_generated = aclperms.htmlinc aclperms.htmlinc: $(top_srcdir)/src/access/viraccessperm.h \ - $(srcdir)/genaclperms.pl Makefile.am - $(AM_V_GEN)$(PERL) $(srcdir)/genaclperms.pl $< > $@ + $(top_srcdir)/scripts/genaclperms.py Makefile.am + $(AM_V_GEN)$(RUNUTF8) $(PYTHON) $(top_srcdir)/scripts/genaclperms.py $< > $@ CLEANFILES = \ $(dot_html) \ diff --git a/docs/genaclperms.pl b/docs/genaclperms.pl deleted file mode 100755 index e20b4c11c3..0000000000 --- a/docs/genaclperms.pl +++ /dev/null @@ -1,125 +0,0 @@ -#!/usr/bin/env perl -# -# Copyright (C) 2013 Red Hat, Inc. -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2.1 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library. If not, see -# <http://www.gnu.org/licenses/>. -# - -use strict; -use warnings; - -my @objects = ( - "CONNECT", "DOMAIN", "INTERFACE", - "NETWORK_PORT", "NETWORK", "NODE_DEVICE", - "NWFILTER_BINDING", "NWFILTER", - "SECRET", "STORAGE_POOL", "STORAGE_VOL", - ); - -my %class; - -foreach my $object (@objects) { - my $class = lc $object; - - $class =~ s/(^\w|_\w)/uc $1/eg; - $class =~ s/_//g; - $class =~ s/Nwfilter/NWFilter/; - $class = "vir" . $class . "Ptr"; - - $class{$object} = $class; -} - -my $objects = join ("|", @objects); - -my %opts; -my $in_opts = 0; - -my %perms; - -while (<>) { - if ($in_opts) { - if (m,\*/,) { - $in_opts = 0; - } elsif (/\*\s*\@(\w+):\s*(.*?)\s*$/) { - $opts{$1} = $2; - } - } elsif (m,/\*\*,) { - $in_opts = 1; - } elsif (/VIR_ACCESS_PERM_($objects)_((?:\w|_)+),/) { - my $object = $1; - my $perm = lc $2; - next if $perm eq "last"; - - $perm =~ s/_/-/g; - - $perms{$object} = {} unless exists $perms{$object}; - $perms{$object}->{$perm} = { - desc => $opts{desc}, - message => $opts{message}, - anonymous => $opts{anonymous} - }; - %opts = (); - } -} - -print <<EOF; -<?xml version="1.0" encoding="UTF-8"?> -<!DOCTYPE html> -<html xmlns="http://www.w3.org/1999/xhtml"> - <body> -EOF - -foreach my $object (sort { $a cmp $b } keys %perms) { - my $class = $class{$object}; - my $olink = lc "object_" . $object; - print <<EOF; -<h3><a id="$olink">$class</a></h3> -<table class="acl"> - <thead> - <tr> - <th>Permission</th> - <th>Description</th> - </tr> - </thead> - <tbody> -EOF - - foreach my $perm (sort { $a cmp $b } keys %{$perms{$object}}) { - my $description = $perms{$object}->{$perm}->{desc}; - - die "missing description for $object.$perm" unless - defined $description; - - my $plink = lc "perm_" . $object . "_" . $perm; - $plink =~ s/-/_/g; - - print <<EOF; - <tr> - <td><a id="$plink">$perm</a></td> - <td>$description</td> - </tr> -EOF - - } - - print <<EOF; - </tbody> -</table> -EOF -} - -print <<EOF; - </body> -</html> -EOF diff --git a/scripts/genaclperms.py b/scripts/genaclperms.py new file mode 100755 index 0000000000..cd92346449 --- /dev/null +++ b/scripts/genaclperms.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python +# +# Copyright (C) 2013-2019 Red Hat, Inc. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see +# <http://www.gnu.org/licenses/>. +# + +from __future__ import print_function + +import re +import sys + +objects = [ + "CONNECT", "DOMAIN", "INTERFACE", + "NETWORK_PORT", "NETWORK", "NODE_DEVICE", + "NWFILTER_BINDING", "NWFILTER", + "SECRET", "STORAGE_POOL", "STORAGE_VOL", +] + + +classes = {} + +for obj in objects: + klass = obj.lower() + + klass = re.sub(r'''(^\w|_\w)''', lambda a: a.group(1).upper(), klass) + klass = klass.replace("_", "") + klass = klass.replace("Nwfilter", "NWFilter") + klass = "vir" + klass + "Ptr" + + classes[obj] = klass + + +objectstr = "|".join(objects) + +opts = {} +in_opts = {} + +perms = {} + +aclfile = sys.argv[1] +with open(aclfile, "r") as fh: + for line in fh: + if in_opts: + if line.find("*/") != -1: + in_opts = False + else: + m = re.search(r'''\*\s*\@(\w+):\s*(.*?)\s*$''', line) + if m is not None: + opts[m.group(1)] = m.group(2) + elif line.find("**") != -1: + in_opts = True + else: + m = re.search(r'''VIR_ACCESS_PERM_(%s)_((?:\w|_)+),''' % + objectstr, line) + if m is not None: + obj = m.group(1) + perm = m.group(2).lower() + if perm == "last": + continue + + perm = perm.replace("_", "-") + + if obj not in perms: + perms[obj] = {} + perms[obj][perm] = { + "desc": opts.get("desc", None), + "message": opts.get("message", None), + "anonymous": opts.get("anonymous", None), + } + opts = {} + +print('<?xml version="1.0" encoding="UTF-8"?>') +print('<!DOCTYPE html>') +print('<html xmlns="http://www.w3.org/1999/xhtml">') +print(' <body>') + +for obj in sorted(perms.keys()): + klass = classes[obj] + + olink = "object_" + obj.lower() + + print(' <h3><a id="%s">%s</a></h3>' % (olink, klass)) + print(' <table class="acl">') + print(' <thead>') + print(' <tr>') + print(' <th>Permission</th>') + print(' <th>Description</th>') + print(' </tr>') + print(' </thead>') + print(' <tbody>') + + for perm in sorted(perms[obj].keys()): + description = perms[obj][perm]["desc"] + + if description is None: + raise Exception("missing description for %s.%s" % (obj, perm)) + + plink = "perm_" + obj.lower() + "_" + perm.lower() + plink = plink.replace("-", "_") + + print(' <tr>') + print(' <td><a id="%s">%s</a></td>' % (plink, perm)) + print(' <td>%s</td>' % description) + print(' </tr>') + + print(' </tbody>') + print(' </table>') + +print(' </body>') +print('</html>') -- 2.21.0

Signed-off-by: Daniel P. Berrangé <berrange@redhat.com> --- Makefile.am | 1 + build-aux/syntax-check.mk | 2 +- docs/Makefile.am | 5 ++--- {docs => scripts}/apibuild.py | 0 4 files changed, 4 insertions(+), 4 deletions(-) rename {docs => scripts}/apibuild.py (100%) diff --git a/Makefile.am b/Makefile.am index a14474f535..67d07eff41 100644 --- a/Makefile.am +++ b/Makefile.am @@ -45,6 +45,7 @@ EXTRA_DIST = \ run.in \ README.md \ AUTHORS.in \ + scripts/apibuild.py \ scripts/augeas-gentest.py \ scripts/check-aclperms.py \ scripts/check-aclrules.py \ diff --git a/build-aux/syntax-check.mk b/build-aux/syntax-check.mk index 44639f499e..ee8a496efa 100644 --- a/build-aux/syntax-check.mk +++ b/build-aux/syntax-check.mk @@ -2284,7 +2284,7 @@ exclude_file_name_regexp--sc_trailing_blank = \ /sysinfodata/.*\.data|/virhostcpudata/.*\.cpuinfo|^gnulib/local/.*/.*diff$$ exclude_file_name_regexp--sc_unmarked_diagnostics = \ - ^(docs/apibuild.py|tests/virt-aa-helper-test|docs/js/.*\.js)$$ + ^(scripts/apibuild.py|tests/virt-aa-helper-test|docs/js/.*\.js)$$ exclude_file_name_regexp--sc_size_of_brackets = build-aux/syntax-check\.mk diff --git a/docs/Makefile.am b/docs/Makefile.am index ca7a2ec260..f37417cafb 100644 --- a/docs/Makefile.am +++ b/docs/Makefile.am @@ -163,7 +163,6 @@ schemadir = $(pkgdatadir)/schemas schema_DATA = $(wildcard $(srcdir)/schemas/*.rng) EXTRA_DIST= \ - apibuild.py \ site.xsl subsite.xsl newapi.xsl page.xsl \ wrapstring.xsl \ $(dot_html_in) $(gif) $(apipng) \ @@ -278,13 +277,13 @@ python_generated_files = \ libvirt-admin-refs.xml \ $(NULL) -APIBUILD=$(srcdir)/apibuild.py +APIBUILD=$(top_srcdir)/scripts/apibuild.py APIBUILD_STAMP=apibuild.py.stamp CLEANFILES += $(APIBUILD_STAMP) $(python_generated_files): $(APIBUILD_STAMP) -$(APIBUILD_STAMP): $(srcdir)/apibuild.py \ +$(APIBUILD_STAMP): $(top_srcdir)/scripts/apibuild.py \ $(top_srcdir)/include/libvirt/libvirt.h \ $(top_srcdir)/include/libvirt/libvirt-common.h.in \ $(top_srcdir)/include/libvirt/libvirt-domain-checkpoint.h \ diff --git a/docs/apibuild.py b/scripts/apibuild.py similarity index 100% rename from docs/apibuild.py rename to scripts/apibuild.py -- 2.21.0

Signed-off-by: Daniel P. Berrangé <berrange@redhat.com> --- Makefile.am | 1 + {src/esx => scripts}/esx_vi_generator.py | 0 src/esx/Makefile.inc.am | 6 +++--- 3 files changed, 4 insertions(+), 3 deletions(-) rename {src/esx => scripts}/esx_vi_generator.py (100%) diff --git a/Makefile.am b/Makefile.am index 8359ebc6ad..14456161c3 100644 --- a/Makefile.am +++ b/Makefile.am @@ -57,6 +57,7 @@ EXTRA_DIST = \ scripts/check-symfile.py \ scripts/check-symsorting.py \ scripts/dtrace2systemtap.py \ + scripts/esx_vi_generator.py \ scripts/genaclperms.py \ scripts/genpolkit.py \ scripts/gensystemtap.py \ diff --git a/src/esx/esx_vi_generator.py b/scripts/esx_vi_generator.py similarity index 100% rename from src/esx/esx_vi_generator.py rename to scripts/esx_vi_generator.py diff --git a/src/esx/Makefile.inc.am b/src/esx/Makefile.inc.am index 6b10755b7e..aa2d17c791 100644 --- a/src/esx/Makefile.inc.am +++ b/src/esx/Makefile.inc.am @@ -43,7 +43,6 @@ ESX_DRIVER_GENERATED = \ ESX_DRIVER_EXTRA_DIST = \ esx/README \ esx/esx_vi_generator.input \ - esx/esx_vi_generator.py \ $(NULL) ESX_GENERATED_STAMP = .esx_vi_generator.stamp @@ -60,9 +59,10 @@ BUILT_SOURCES += $(ESX_DRIVER_GENERATED) $(ESX_DRIVER_GENERATED): $(ESX_GENERATED_STAMP) $(ESX_GENERATED_STAMP): $(srcdir)/esx/esx_vi_generator.input \ - $(srcdir)/esx/esx_vi_generator.py + $(top_srcdir)/scripts/esx_vi_generator.py $(AM_V_GEN) $(RUNUTF8) $(PYTHON) \ - $(srcdir)/esx/esx_vi_generator.py $(srcdir) $(builddir) && touch $@ + $(top_srcdir)/scripts/esx_vi_generator.py \ + $(srcdir) $(builddir) && touch $@ CLEANFILES += $(ESX_DRIVER_GENERATED) $(ESX_GENERATED_STAMP) -- 2.21.0