[Libvir] PATCH: Cleanup generator & force build fail on missing APIs
The python generator will happily ignore functions it can't handle and pretend everything completed without error. This leads to the situation where we add new APis to C library and no one ever notices that they are missing from the python until months later. This requires that my previous patch be applied first to implement the missing APIs we already have :-) This patch causes the generator to return a non-zero exit status if there are any APIs marked as FAILED. It will also explicitly print out their names so its clear what is missing. In doing this I added a bunch more functions to the skip list - ones that we already manually wrote. It also removes the manually written virCloseConnect/virDomainFree/ virNetworkFree C code, since the generated code is just fine. Finally, it makes all manually written C functions static for consistency generator.py | 50 ++++++++++++++++++++++------------ libvir.c | 85 ++++++----------------------------------------------------- 2 files changed, 43 insertions(+), 92 deletions(-) Dan. diff -r 842dc0869c87 python/generator.py --- a/python/generator.py Sat Jan 19 14:54:24 2008 -0500 +++ b/python/generator.py Sat Jan 19 14:58:50 2008 -0500 @@ -212,7 +212,7 @@ skipped_modules = { } skipped_types = { - 'int *': "usually a return type", +# 'int *': "usually a return type", } ####################################################################### @@ -277,6 +277,8 @@ skip_impl = ( 'virDomainLookupByUUID', 'virNetworkGetUUID', 'virNetworkLookupByUUID', + 'virDomainGetAutostart', + 'virNetworkGetAutostart', 'virDomainBlockStats', 'virDomainInterfaceStats', 'virNodeGetCellsFreeMemory', @@ -287,18 +289,23 @@ skip_impl = ( 'virDomainPinVcpu', ) -def skip_function(name): - if name == "virConnectClose": - return 1 - if name == "virDomainFree": - return 1 - if name == "virNetworkFree": - return 1 - if name == "vshRunConsole": - return 1 - if name == "virGetVersion": - return 1 - return 0 + +# These are functions which the generator skips completly - no python +# or C code is generated. Generally should not be used for any more +# functions than those already listed +skip_function = ( + 'virConnectListDomains', # Python API is called virConectListDomainsID for unknown reasons + 'virConnSetErrorFunc', # Not used in Python API XXX is this a bug ? + 'virResetError', # Not used in Python API XXX is this a bug ? + 'virConnectGetVersion', # Not used in Python API XXX is this a bug ? + 'virGetVersion', # Python C code is manually written + 'virSetErrorFunc', # Python API is called virRegisterErrorHandler for unknown reasons + 'virConnCopyLastError', # Python API is called virConnGetLastError instead + 'virCopyLastError', # Python API is called virGetLastError instead + 'virConnectOpenAuth', # Python C code is manually written + 'virDefaultErrorFunc', # Python virErrorFuncHandler impl calls this from C +) + def print_function_wrapper(name, output, export, include): global py_types @@ -314,7 +321,7 @@ def print_function_wrapper(name, output, if skipped_modules.has_key(file): return 0 - if skip_function(name) == 1: + if name in skip_function: return 0 if name in skip_impl: # Don't delete the function entry in the caller. @@ -527,12 +534,19 @@ def buildStubs(): export.close() wrapper.close() - print "Generated %d wrapper functions, %d failed, %d skipped\n" % (nb_wrap, - failed, skipped); + print "Generated %d wrapper functions" % nb_wrap + print "Missing type converters: " for type in unknown_types.keys(): print "%s:%d " % (type, len(unknown_types[type])), print + + for f in functions_failed: + print "ERROR: failed %s" % f + + if failed > 0: + return -1 + return 0 ####################################################################### # @@ -1130,5 +1144,7 @@ def buildWrappers(): txt.close() classes.close() -buildStubs() +if buildStubs() < 0: + sys.exit(1) buildWrappers() +sys.exit(0) diff -r 842dc0869c87 python/libvir.c --- a/python/libvir.c Sat Jan 19 14:54:24 2008 -0500 +++ b/python/libvir.c Sat Jan 19 14:58:50 2008 -0500 @@ -23,14 +23,6 @@ extern void initcygvirtmod(void); extern void initcygvirtmod(void); #endif -PyObject *libvirt_virDomainGetUUID(PyObject *self ATTRIBUTE_UNUSED, PyObject *args); -PyObject *libvirt_virNetworkGetUUID(PyObject *self ATTRIBUTE_UNUSED, PyObject *args); -PyObject *libvirt_virGetLastError(PyObject *self ATTRIBUTE_UNUSED, PyObject *args); -PyObject *libvirt_virConnGetLastError(PyObject *self ATTRIBUTE_UNUSED, PyObject *args); -PyObject * libvirt_virDomainBlockStats(PyObject *self ATTRIBUTE_UNUSED, PyObject *args); -PyObject * libvirt_virDomainInterfaceStats(PyObject *self ATTRIBUTE_UNUSED, PyObject *args); -PyObject * libvirt_virNodeGetCellsFreeMemory(PyObject *self ATTRIBUTE_UNUSED, PyObject *args); - /* The two-statement sequence "Py_INCREF(Py_None); return Py_None;" is so common that we encapsulate it here. Now, each use is simply return VIR_PY_NONE; */ @@ -42,7 +34,7 @@ PyObject * libvirt_virNodeGetCellsFreeMe * * ************************************************************************/ -PyObject * +static PyObject * libvirt_virDomainBlockStats(PyObject *self ATTRIBUTE_UNUSED, PyObject *args) { virDomainPtr domain; PyObject *pyobj_domain; @@ -71,7 +63,7 @@ libvirt_virDomainBlockStats(PyObject *se return(info); } -PyObject * +static PyObject * libvirt_virDomainInterfaceStats(PyObject *self ATTRIBUTE_UNUSED, PyObject *args) { virDomainPtr domain; PyObject *pyobj_domain; @@ -422,7 +414,7 @@ static PyObject *libvirt_virPythonErrorF static PyObject *libvirt_virPythonErrorFuncHandler = NULL; static PyObject *libvirt_virPythonErrorFuncCtxt = NULL; -PyObject * +static PyObject * libvirt_virGetLastError(PyObject *self ATTRIBUTE_UNUSED, PyObject *args ATTRIBUTE_UNUSED) { virError err; @@ -446,7 +438,7 @@ libvirt_virGetLastError(PyObject *self A return info; } -PyObject * +static PyObject * libvirt_virConnGetLastError(PyObject *self ATTRIBUTE_UNUSED, PyObject *args) { virError err; @@ -714,41 +706,6 @@ libvirt_virGetVersion (PyObject *self AT return Py_BuildValue ((char *) "kk", libVer, typeVer); } -static PyObject * -libvirt_virDomainFree(PyObject *self ATTRIBUTE_UNUSED, PyObject *args) { - PyObject *py_retval; - int c_retval; - virDomainPtr domain; - PyObject *pyobj_domain; - - if (!PyArg_ParseTuple(args, (char *)"O:virDomainFree", &pyobj_domain)) - return(NULL); - domain = (virDomainPtr) PyvirDomain_Get(pyobj_domain); - - LIBVIRT_BEGIN_ALLOW_THREADS; - c_retval = virDomainFree(domain); - LIBVIRT_END_ALLOW_THREADS; - py_retval = libvirt_intWrap((int) c_retval); - return(py_retval); -} - -static PyObject * -libvirt_virConnectClose(PyObject *self ATTRIBUTE_UNUSED, PyObject *args) { - PyObject *py_retval; - int c_retval; - virConnectPtr conn; - PyObject *pyobj_conn; - - if (!PyArg_ParseTuple(args, (char *)"O:virConnectClose", &pyobj_conn)) - return(NULL); - conn = (virConnectPtr) PyvirConnect_Get(pyobj_conn); - - LIBVIRT_BEGIN_ALLOW_THREADS; - c_retval = virConnectClose(conn); - LIBVIRT_END_ALLOW_THREADS; - py_retval = libvirt_intWrap((int) c_retval); - return(py_retval); -} static PyObject * libvirt_virConnectListDomainsID(PyObject *self ATTRIBUTE_UNUSED, @@ -872,7 +829,7 @@ libvirt_virNodeGetInfo(PyObject *self AT return(py_retval); } -PyObject * +static PyObject * libvirt_virDomainGetUUID(PyObject *self ATTRIBUTE_UNUSED, PyObject *args) { PyObject *py_retval; unsigned char uuid[VIR_UUID_BUFLEN]; @@ -917,25 +874,6 @@ libvirt_virDomainLookupByUUID(PyObject * c_retval = virDomainLookupByUUID(conn, uuid); LIBVIRT_END_ALLOW_THREADS; py_retval = libvirt_virDomainPtrWrap((virDomainPtr) c_retval); - return(py_retval); -} - - -static PyObject * -libvirt_virNetworkFree(PyObject *self ATTRIBUTE_UNUSED, PyObject *args) { - PyObject *py_retval; - int c_retval; - virNetworkPtr domain; - PyObject *pyobj_domain; - - if (!PyArg_ParseTuple(args, (char *)"O:virNetworkFree", &pyobj_domain)) - return(NULL); - domain = (virNetworkPtr) PyvirNetwork_Get(pyobj_domain); - - LIBVIRT_BEGIN_ALLOW_THREADS; - c_retval = virNetworkFree(domain); - LIBVIRT_END_ALLOW_THREADS; - py_retval = libvirt_intWrap((int) c_retval); return(py_retval); } @@ -1024,7 +962,7 @@ libvirt_virConnectListDefinedNetworks(Py } -PyObject * +static PyObject * libvirt_virNetworkGetUUID(PyObject *self ATTRIBUTE_UNUSED, PyObject *args) { PyObject *py_retval; unsigned char uuid[VIR_UUID_BUFLEN]; @@ -1073,7 +1011,7 @@ libvirt_virNetworkLookupByUUID(PyObject } -PyObject * +static PyObject * libvirt_virDomainGetAutostart(PyObject *self ATTRIBUTE_UNUSED, PyObject *args) { PyObject *py_retval; int c_retval, autostart; @@ -1096,7 +1034,7 @@ libvirt_virDomainGetAutostart(PyObject * } -PyObject * +static PyObject * libvirt_virNetworkGetAutostart(PyObject *self ATTRIBUTE_UNUSED, PyObject *args) { PyObject *py_retval; int c_retval, autostart; @@ -1118,8 +1056,8 @@ libvirt_virNetworkGetAutostart(PyObject return(py_retval); } -PyObject * libvirt_virNodeGetCellsFreeMemory(PyObject *self ATTRIBUTE_UNUSED, - PyObject *args) +static PyObject * +libvirt_virNodeGetCellsFreeMemory(PyObject *self ATTRIBUTE_UNUSED, PyObject *args) { PyObject *py_retval; PyObject *pyobj_conn; @@ -1165,9 +1103,7 @@ static PyMethodDef libvirtMethods[] = { static PyMethodDef libvirtMethods[] = { #include "libvirt-export.c" {(char *) "virGetVersion", libvirt_virGetVersion, METH_VARARGS, NULL}, - {(char *) "virDomainFree", libvirt_virDomainFree, METH_VARARGS, NULL}, {(char *) "virConnectOpenAuth", libvirt_virConnectOpenAuth, METH_VARARGS, NULL}, - {(char *) "virConnectClose", libvirt_virConnectClose, METH_VARARGS, NULL}, {(char *) "virConnectListDomainsID", libvirt_virConnectListDomainsID, METH_VARARGS, NULL}, {(char *) "virConnectListDefinedDomains", libvirt_virConnectListDefinedDomains, METH_VARARGS, NULL}, {(char *) "virDomainGetInfo", libvirt_virDomainGetInfo, METH_VARARGS, NULL}, @@ -1177,7 +1113,6 @@ static PyMethodDef libvirtMethods[] = { {(char *) "virRegisterErrorHandler", libvirt_virRegisterErrorHandler, METH_VARARGS, NULL}, {(char *) "virGetLastError", libvirt_virGetLastError, METH_VARARGS, NULL}, {(char *) "virConnGetLastError", libvirt_virConnGetLastError, METH_VARARGS, NULL}, - {(char *) "virNetworkFree", libvirt_virNetworkFree, METH_VARARGS, NULL}, {(char *) "virConnectListNetworks", libvirt_virConnectListNetworks, METH_VARARGS, NULL}, {(char *) "virConnectListDefinedNetworks", libvirt_virConnectListDefinedNetworks, METH_VARARGS, NULL}, {(char *) "virNetworkGetUUID", libvirt_virNetworkGetUUID, METH_VARARGS, NULL}, -- |=- Red Hat, Engineering, Emerging Technologies, Boston. +1 978 392 2496 -=| |=- Perl modules: http://search.cpan.org/~danberr/ -=| |=- Projects: http://freshmeat.net/~danielpb/ -=| |=- GnuPG: 7D3B9505 F3C9 553F A1DA 4AC2 5648 23C1 B3DF F742 7D3B 9505 -=|
On Sun, Jan 20, 2008 at 05:29:06PM +0000, Daniel P. Berrange wrote:
The python generator will happily ignore functions it can't handle and pretend everything completed without error. This leads to the situation where we add new APis to C library and no one ever notices that they are missing from the python until months later. This requires that my previous patch be applied first to implement the missing APIs we already have :-)
This patch causes the generator to return a non-zero exit status if there are any APIs marked as FAILED. It will also explicitly print out their names so its clear what is missing. In doing this I added a bunch more functions to the skip list - ones that we already manually wrote.
It also removes the manually written virCloseConnect/virDomainFree/ virNetworkFree C code, since the generated code is just fine.
Finally, it makes all manually written C functions static for consistency
Okay, the default behaviour prints the number of functions which failed (and the number skipped) but now that we have full coverage, yes this is a good thing to do. Note however that you're likely to still see the problem of late discovery of missing bindings because: - people submitting patches are likely to just run 'make' - people applying it will do the same. - only on 'make rebuild' in docs or when preparing the release will we hit the docs/libvirt-api.xml , leading to the subsequent error on a missing part. - and I'm afraid I will be the one hitting them ... at time of release i.e. at the worse moment with a make exiting on an error. So okay to fail, but only if 'make' leads to a failure after modifying the API headers, which probably means changing docs/Makefile.am to systematically rebuild when libvirt.h (or .h.in since libvirt.h is autogenerated ?). Or some other mechanism which somehow force people changing the API to rebuild the XML and the bindings. Daniel -- Red Hat Virtualization group http://redhat.com/virtualization/ Daniel Veillard | virtualization library http://libvirt.org/ veillard@redhat.com | libxml GNOME XML XSLT toolkit http://xmlsoft.org/ http://veillard.com/ | Rpmfind RPM search engine http://rpmfind.net/
On Mon, Jan 21, 2008 at 04:14:15AM -0500, Daniel Veillard wrote:
On Sun, Jan 20, 2008 at 05:29:06PM +0000, Daniel P. Berrange wrote:
The python generator will happily ignore functions it can't handle and pretend everything completed without error. This leads to the situation where we add new APis to C library and no one ever notices that they are missing from the python until months later. This requires that my previous patch be applied first to implement the missing APIs we already have :-)
This patch causes the generator to return a non-zero exit status if there are any APIs marked as FAILED. It will also explicitly print out their names so its clear what is missing. In doing this I added a bunch more functions to the skip list - ones that we already manually wrote.
It also removes the manually written virCloseConnect/virDomainFree/ virNetworkFree C code, since the generated code is just fine.
Finally, it makes all manually written C functions static for consistency
Okay, the default behaviour prints the number of functions which failed (and the number skipped) but now that we have full coverage, yes this is a good thing to do. Note however that you're likely to still see the problem of late discovery of missing bindings because: - people submitting patches are likely to just run 'make' - people applying it will do the same. - only on 'make rebuild' in docs or when preparing the release will we hit the docs/libvirt-api.xml , leading to the subsequent error on a missing part. - and I'm afraid I will be the one hitting them ... at time of release i.e. at the worse moment with a make exiting on an error.
Never fear - this is exactly the sort of problem the nightly autobuild is intended to catch - it'll fail the night after commit and send an email alert so we can fix it the very next day - hopefully long before release.
So okay to fail, but only if 'make' leads to a failure after modifying the API headers, which probably means changing docs/Makefile.am to systematically rebuild when libvirt.h (or .h.in since libvirt.h is autogenerated ?). Or some other mechanism which somehow force people changing the API to rebuild the XML and the bindings.
IMHO the value of it is that it always runs & fails - causing an immediate failure will ensure that it gets addressed immediately and not ignored / undiscovered for months. Dan. -- |=- Red Hat, Engineering, Emerging Technologies, Boston. +1 978 392 2496 -=| |=- Perl modules: http://search.cpan.org/~danberr/ -=| |=- Projects: http://freshmeat.net/~danielpb/ -=| |=- GnuPG: 7D3B9505 F3C9 553F A1DA 4AC2 5648 23C1 B3DF F742 7D3B 9505 -=|
On Mon, Jan 21, 2008 at 02:02:28PM +0000, Daniel P. Berrange wrote:
On Mon, Jan 21, 2008 at 04:14:15AM -0500, Daniel Veillard wrote:
On Sun, Jan 20, 2008 at 05:29:06PM +0000, Daniel P. Berrange wrote:
The python generator will happily ignore functions it can't handle and pretend everything completed without error. This leads to the situation where we add new APis to C library and no one ever notices that they are missing from the python until months later. This requires that my previous patch be applied first to implement the missing APIs we already have :-)
This patch causes the generator to return a non-zero exit status if there are any APIs marked as FAILED. It will also explicitly print out their names so its clear what is missing. In doing this I added a bunch more functions to the skip list - ones that we already manually wrote.
It also removes the manually written virCloseConnect/virDomainFree/ virNetworkFree C code, since the generated code is just fine.
Finally, it makes all manually written C functions static for consistency
Okay, the default behaviour prints the number of functions which failed (and the number skipped) but now that we have full coverage, yes this is a good thing to do. Note however that you're likely to still see the problem of late discovery of missing bindings because: - people submitting patches are likely to just run 'make' - people applying it will do the same. - only on 'make rebuild' in docs or when preparing the release will we hit the docs/libvirt-api.xml , leading to the subsequent error on a missing part. - and I'm afraid I will be the one hitting them ... at time of release i.e. at the worse moment with a make exiting on an error.
Never fear - this is exactly the sort of problem the nightly autobuild is intended to catch - it'll fail the night after commit and send an email alert so we can fix it the very next day - hopefully long before release.
okay but still asynchronous :-) Since we don't often extend the public API you're probably right that it's sufficient. okay, +1 Daniel -- Red Hat Virtualization group http://redhat.com/virtualization/ Daniel Veillard | virtualization library http://libvirt.org/ veillard@redhat.com | libxml GNOME XML XSLT toolkit http://xmlsoft.org/ http://veillard.com/ | Rpmfind RPM search engine http://rpmfind.net/
On Mon, Jan 21, 2008 at 09:12:12AM -0500, Daniel Veillard wrote:
On Mon, Jan 21, 2008 at 02:02:28PM +0000, Daniel P. Berrange wrote:
On Mon, Jan 21, 2008 at 04:14:15AM -0500, Daniel Veillard wrote:
On Sun, Jan 20, 2008 at 05:29:06PM +0000, Daniel P. Berrange wrote:
The python generator will happily ignore functions it can't handle and pretend everything completed without error. This leads to the situation where we add new APis to C library and no one ever notices that they are missing from the python until months later. This requires that my previous patch be applied first to implement the missing APIs we already have :-)
This patch causes the generator to return a non-zero exit status if there are any APIs marked as FAILED. It will also explicitly print out their names so its clear what is missing. In doing this I added a bunch more functions to the skip list - ones that we already manually wrote.
It also removes the manually written virCloseConnect/virDomainFree/ virNetworkFree C code, since the generated code is just fine.
Finally, it makes all manually written C functions static for consistency
Okay, the default behaviour prints the number of functions which failed (and the number skipped) but now that we have full coverage, yes this is a good thing to do. Note however that you're likely to still see the problem of late discovery of missing bindings because: - people submitting patches are likely to just run 'make' - people applying it will do the same. - only on 'make rebuild' in docs or when preparing the release will we hit the docs/libvirt-api.xml , leading to the subsequent error on a missing part. - and I'm afraid I will be the one hitting them ... at time of release i.e. at the worse moment with a make exiting on an error.
Never fear - this is exactly the sort of problem the nightly autobuild is intended to catch - it'll fail the night after commit and send an email alert so we can fix it the very next day - hopefully long before release.
okay but still asynchronous :-) Since we don't often extend the public API you're probably right that it's sufficient.
Ok, comitted this. Dan. -- |=- Red Hat, Engineering, Emerging Technologies, Boston. +1 978 392 2496 -=| |=- Perl modules: http://search.cpan.org/~danberr/ -=| |=- Projects: http://freshmeat.net/~danielpb/ -=| |=- GnuPG: 7D3B9505 F3C9 553F A1DA 4AC2 5648 23C1 B3DF F742 7D3B 9505 -=|
Daniel P. Berrange wrote:
The python generator will happily ignore functions it can't handle and pretend everything completed without error. This leads to the situation where we add new APis to C library and no one ever notices that they are missing from the python until months later. This requires that my previous patch be applied first to implement the missing APIs we already have :-)
This patch causes the generator to return a non-zero exit status if there are any APIs marked as FAILED. It will also explicitly print out their names so its clear what is missing. In doing this I added a bunch more functions to the skip list - ones that we already manually wrote.
Hmmm, I think it should warn rather than fail. If no one notices the missing APIs from Python, well that just indicates that they didn't need them. Rich. -- Emerging Technologies, Red Hat - http://et.redhat.com/~rjones/ Registered Address: Red Hat UK Ltd, Amberley Place, 107-111 Peascod Street, Windsor, Berkshire, SL4 1TE, United Kingdom. Registered in England and Wales under Company Registration No. 03798903
participants (3)
-
Daniel P. Berrange -
Daniel Veillard -
Richard W.M. Jones