From: Wojtek Porczyk <woju@invisiblethingslab.com> This is usable only on python >= 3.4 (or 3.3 with out-of-tree asyncio), however it should be harmless for anyone with older python versions. In simplest case, to have the callbacks queued on the default loop: >>> import libvirtaio >>> libvirtaio.virEventRegisterAsyncIOImpl() The function is not present on non-compatible platforms. Signed-off-by: Wojtek Porczyk <woju@invisiblethingslab.com> --- MANIFEST.in | 1 + libvirt-python.spec.in | 2 + libvirtaio.py | 399 +++++++++++++++++++++++++++++++++++++++++++++++++ setup.py | 12 ++ 4 files changed, 414 insertions(+) create mode 100644 libvirtaio.py diff --git a/MANIFEST.in b/MANIFEST.in index 0d66f9c..b6788f4 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -31,6 +31,7 @@ include libvirt-qemu-override-api.xml include libvirt-qemu-override.c include libvirt-utils.h include libvirt-utils.c +include libvirtaio.py include MANIFEST include README include sanitytest.py diff --git a/libvirt-python.spec.in b/libvirt-python.spec.in index 3021ebd..5ad0292 100644 --- a/libvirt-python.spec.in +++ b/libvirt-python.spec.in @@ -86,11 +86,13 @@ rm -f %{buildroot}%{_libdir}/python*/site-packages/*egg-info %defattr(-,root,root) %doc ChangeLog AUTHORS NEWS README COPYING COPYING.LESSER examples/ %{_libdir}/python3*/site-packages/libvirt.py* +%{_libdir}/python3*/site-packages/libvirtaio.py* %{_libdir}/python3*/site-packages/libvirt_qemu.py* %{_libdir}/python3*/site-packages/libvirt_lxc.py* %{_libdir}/python3*/site-packages/__pycache__/libvirt.cpython-*.py* %{_libdir}/python3*/site-packages/__pycache__/libvirt_qemu.cpython-*.py* %{_libdir}/python3*/site-packages/__pycache__/libvirt_lxc.cpython-*.py* +%{_libdir}/python3*/site-packages/__pycache__/libvirtaio.cpython-*.py* %{_libdir}/python3*/site-packages/libvirtmod* %endif diff --git a/libvirtaio.py b/libvirtaio.py new file mode 100644 index 0000000..7c8c396 --- /dev/null +++ b/libvirtaio.py @@ -0,0 +1,399 @@ +# +# libvirtaio -- asyncio adapter for libvirt +# Copyright (C) 2017 Wojtek Porczyk <woju@invisiblethingslab.com> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, see +# <http://www.gnu.org/licenses/>. +# + +'''Libvirt event loop implementation using asyncio + +Register the implementation of default loop: + + >>> import libvirtaio + >>> libvirtaio.virEventRegisterAsyncIOImpl() + +.. seealso:: + https://libvirt.org/html/libvirt-libvirt-event.html +''' + +__author__ = 'Wojtek Porczyk <woju@invisiblethingslab.com>' +__license__ = 'LGPL-2.1+' +__all__ = ['virEventAsyncIOImpl', 'virEventRegisterAsyncIOImpl'] + +import asyncio +import itertools +import logging +import warnings + +import libvirt + +try: + from asyncio import ensure_future +except ImportError: + from asyncio import async as ensure_future + + +class Callback(object): + '''Base class for holding callback + + :param virEventAsyncIOImpl impl: the implementation in which we run + :param cb: the callback itself + :param opaque: the opaque tuple passed by libvirt + ''' + # pylint: disable=too-few-public-methods + + _iden_counter = itertools.count() + + def __init__(self, impl, cb, opaque, *args, **kwargs): + super().__init__(*args, **kwargs) + self.iden = next(self._iden_counter) + self.impl = impl + self.cb = cb + self.opaque = opaque + + assert self.iden not in self.impl.callbacks, \ + 'found {} callback: {!r}'.format( + self.iden, self.impl.callbacks[self.iden]) + self.impl.callbacks[self.iden] = self + + def __repr__(self): + return '<{} iden={}>'.format(self.__class__.__name__, self.iden) + + def close(self): + '''Schedule *ff* callback''' + self.impl.log.debug('callback %d close(), scheduling ff', self.iden) + self.impl.schedule_ff_callback(self.opaque) + +# +# file descriptors +# + +class Descriptor(object): + '''Manager of one file descriptor + + :param virEventAsyncIOImpl impl: the implementation in which we run + :param int fd: the file descriptor + ''' + def __init__(self, impl, fd): + self.impl = impl + self.fd = fd + self.callbacks = {} + + def _handle(self, event): + '''Dispatch the event to the descriptors + + :param int event: The event (from libvirt's constants) being dispatched + ''' + for callback in self.callbacks.values(): + if callback.event is not None and callback.event & event: + callback.cb(callback.iden, self.fd, event, callback.opaque) + + def update(self): + '''Register or unregister callbacks at event loop + + This should be called after change of any ``.event`` in callbacks. + ''' + # It seems like loop.add_{reader,writer} can be run multiple times + # and will still register the callback only once. Likewise, + # remove_{reader,writer} may be run even if the reader/writer + # is not registered (and will just return False). + + # For the edge case of empty callbacks, any() returns False. + if any(callback.event & ~( + libvirt.VIR_EVENT_HANDLE_READABLE | + libvirt.VIR_EVENT_HANDLE_WRITABLE) + for callback in self.callbacks.values()): + warnings.warn( + 'The only event supported are VIR_EVENT_HANDLE_READABLE ' + 'and VIR_EVENT_HANDLE_WRITABLE', + UserWarning) + + if any(callback.event & libvirt.VIR_EVENT_HANDLE_READABLE + for callback in self.callbacks.values()): + self.impl.loop.add_reader( + self.fd, self._handle, libvirt.VIR_EVENT_HANDLE_READABLE) + else: + self.impl.loop.remove_reader(self.fd) + + if any(callback.event & libvirt.VIR_EVENT_HANDLE_WRITABLE + for callback in self.callbacks.values()): + self.impl.loop.add_writer( + self.fd, self._handle, libvirt.VIR_EVENT_HANDLE_WRITABLE) + else: + self.impl.loop.remove_writer(self.fd) + + def add_handle(self, callback): + '''Add a callback to the descriptor + + :param FDCallback callback: the callback to add + :rtype: None + + After adding the callback, it is immediately watched. + ''' + self.callbacks[callback.iden] = callback + self.update() + + def remove_handle(self, iden): + '''Remove a callback from the descriptor + + :param int iden: the identifier of the callback + :returns: the callback + :rtype: FDCallback + + After removing the callback, the descriptor may be unwatched, if there + are no more handles for it. + ''' + callback = self.callbacks.pop(iden) + self.update() + return callback + + def close(self): + '''''' + self.callbacks.clear() + self.update() + +class DescriptorDict(dict): + '''Descriptors collection + + This is used internally by virEventAsyncIOImpl to hold descriptors. + ''' + def __init__(self, impl): + super().__init__() + self.impl = impl + + def __missing__(self, fd): + descriptor = Descriptor(self.impl, fd) + self[fd] = descriptor + return descriptor + +class FDCallback(Callback): + '''Callback for file descriptor (watcher) + + :param Descriptor descriptor: the descriptor manager + :param int event: bitset of events on which to fire the callback + ''' + # pylint: disable=too-few-public-methods + + def __init__(self, *args, descriptor, event, **kwargs): + super().__init__(*args, **kwargs) + self.descriptor = descriptor + self.event = event + + def __repr__(self): + return '<{} iden={} fd={} event={}>'.format( + self.__class__.__name__, self.iden, self.descriptor.fd, self.event) + + def update(self, event): + '''Update the callback and fix descriptor's watchers''' + self.event = event + self.descriptor.update() + +# +# timeouts +# + +class TimeoutCallback(Callback): + '''Callback for timer''' + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.timeout = -1 + self._task = None + + def __repr__(self): + return '<{} iden={} timeout={}>'.format( + self.__class__.__name__, self.iden, self.timeout) + + @asyncio.coroutine + def _timer(self): + '''An actual timer running on the event loop. + + This is a coroutine. + ''' + while True: + try: + if self.timeout > 0: + timeout = self.timeout * 1e-3 + self.impl.log.debug('sleeping %r', timeout) + yield from asyncio.sleep(timeout) + else: + # scheduling timeout for next loop iteration + yield + + except asyncio.CancelledError: + self.impl.log.debug('timer %d cancelled', self.iden) + break + + self.cb(self.iden, self.opaque) + self.impl.log.debug('timer %r callback ended', self.iden) + + def update(self, timeout): + '''Start or the timer, possibly updating timeout''' + self.timeout = timeout + + if self.timeout >= 0 and self._task is None: + self.impl.log.debug('timer %r start', self.iden) + self._task = ensure_future(self._timer(), + loop=self.impl.loop) + + elif self.timeout < 0 and self._task is not None: + self.impl.log.debug('timer %r stop', self.iden) + self._task.cancel() # pylint: disable=no-member + self._task = None + + def close(self): + '''Stop the timer and call ff callback''' + super(TimeoutCallback, self).close() + self.update(timeout=-1) + +# +# main implementation +# + +class virEventAsyncIOImpl(object): + '''Libvirt event adapter to asyncio. + + :param loop: asyncio's event loop + + If *loop* is not specified, the current (or default) event loop is used. + ''' + + def __init__(self, loop=None): + self.loop = loop or asyncio.get_event_loop() + self.callbacks = {} + self.descriptors = DescriptorDict(self) + self.log = logging.getLogger(self.__class__.__name__) + + def register(self): + '''Register this instance as event loop implementation''' + # pylint: disable=bad-whitespace + self.log.debug('register()') + libvirt.virEventRegisterImpl( + self._add_handle, self._update_handle, self._remove_handle, + self._add_timeout, self._update_timeout, self._remove_timeout) + return self + + def schedule_ff_callback(self, opaque): + '''Schedule a ff callback from one of the handles or timers''' + self.loop.call_soon(libvirt.virEventInvokeFreeCallback, opaque) + + def is_idle(self): + '''Returns False if there are leftovers from a connection + + Those may happen if there are sematical problems while closing + a connection. For example, not deregistered events before .close(). + ''' + return not self.callbacks + + def _add_handle(self, fd, event, cb, opaque): + '''Register a callback for monitoring file handle events + + :param int fd: file descriptor to listen on + :param int event: bitset of events on which to fire the callback + :param cb: the callback to be called when an event occurrs + :param opaque: user data to pass to the callback + :rtype: int + :returns: handle watch number to be used for updating and unregistering for events + + .. seealso:: + https://libvirt.org/html/libvirt-libvirt-event.html#virEventAddHandleFuncFun... + ''' + self.log.debug('add_handle(fd=%d, event=%d, cb=%r, opaque=%r)', + fd, event, cb, opaque) + callback = FDCallback(self, cb, opaque, + descriptor=self.descriptors[fd], event=event) + self.callbacks[callback.iden] = callback + self.descriptors[fd].add_handle(callback) + return callback.iden + + def _update_handle(self, watch, event): + '''Change event set for a monitored file handle + + :param int watch: file descriptor watch to modify + :param int event: new events to listen on + + .. seealso:: + https://libvirt.org/html/libvirt-libvirt-event.html#virEventUpdateHandleFunc + ''' + self.log.debug('update_handle(watch=%d, event=%d)', watch, event) + return self.callbacks[watch].update(event=event) + + def _remove_handle(self, watch): + '''Unregister a callback from a file handle. + + :param int watch: file descriptor watch to stop listening on + :returns: None (see source for explanation) + + .. seealso:: + https://libvirt.org/html/libvirt-libvirt-event.html#virEventRemoveHandleFunc + ''' + self.log.debug('remove_handle(watch=%d)', watch) + callback = self.callbacks.pop(watch) + fd = callback.descriptor.fd + assert callback is self.descriptors[fd].remove_handle(watch) + if len(self.descriptors[fd].callbacks) == 0: + del self.descriptors[fd] + callback.close() + + def _add_timeout(self, timeout, cb, opaque): + '''Register a callback for a timer event + + :param int timeout: the timeout to monitor + :param cb: the callback to call when timeout has expired + :param opaque: user data to pass to the callback + :rtype: int + :returns: a timer value + + .. seealso:: + https://libvirt.org/html/libvirt-libvirt-event.html#virEventAddTimeoutFunc + ''' + self.log.debug('add_timeout(timeout=%d, cb=%r, opaque=%r)', + timeout, cb, opaque) + callback = TimeoutCallback(self, cb, opaque) + self.callbacks[callback.iden] = callback + callback.update(timeout=timeout) + return callback.iden + + def _update_timeout(self, timer, timeout): + '''Change frequency for a timer + + :param int timer: the timer to modify + :param int timeout: the new timeout value in ms + + .. seealso:: + https://libvirt.org/html/libvirt-libvirt-event.html#virEventUpdateTimeoutFun... + ''' + self.log.debug('update_timeout(timer=%d, timeout=%d)', timer, timeout) + return self.callbacks[timer].update(timeout=timeout) + + def _remove_timeout(self, timer): + '''Unregister a callback for a timer + + :param int timer: the timer to remove + :returns: None (see source for explanation) + + .. seealso:: + https://libvirt.org/html/libvirt-libvirt-event.html#virEventRemoveTimeoutFun... + ''' + self.log.debug('remove_timeout(timer=%d)', timer) + callback = self.callbacks.pop(timer) + callback.close() + +def virEventRegisterAsyncIOImpl(loop=None): + '''Arrange for libvirt's callbacks to be dispatched via asyncio event loop + + The implementation object is returned, but in normal usage it can safely be + discarded. + ''' + return virEventAsyncIOImpl(loop=loop).register() diff --git a/setup.py b/setup.py index 71998e2..c95170c 100755 --- a/setup.py +++ b/setup.py @@ -14,6 +14,7 @@ import sys import os import os.path import re +import shutil import time MIN_LIBVIRT = "0.9.11" @@ -50,6 +51,12 @@ def have_libvirt_lxc(): except DistutilsExecError: return False +def have_libvirtaio(): + # This depends on asyncio, which in turn depends on "yield from" syntax. + # The asyncio module itself is in standard library since 3.4, but there is + # an out-of-tree version compatible with 3.3. + return sys.version_info >= (3, 3) + def get_pkgconfig_data(args, mod, required=True): """Run pkg-config to and return content associated with it""" f = os.popen("%s %s %s" % (get_pkgcfg(), " ".join(args), mod)) @@ -124,6 +131,9 @@ def get_module_lists(): c_modules.append(modulelxc) py_modules.append("libvirt_lxc") + if have_libvirtaio(): + py_modules.append("libvirtaio") + return c_modules, py_modules @@ -141,6 +151,8 @@ class my_build(build): self.spawn([sys.executable, "generator.py", "libvirt-qemu", apis[1]]) if have_libvirt_lxc(): self.spawn([sys.executable, "generator.py", "libvirt-lxc", apis[2]]) + if have_libvirtaio(): + shutil.copy('libvirtaio.py', 'build') build.run(self) -- 2.9.3

Signed-off-by: Daniel P. Berrange <berrange@redhat.com> --- examples/event-test.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/examples/event-test.py b/examples/event-test.py index 851c09b..751a140 100755 --- a/examples/event-test.py +++ b/examples/event-test.py @@ -651,15 +651,17 @@ def usage(): print(" --help, -h Print(this help message") print(" --debug, -d Print(debug output") print(" --loop, -l Toggle event-loop-implementation") + print(" --timeout=SECS Quit after SECS seconds running") def main(): try: - opts, args = getopt.getopt(sys.argv[1:], "hdl", ["help", "debug", "loop"]) + opts, args = getopt.getopt(sys.argv[1:], "hdl", ["help", "debug", "loop", "timeout="]) except getopt.GetoptError as err: # print help information and exit: print(str(err)) # will print something like "option -a not recognized" usage() sys.exit(2) + timeout = None for o, a in opts: if o in ("-h", "--help"): usage() @@ -670,6 +672,8 @@ def main(): if o in ("-l", "--loop"): global use_pure_python_event_loop use_pure_python_event_loop ^= True + if o in ("--timeout"): + timeout = int(a) if len(args) >= 1: uri = args[0] @@ -741,7 +745,9 @@ def main(): # of demo we'll just go to sleep. The other option is to # run the event loop in your main thread if your app is # totally event based. - while run: + count = 0 + while run and (timeout is None or count < timeout): + count = count + 1 time.sleep(1) -- 2.9.3

Use the name 'Poll' instead of 'Pure' for the event loop demo, since there's now a second pure python loop impl available. Signed-off-by: Daniel P. Berrange <berrange@redhat.com> --- examples/event-test.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/examples/event-test.py b/examples/event-test.py index ac9fbe1..e67984f 100755 --- a/examples/event-test.py +++ b/examples/event-test.py @@ -39,10 +39,10 @@ def debug(msg): # # It is a pure python implementation based around the poll() API # -class virEventLoopPure: +class virEventLoopPoll: # This class contains the data we need to track for a # single file handle - class virEventLoopPureHandle: + class virEventLoopPollHandle: def __init__(self, handle, fd, events, cb, opaque): self.handle = handle self.fd = fd @@ -70,7 +70,7 @@ class virEventLoopPure: # This class contains the data we need to track for a # single periodic timer - class virEventLoopPureTimer: + class virEventLoopPollTimer: def __init__(self, timer, interval, cb, opaque): self.timer = timer self.interval = interval @@ -142,14 +142,14 @@ class virEventLoopPure: return next - # Lookup a virEventLoopPureHandle object based on file descriptor + # Lookup a virEventLoopPollHandle object based on file descriptor def get_handle_by_fd(self, fd): for h in self.handles: if h.get_fd() == fd: return h return None - # Lookup a virEventLoopPureHandle object based on its event loop ID + # Lookup a virEventLoopPollHandle object based on its event loop ID def get_handle_by_id(self, handleID): for h in self.handles: if h.get_id() == handleID: @@ -253,7 +253,7 @@ class virEventLoopPure: handleID = self.nextHandleID + 1 self.nextHandleID = self.nextHandleID + 1 - h = self.virEventLoopPureHandle(handleID, fd, events, cb, opaque) + h = self.virEventLoopPollHandle(handleID, fd, events, cb, opaque) self.handles.append(h) self.poll.register(fd, self.events_to_poll(events)) @@ -272,7 +272,7 @@ class virEventLoopPure: timerID = self.nextTimerID + 1 self.nextTimerID = self.nextTimerID + 1 - h = self.virEventLoopPureTimer(timerID, interval, cb, opaque) + h = self.virEventLoopPollTimer(timerID, interval, cb, opaque) self.timers.append(h) self.interrupt() @@ -361,7 +361,7 @@ class virEventLoopPure: # This single global instance of the event loop wil be used for # monitoring libvirt events -eventLoop = virEventLoopPure() +eventLoop = virEventLoopPoll() # This keeps track of what thread is running the event loop, # (if it is run in a background thread) @@ -371,7 +371,7 @@ eventLoopThread = None # These next set of 6 methods are the glue between the official # libvirt events API, and our particular impl of the event loop # -# There is no reason why the 'virEventLoopPure' has to be used. +# There is no reason why the 'virEventLoopPoll' has to be used. # An application could easily may these 6 glue methods hook into # another event loop such as GLib's, or something like the python # Twisted event framework. @@ -402,7 +402,7 @@ def virEventRemoveTimerImpl(timerID): # This tells libvirt what event loop implementation it # should use -def virEventLoopPureRegister(): +def virEventLoopPollRegister(): libvirt.virEventRegisterImpl(virEventAddHandleImpl, virEventUpdateHandleImpl, virEventRemoveHandleImpl, @@ -411,7 +411,7 @@ def virEventLoopPureRegister(): virEventRemoveTimerImpl) # Directly run the event loop in the current thread -def virEventLoopPureRun(): +def virEventLoopPollRun(): global eventLoop eventLoop.run_loop() @@ -420,10 +420,10 @@ def virEventLoopNativeRun(): libvirt.virEventRunDefaultImpl() # Spawn a background thread to run the event loop -def virEventLoopPureStart(): +def virEventLoopPollStart(): global eventLoopThread - virEventLoopPureRegister() - eventLoopThread = threading.Thread(target=virEventLoopPureRun, name="libvirtEventLoop") + virEventLoopPollRegister() + eventLoopThread = threading.Thread(target=virEventLoopPollRun, name="libvirtEventLoop") eventLoopThread.setDaemon(True) eventLoopThread.start() @@ -684,7 +684,7 @@ def main(): # Run a background thread with the event loop if use_pure_python_event_loop: - virEventLoopPureStart() + virEventLoopPollStart() else: virEventLoopNativeStart() -- 2.9.3