[libvirt] [Patch 0/2] Add new feature into live migration
If the memory of guest OS is changed constantly, the live migration can not be ended ever for ever. We can use the command 'virsh migrate-setmaxdowntime' to control the live migration. But the value of maxdowntime is diffcult to calculate because it depends on the transfer speed of network and constantly changing memroy size. We need a easy way to control the live migration. This patch set add the support of auto cold migration fallback on timeout. With this patch set, when we migrate the guest OS, we can specify a timeout. If the live migration timeouts, the migration will fallback to cold migration. TODO: 1. The timer for Windows has not been implemented. Test of this patchset: Env: a. The size of guest OS's memory: 1GB b. The transfer speed of network: about 100Mb/s c. The size of constantly changing memory: more than 900MB 1. migrate without timeout # virsh migrate --live RHEL6RC qemu+ssh://<dest IP>/system tcp://<dest IP>:49152 The migration does not end after 12 hours. 2. migrate with timeout(30 minutes): # date Tue Dec 7 10:00:13 CST 2010 # virsh migrate --live --timeout 1800 RHEL6RC qemu+ssh://10.167.225.67/system tcp:10.167.225.67:49152 # date Tue Dec 7 10:43:28 CST 2010 Wen Congyang (2): timer impl auto cold migration fallback at timeout configure.ac | 4 ++ src/Makefile.am | 6 ++- src/libvirt.c | 2 + src/libvirt_private.syms | 6 ++ src/util/timer.c | 116 ++++++++++++++++++++++++++++++++++++++++++++ src/util/timer.h | 37 ++++++++++++++ src/util/timer_linux.c | 121 ++++++++++++++++++++++++++++++++++++++++++++++ tools/virsh.c | 53 ++++++++++++++++++++ 8 files changed, 343 insertions(+), 2 deletions(-) create mode 100644 src/util/timer.c create mode 100644 src/util/timer.h create mode 100644 src/util/timer_linux.c
* src/util/timer.c src/util/timer.h src/util/timer_linux.c: timer implementation * src/Makefile.am: build timer * src/libvirt_private.syms: Export public functions * src/libvirt.c: Initialize timer * configure.ac: check the functions in librt used by timer Signed-off-by: Wen Congyang <wency@cn.fujitsu.com> --- configure.ac | 4 ++ src/Makefile.am | 6 ++- src/libvirt.c | 2 + src/libvirt_private.syms | 6 ++ src/util/timer.c | 116 ++++++++++++++++++++++++++++++++++++++++++++ src/util/timer.h | 37 ++++++++++++++ src/util/timer_linux.c | 121 ++++++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 290 insertions(+), 2 deletions(-) create mode 100644 src/util/timer.c create mode 100644 src/util/timer.h create mode 100644 src/util/timer_linux.c diff --git a/configure.ac b/configure.ac index d26bc68..4e218df 100644 --- a/configure.ac +++ b/configure.ac @@ -107,6 +107,10 @@ LIBS="$LIBS $LIB_PTHREAD" AC_CHECK_FUNCS([pthread_sigmask pthread_mutexattr_init]) LIBS=$old_libs +dnl Availability of rt functions +AC_CHECK_LIB([rt],[timer_gettime],[]) +AC_CHECK_FUNCS([clock_gettime timer_create timer_settime timer_delete]) + dnl Availability of various common headers (non-fatal if missing). AC_CHECK_HEADERS([pwd.h paths.h regex.h sys/syslimits.h sys/un.h \ sys/poll.h syslog.h mntent.h net/ethernet.h linux/magic.h \ diff --git a/src/Makefile.am b/src/Makefile.am index 0923d60..05c1140 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -78,9 +78,11 @@ UTIL_SOURCES = \ util/util.c util/util.h \ util/xml.c util/xml.h \ util/virtaudit.c util/virtaudit.h \ - util/virterror.c util/virterror_internal.h + util/virterror.c util/virterror_internal.h \ + util/timer.c util/timer.h -EXTRA_DIST += util/threads-pthread.c util/threads-win32.c +EXTRA_DIST += util/threads-pthread.c util/threads-win32.c \ + util/timer_linux.c # Internal generic driver infrastructure NODE_INFO_SOURCES = nodeinfo.h nodeinfo.c diff --git a/src/libvirt.c b/src/libvirt.c index 4188b45..a5578c0 100644 --- a/src/libvirt.c +++ b/src/libvirt.c @@ -40,6 +40,7 @@ #include "util.h" #include "memory.h" #include "configmake.h" +#include "timer.h" #ifndef WITH_DRIVER_MODULES # ifdef WITH_TEST @@ -329,6 +330,7 @@ virInitialize(void) if (virThreadInitialize() < 0 || virErrorInitialize() < 0 || + virTimerInitialize() < 0 || virRandomInitialize(time(NULL) ^ getpid())) return -1; diff --git a/src/libvirt_private.syms b/src/libvirt_private.syms index e2def6c..d76bce3 100644 --- a/src/libvirt_private.syms +++ b/src/libvirt_private.syms @@ -779,6 +779,12 @@ virThreadJoin; virThreadSelf; virThreadSelfID; +# timer.h +get_clock; +virNewTimer; +virFreeTimer; +virModTimer; +virDelTimer; # usb.h usbDeviceFileIterate; diff --git a/src/util/timer.c b/src/util/timer.c new file mode 100644 index 0000000..286d602 --- /dev/null +++ b/src/util/timer.c @@ -0,0 +1,116 @@ +/* + * timer.c: timer functions + * + * Copyright (C) 2010 Fujitsu Limited + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + * Author: Wen Congyang <wency@cn.fujitsu.com> + */ + +#include <config.h> + +#include "timer.h" +#include "virterror_internal.h" +#include "memory.h" + +#define VIR_FROM_THIS VIR_FROM_NONE + +struct virTimer { + uint64_t expire_time; + virTimerCB function; + void *data; + virTimerPtr next; +}; + +static virTimerPtr timer_list = NULL; + +virTimerPtr virNewTimer(virTimerCB callback, void *data) +{ + virTimerPtr timer = NULL; + + if (VIR_ALLOC(timer) < 0) { + virReportOOMError(); + return NULL; + } + + timer->function = callback; + timer->data = data; + + return timer; +} + +void virFreeTimer(virTimerPtr timer) +{ + VIR_FREE(timer); +} + +void virDelTimer(virTimerPtr timer) +{ + virTimerPtr *pt, t; + + pt = &timer_list; + for (t = *pt; t; t = t->next) { + if (t == timer) { + *pt = t->next; + break; + } + pt = &t->next; + } +} + +void virModTimer(virTimerPtr timer, uint64_t expire_time) +{ + virTimerPtr *pt, t; + + virDelTimer(timer); + + pt = &timer_list; + for (t = *pt; t; t = t->next) { + if (t->expire_time > expire_time) + break; + pt = &t->next; + } + timer->expire_time = expire_time; + timer->next = *pt; + *pt = timer; +} + +static void timer_handler(void) +{ + uint64_t current_time = 0; + virTimerPtr *timer_list_head = &timer_list; + virTimerPtr timer = NULL; + + if (!timer_list) + return; + + current_time = get_clock(); + for (timer = timer_list; timer; timer = timer->next) { + if (current_time < timer->expire_time) + break; + + /* remove timer from the list before calling the callback */ + *timer_list_head = timer->next; + timer->next = NULL; + timer->function(timer->data); + } +} + +#if defined HAVE_TIMER_CREATE +#include "timer_linux.c" +#else +# error "gnu timer is required" +#endif diff --git a/src/util/timer.h b/src/util/timer.h new file mode 100644 index 0000000..f521c1b --- /dev/null +++ b/src/util/timer.h @@ -0,0 +1,37 @@ +/* + * timer.h: structure and entry points for timer support + * + * Copyright (C) 2010 Fujitsu Limited + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + * Author: Wen Congyang <wency@cn.fujitsu.com> + */ + +#ifndef __VIR_TIMER_H__ +# define __VIR_TIMER_H__ + +#include <stdint.h> +typedef struct virTimer virTimer; +typedef virTimer *virTimerPtr; +typedef void (*virTimerCB)(void *); + +extern uint64_t get_clock(void); +extern virTimerPtr virNewTimer(virTimerCB, void *); +extern void virFreeTimer(virTimerPtr); +extern void virModTimer(virTimerPtr, uint64_t); +extern void virDelTimer(virTimerPtr); +extern int virTimerInitialize(void); +#endif /* __VIR_TIMER_H__ */ diff --git a/src/util/timer_linux.c b/src/util/timer_linux.c new file mode 100644 index 0000000..30e189e --- /dev/null +++ b/src/util/timer_linux.c @@ -0,0 +1,121 @@ +/* + * timer.c: timer functions + * + * Copyright (C) 2010 Fujitsu Limited + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + * Author: Wen Congyang <wency@cn.fujitsu.com> + */ + +#include <unistd.h> +#include <sys/syscall.h> +#include <sys/types.h> +#include <stddef.h> +#include <time.h> +#include <sys/time.h> +#include <signal.h> +#include <sys/select.h> +#include <pthread.h> + +#include "timer.h" +#include "threads.h" + +static int timer_initialized = 0; +static int timer_fd[2] = {-1, -1}; +static pthread_attr_t timer_thread_attr; +static timer_t timer_id; + +uint64_t get_clock(void) +{ + struct timespec ts; + + clock_gettime(CLOCK_MONOTONIC, &ts); + return ts.tv_sec * 1000000000ULL + ts.tv_nsec; +} + +static void timer_loop(void *arg ATTRIBUTE_UNUSED) +{ + fd_set readfds; + int ret; + int readinfo = 0; + + while(1) { + FD_ZERO(&readfds); + FD_SET(timer_fd[0], &readfds); + + reselect: + ret = select(timer_fd[0] + 1, &readfds, NULL, NULL, NULL); + if (ret < 0) { + if (errno == EAGAIN || errno == EINTR) + goto reselect; + else + continue; + } + + ret = read(timer_fd[0], &readinfo, sizeof(int)); + timer_handler(); + } + + return; +} + +static void timer_nofity_thread(sigval_t sigev_value ATTRIBUTE_UNUSED) +{ + int write_info = 1; + int ret; + + ret = write(timer_fd[1], &write_info, sizeof(int)); +} + +int virTimerInitialize(void) +{ + struct sigevent timer_event; + struct itimerspec its; + struct virThread timer_thread; + int ret; + + if (timer_initialized) + return 0; + + ret = pipe(timer_fd); + if (ret < 0) + return -1; + + pthread_attr_init(&timer_thread_attr); + pthread_attr_setdetachstate(&timer_thread_attr, 1); + + timer_event.sigev_notify = SIGEV_THREAD; + timer_event.sigev_notify_function = timer_nofity_thread; + timer_event.sigev_notify_attributes = &timer_thread_attr; + timer_event.sigev_value.sival_ptr = NULL; + timer_create(CLOCK_MONOTONIC, &timer_event, &timer_id); + + its.it_interval.tv_sec = 0; + its.it_interval.tv_nsec = 99 * 1000; + its.it_value.tv_sec = 0; + its.it_value.tv_nsec = 10L * 1000L * 1000L; + + ret = timer_settime(timer_id, 0, &its, NULL); + if (ret < 0) + return -1; + + if (virThreadCreate(&timer_thread, 0, timer_loop, NULL) < 0) { + timer_delete(timer_id); + return -1; + } + + return 0; +} -- 1.7.1
implement auto cold migration fallback at timeout Signed-off-by: Wen Congyang <wency@cn.fujitsu.com> --- tools/virsh.c | 53 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 files changed, 53 insertions(+), 0 deletions(-) diff --git a/tools/virsh.c b/tools/virsh.c index 441fd77..c6b9009 100644 --- a/tools/virsh.c +++ b/tools/virsh.c @@ -54,6 +54,7 @@ #include "files.h" #include "../daemon/event.h" #include "configmake.h" +#include "timer.h" static char *progname; @@ -3377,9 +3378,32 @@ static const vshCmdOptDef opts_migrate[] = { {"desturi", VSH_OT_DATA, VSH_OFLAG_REQ, N_("connection URI of the destination host")}, {"migrateuri", VSH_OT_DATA, 0, N_("migration URI, usually can be omitted")}, {"dname", VSH_OT_DATA, 0, N_("rename to new name during migration (if supported)")}, + {"timeout", VSH_OT_INT, VSH_OFLAG_REQ, N_("auto cold migration fallback when live migration timeouts(in seconds)")}, {NULL, 0, 0, NULL} }; +static void migrateTimeoutHandler(void *data) +{ + virDomainPtr dom = (virDomainPtr)data; + virDomainInfo info; + unsigned int id; + + id = virDomainGetID(dom); + if (id == ((unsigned int)-1)) + return; + + /* The error reason has been reported in virDomainGetInfo() and + * virDomainSuspend() when it fails. So we do not check the return value. + */ + if (virDomainGetInfo(dom, &info) == 0) { + if (info.state == VIR_DOMAIN_SHUTOFF) + return; + + /* suspend the domain when migration timeouts. */ + virDomainSuspend(dom); + } +} + static int cmdMigrate (vshControl *ctl, const vshCmd *cmd) { @@ -3387,6 +3411,8 @@ cmdMigrate (vshControl *ctl, const vshCmd *cmd) const char *desturi; const char *migrateuri; const char *dname; + long long timeout; + virTimerPtr migratetimer = NULL; int flags = 0, found, ret = FALSE; if (!vshConnectionUsability (ctl, ctl->conn)) @@ -3424,6 +3450,23 @@ cmdMigrate (vshControl *ctl, const vshCmd *cmd) if (vshCommandOptBool (cmd, "copy-storage-inc")) flags |= VIR_MIGRATE_NON_SHARED_INC; + timeout = vshCommandOptLongLong(cmd, "timeout", &found); + if (found) { + if (! flags & VIR_MIGRATE_LIVE) { + vshError(ctl, "%s", _("migrate: Unexpected timeout for cold migration")); + goto done; + } + + if (timeout < 1) { + vshError(ctl, "%s", _("migrate: Invalid timeout")); + goto done; + } + + migratetimer = virNewTimer(migrateTimeoutHandler, (void *)dom); + if (!migratetimer) + goto done; + } + if ((flags & VIR_MIGRATE_PEER2PEER) || vshCommandOptBool (cmd, "direct")) { /* For peer2peer migration or direct migration we only expect one URI @@ -3434,6 +3477,9 @@ cmdMigrate (vshControl *ctl, const vshCmd *cmd) goto done; } + if (migratetimer) + virModTimer(migratetimer, get_clock() + timeout * 1000000000ULL); + if (virDomainMigrateToURI (dom, desturi, flags, dname, 0) == 0) ret = TRUE; } else { @@ -3444,6 +3490,9 @@ cmdMigrate (vshControl *ctl, const vshCmd *cmd) dconn = virConnectOpenAuth (desturi, virConnectAuthPtrDefault, 0); if (!dconn) goto done; + if (migratetimer) + virModTimer(migratetimer, get_clock() + timeout * 1000000000ULL); + ddom = virDomainMigrate (dom, dconn, flags, dname, migrateuri, 0); if (ddom) { virDomainFree(ddom); @@ -3453,6 +3502,10 @@ cmdMigrate (vshControl *ctl, const vshCmd *cmd) } done: + if (migratetimer) { + virDelTimer(migratetimer); + virFreeTimer(migratetimer); + } if (dom) virDomainFree (dom); return ret; } -- 1.7.1
participants (1)
-
Wen Congyang