It has two modes, normal and interactive. In normal mode, messages are passed from libvirt to qemu and back as they come and copyied to stdout so that one can monitor the communication. In interactive mode, whenever a message comes from either libvirt or qemu, it is printed to stdout and the user is asked what action to take. One can also inject any message to either libvirt or qemu. Interactive mode can be entered by sending any command to monitor-proxy (sending an empty command, i.e., hitting enter enters interactive mode without doing anything else). The proxy can also be started directly in interactive mode by specifying "-i" command line option. Command "cont" can be used to return back to normal mode. For list of supported commands and their description, use "help" or "h" command. To start monitor-proxy one needs to follow the following steps: - start a domain - run monitor-proxy [-i] <monitor socket path> for example: monitor-proxy /var/lib/libvirt/qemu/vm1.monitor - restart libvirtd (this makes libvirtd disconnect from qemu and reconnect to the proxy) monitor-proxy supports libvirtd restarts, i.e., if libvirtd disconnects, the proxy just waits until it reconnects again. Stopping the proxy while libvirtd is running kills the domain since libvirtd sees monitor EOF. To stop using monitor-proxy without killing the domain, one needs to stop libvirtd, stop monitor-proxy, and start libvirtd. The proxy works by renaming qemu monitor socket so that it can listen at the original place for connections from libvirtd. Once libvirtd connects to monitor-proxy, the proxy connects to qemu through the renamed socket. The socket is renamed back to its original name when proxy is killed or asked to exit. The proxy also supports fd passing through SCM_RIGHTS so to be usable for testing migrations. --- Makefile.am | 3 +- configure.ac | 3 +- examples/monitor-proxy/Makefile.am | 3 + examples/monitor-proxy/README | 36 ++ examples/monitor-proxy/monitor-proxy.c | 751 ++++++++++++++++++++++++++++++++ 5 files changed, 794 insertions(+), 2 deletions(-) create mode 100644 examples/monitor-proxy/Makefile.am create mode 100644 examples/monitor-proxy/README create mode 100644 examples/monitor-proxy/monitor-proxy.c diff --git a/Makefile.am b/Makefile.am index 49e42bf..1ac65e3 100644 --- a/Makefile.am +++ b/Makefile.am @@ -6,7 +6,8 @@ GENHTML = genhtml SUBDIRS = gnulib/lib include src daemon tools docs gnulib/tests \ python tests po examples/domain-events/events-c examples/hellolibvirt \ examples/dominfo examples/domsuspend examples/python examples/apparmor \ - examples/xml/nwfilter examples/openauth examples/systemtap + examples/xml/nwfilter examples/openauth examples/systemtap \ + examples/monitor-proxy ACLOCAL_AMFLAGS = -I m4 -I gnulib/m4 diff --git a/configure.ac b/configure.ac index cd22afb..9d2fe04 100644 --- a/configure.ac +++ b/configure.ac @@ -2414,7 +2414,8 @@ AC_OUTPUT(Makefile src/Makefile include/Makefile docs/Makefile \ examples/python/Makefile \ examples/hellolibvirt/Makefile \ examples/systemtap/Makefile \ - examples/xml/nwfilter/Makefile) + examples/xml/nwfilter/Makefile \ + examples/monitor-proxy/Makefile) AC_MSG_NOTICE([]) AC_MSG_NOTICE([Configuration summary]) diff --git a/examples/monitor-proxy/Makefile.am b/examples/monitor-proxy/Makefile.am new file mode 100644 index 0000000..4fbbed5 --- /dev/null +++ b/examples/monitor-proxy/Makefile.am @@ -0,0 +1,3 @@ +noinst_PROGRAMS = monitor-proxy +monitor_proxy_CFLAGS = $(WARN_CFLAGS) +monitor_proxy_SOURCES = monitor-proxy.c diff --git a/examples/monitor-proxy/README b/examples/monitor-proxy/README new file mode 100644 index 0000000..bf00460 --- /dev/null +++ b/examples/monitor-proxy/README @@ -0,0 +1,36 @@ +monitor-proxy is meant as a developer tool for testing libvirt-qemu +communication. + +It has two modes, normal and interactive. In normal mode, messages are passed +from libvirt to qemu and back as they come and copyied to stdout so that one +can monitor the communication. In interactive mode, whenever a message comes +from either libvirt or qemu, it is printed to stdout and the user is asked +what action to take. One can also inject any message to either libvirt or +qemu. Interactive mode can be entered by sending any command to monitor-proxy +(sending an empty command, i.e., hitting enter enters interactive mode without +doing anything else). The proxy can also be started directly in interactive +mode by specifying "-i" command line option. Command "cont" can be used to +return back to normal mode. For list of supported commands and their +description, use "help" or "h" command. + +To start monitor-proxy one needs to follow the following steps: +- start a domain +- run monitor-proxy [-i] <monitor socket path> + for example: monitor-proxy /var/lib/libvirt/qemu/vm1.monitor +- restart libvirtd (this makes libvirtd disconnect from qemu and reconnect to + the proxy) + +monitor-proxy supports libvirtd restarts, i.e., if libvirtd disconnects, the +proxy just waits until it reconnects again. Stopping the proxy while libvirtd +is running kills the domain since libvirtd sees monitor EOF. To stop using +monitor-proxy without killing the domain, one needs to stop libvirtd, stop +monitor-proxy, and start libvirtd. + +The proxy works by renaming qemu monitor socket so that it can listen at the +original place for connections from libvirtd. Once libvirtd connects to +monitor-proxy, the proxy connects to qemu through the renamed socket. The +socket is renamed back to its original name when proxy is killed or asked to +exit. + +The proxy also supports fd passing through SCM_RIGHTS so to be usable for +testing migrations. diff --git a/examples/monitor-proxy/monitor-proxy.c b/examples/monitor-proxy/monitor-proxy.c new file mode 100644 index 0000000..403033d --- /dev/null +++ b/examples/monitor-proxy/monitor-proxy.c @@ -0,0 +1,751 @@ +#include <stdlib.h> +#include <stdio.h> +#include <stdbool.h> +#include <stdarg.h> +#include <string.h> +#include <signal.h> +#include <unistd.h> +#include <fcntl.h> +#include <errno.h> +#include <sys/select.h> +#include <sys/socket.h> +#include <sys/un.h> + +# ifndef ATTRIBUTE_FMT_PRINTF +# if __GNUC_PREREQ (4, 4) +# define ATTRIBUTE_FMT_PRINTF(fmtpos,argpos) __attribute__((__format__ (gnu_printf, fmtpos,argpos))) +# else +# define ATTRIBUTE_FMT_PRINTF(fmtpos,argpos) __attribute__((__format__ (printf, fmtpos,argpos))) +# endif +# endif + +# ifndef ATTRIBUTE_NORETURN +# define ATTRIBUTE_NORETURN __attribute__((__noreturn__)) +# endif + +#ifndef UNIX_PATH_MAX +# define UNIX_PATH_MAX 108 +#endif +#define SOCK_SUFFIX ".proxy" +#define SIZE (1 << 14) + +#define LABEL_LIBVIRT "libvirt> " +#define LABEL_QEMU "qemu> " +#define LABEL_PROMPT "command> " + +enum party { + PARTY_QEMU, + PARTY_LIBVIRT +}; + +enum command { + CMD_NONE, + CMD_NOOP, + CMD_UNKNOWN, + CMD_CONTINUE, + CMD_QUIT, + CMD_HELP, + CMD_PASS, + CMD_DISCARD, + CMD_SEND_QEMU, + CMD_SEND_LIBVIRT +}; + +struct proxy_command { + enum command command; + const char *name; + bool shortcut; + bool argument; + const char *description; +}; + +const char *monitor; +char *monitor_qemu; + +char *buffer; +char *control; +char *commands; + +struct message { + enum party source; + int fd_qemu; + int fd_libvirt; + struct msghdr hdr; +}; + + +#define ncommands (sizeof(proxy_commands) / sizeof(*proxy_commands)) +static const struct proxy_command proxy_commands[] = { + { + CMD_CONTINUE, + "cont", true, false, + "leave interactive mode" + }, { + CMD_QUIT, + "quit", false, false, + "quit monitor-proxy" + }, { + CMD_HELP, + "help", true, false, + "print this help" + }, { + CMD_PASS, + "pass", true, false, + "pass current data to its destination" + }, { + CMD_DISCARD, + "discard", true, false, + "discard current data" + }, { + CMD_SEND_QEMU, + "qemu", true, true, + "send DATA to qemu" + }, { + CMD_SEND_LIBVIRT, + "libvirt", true, true, + "send DATA to libvirt" + } +}; + + +static void ATTRIBUTE_NORETURN +usage(int ret) +{ + fprintf(stderr, "monitor-proxy [-i] <qemu monitor socket>\n"); + exit(ret); +} + + +static void ATTRIBUTE_FMT_PRINTF(1, 2) +print(const char *msg, ...) +{ + va_list args; + + printf("* "); + va_start(args, msg); + vprintf(msg, args); + va_end(args); + printf("\n"); + fflush(NULL); +} + + +#define HELP_LINE 80 +#define HELP_NAME 20 +static void +help(void) +{ + unsigned int i; + char line[HELP_LINE]; + char name[HELP_NAME]; + + print("Available proxy commands:"); + for (i = 0; i < ncommands; i++) { + const struct proxy_command *cmd = proxy_commands + i; + + if (cmd->shortcut) + snprintf(name, HELP_NAME, "%c(%s)", cmd->name[0], cmd->name + 1); + else + strncpy(name, cmd->name, HELP_NAME); + name[HELP_NAME - 1] = '\0'; + if (cmd->argument) { + strncpy(name + strlen(name), " DATA", HELP_NAME - strlen(name)); + name[HELP_NAME - 1] = '\0'; + } + + snprintf(line, HELP_LINE, " %-*s %s", + HELP_NAME, name, cmd->description); + line[HELP_LINE - 1] = '\0'; + print("%s", line); + } + print("Signals:"); + print(" SIGINT (^C) quit monitor-proxy"); +} + + +static int +move_socket(void) +{ + if (monitor_qemu) { + fprintf(stderr, "Socket already moved to %s\n", monitor_qemu); + return -1; + } + + monitor_qemu = malloc(strlen(monitor) + strlen(SOCK_SUFFIX) + 1); + if (!monitor_qemu) + return -1; + + sprintf(monitor_qemu, "%s%s", monitor, SOCK_SUFFIX); + + if (rename(monitor, monitor_qemu) < 0) { + perror("Cannot move monitor socket"); + free(monitor_qemu); + monitor_qemu = NULL; + return -1; + } + + print("Monitor socket renamed as %s", monitor_qemu); + + return 0; +} + + +static int +restore_socket(void) +{ + if (!monitor_qemu) + return 0; + + if (rename(monitor_qemu, monitor) < 0) { + perror("Cannot restore monitor socket"); + return -1; + } + + print("Monitor socket renamed back to %s", monitor); + + free(monitor_qemu); + monitor_qemu = NULL; + + return 0; +} + + +static void ATTRIBUTE_NORETURN +signal_handler(int signum) +{ + restore_socket(); + + if (signum == SIGINT) + exit(0); + else + exit(1); +} + + +static int +open_qemu(void) +{ + struct sockaddr_un addr; + int fd = -1; + + if ((fd = socket(PF_UNIX, SOCK_STREAM, 0)) == -1) { + perror("Cannot create monitor socket"); + goto error; + } + + addr.sun_family = AF_UNIX; + strncpy(addr.sun_path, monitor_qemu, UNIX_PATH_MAX); + addr.sun_path[UNIX_PATH_MAX - 1] = '\0'; + + if (connect(fd, (struct sockaddr *) &addr, sizeof(addr)) < 0) { + perror("Cannot connect to qemu monitor"); + goto error; + } + + print("Connected to qemu monitor"); + + return fd; + +error: + close(fd); + return -1; +} + + +static int +listen_libvirt(void) +{ + struct sockaddr_un addr; + int fd = -1; + + if ((fd = socket(PF_UNIX, SOCK_STREAM, 0)) == -1) { + perror("Cannot create listening socket"); + goto error; + } + + addr.sun_family = AF_UNIX; + strncpy(addr.sun_path, monitor, UNIX_PATH_MAX); + addr.sun_path[UNIX_PATH_MAX - 1] = '\0'; + + if (bind(fd, (struct sockaddr *) &addr, sizeof(addr)) < 0) { + perror("Cannot bind listening socket"); + goto error; + } + + if (listen(fd, 1) < 0) { + perror("Cannot listen"); + goto error; + } + + return fd; + +error: + close(fd); + return -1; +} + + +static int +accept_libvirt(int sock) +{ + int fd = -1; + + print("Waiting for libvirt connection"); + + if ((fd = accept(sock, NULL, NULL)) == -1) { + perror("Cannot accept connection from libvirt"); + goto error; + } + + print("Libvirt connection accepted"); + + return fd; + +error: + close(fd); + return -1; +} + + +static int +init_message(struct message *msg, int qemu, int libvirt) +{ + struct iovec *iov; + + if (!(iov = malloc(sizeof(*iov)))) + return -1; + + memset(msg, 0, sizeof(*msg)); + msg->fd_qemu = qemu; + msg->fd_libvirt = libvirt; + msg->hdr.msg_iov = iov; + msg->hdr.msg_iovlen = 1; + + return 0; +} + + +static ssize_t +receive_message(enum party source, struct message *msg) +{ + ssize_t count; + int fd = source == PARTY_QEMU ? msg->fd_qemu : msg->fd_libvirt; + + msg->source = source; + msg->hdr.msg_iov[0].iov_base = buffer; + msg->hdr.msg_iov[0].iov_len = SIZE; + msg->hdr.msg_control = control; + msg->hdr.msg_controllen = SIZE; + + if ((count = recvmsg(fd, &msg->hdr, 0)) >= 0) { + msg->hdr.msg_iov[0].iov_len = count; + if (msg->hdr.msg_controllen == 0) + msg->hdr.msg_control = NULL; + } + + return count; +} + + +static void +send_message(enum party target, struct message *msg) +{ + int fd = target == PARTY_QEMU ? msg->fd_qemu : msg->fd_libvirt; + + if (sendmsg(fd, &msg->hdr, 0) < 0) { + if (target == PARTY_QEMU) { + perror("Cannot send message to qemu"); + } else { + perror("Cannot send message to libvirt"); + } + } + + msg->hdr.msg_iov[0].iov_base = NULL; + msg->hdr.msg_iov[0].iov_len = 0; + msg->hdr.msg_control = NULL; + msg->hdr.msg_controllen = 0; +} + + +static int +write_all(int fd, const char *buf, size_t count) +{ + size_t written = 0; + ssize_t w; + + while (count > 0) { + w = write(fd, buf, count); + + if (w == 0) { + break; + } else if (w < 0) { + if (errno == EINTR) + continue; + perror("Cannot write data"); + return -1; + } + + buf += w; + count -= w; + written += w; + } + + return written; +} + + +static int +output(const char *label, const struct message *msg) +{ + ssize_t start = 0; + char *buf = msg->hdr.msg_iov[0].iov_base; + ssize_t count = msg->hdr.msg_iov[0].iov_len; + + while (start < count) { + ssize_t len = 0; + const char* p; + + if (write_all(STDOUT_FILENO, label, strlen(label)) < 0) + return -1; + + p = buf + start; + while (start + len < count && *p != '\n') { + len++; + p++; + } + if (start + len < count) + len++; + + if (write_all(STDOUT_FILENO, buf + start, len) < 0) + return -1; + + start += len; + } + + return 0; +} + + +static ssize_t +read_line(char *buf, size_t size, bool data) +{ + ssize_t count = 0; + struct timeval tv = { 0, 0 }; + struct timeval *timeout; + fd_set fds; + int ret; + + FD_ZERO(&fds); + FD_SET(STDIN_FILENO, &fds); + + if (data) + timeout = NULL; + else + timeout = &tv; + + ret = select(STDIN_FILENO + 1, &fds, NULL, NULL, timeout); + if (ret < 0) + return -1; + if (ret == 0) + return 0; + + while (count < size - 1 && read(STDIN_FILENO, buf, 1) == 1) { + count++; + if (*buf == '\n') { + *buf = '\r'; + *++buf = '\n'; + count++; + break; + } + buf++; + } + return count; +} + + +static enum command +get_command(char **cmd_buf, ssize_t *cmd_len, bool data) +{ + ssize_t len; + unsigned int i; + + if (data) + write_all(STDOUT_FILENO, LABEL_PROMPT, strlen(LABEL_PROMPT)); + + len = read_line(commands, SIZE, data); + if (len < 0) + return CMD_QUIT; + if (len == 0) + return CMD_NONE; + if (len <= 2) + return CMD_NOOP; + + commands[len] = '\0'; + + for (i = 0; i < ncommands; i++) { + const struct proxy_command *cmd = proxy_commands + i; + int skip; + + if (strncmp(commands, cmd->name, strlen(cmd->name)) == 0) + skip = strlen(cmd->name); + else if (cmd->shortcut && *commands == *cmd->name) + skip = 1; + else + continue; + + if (cmd->argument) { + if (commands[skip] == ' ') { + skip++; + *cmd_buf = commands + skip; + *cmd_len = len - skip; + return cmd->command; + } + } else { + if (strncmp(commands + skip, "\r\n", 2) == 0) + return cmd->command; + } + } + + commands[len - 2] = '\0'; + *cmd_buf = commands; + *cmd_len = len - 2; + return CMD_UNKNOWN; +} + + +static int +process_commands(struct message *msg, bool *interactive) +{ + enum party source = msg->source; + ssize_t data_len = msg->hdr.msg_iov[0].iov_len; + ssize_t control_len = msg->hdr.msg_controllen; + bool data = data_len > 0 || control_len > 0; + enum command cmd; + char *cmd_buf = NULL; + ssize_t cmd_len = 0; + + msg->hdr.msg_iov[0].iov_base = NULL; + msg->hdr.msg_iov[0].iov_len = 0; + msg->hdr.msg_control = NULL; + msg->hdr.msg_controllen = 0; + + if (data) { + print("Processing data from %s", + source == PARTY_QEMU ? "qemu" : "libvirt"); + print(" Enter proxy commands (type \"help\" to list them)"); + print(" Finish processing with \"pass\" or \"discard\" command"); + } + + while (*interactive && + (cmd = get_command(&cmd_buf, &cmd_len, data)) != CMD_NONE) { + switch (cmd) { + case CMD_NONE: + case CMD_NOOP: + break; + + case CMD_QUIT: + return -1; + + case CMD_HELP: + help(); + break; + + case CMD_UNKNOWN: + print("Unknown command: \"%s\"", cmd_buf); + break; + + case CMD_CONTINUE: + *interactive = false; + print("Leaving interactive mode"); + case CMD_PASS: + if (data_len > 0) { + msg->hdr.msg_iov[0].iov_len = data_len; + msg->hdr.msg_iov[0].iov_base = buffer; + } + if (control_len > 0) { + msg->hdr.msg_controllen = control_len; + msg->hdr.msg_control = control; + } + if (data) { + print("Passing data from %s to %s", + source == PARTY_QEMU ? "qemu" : "libvirt", + source == PARTY_QEMU ? "libvirt" : "qemu"); + send_message(source == PARTY_QEMU ? PARTY_LIBVIRT : PARTY_QEMU, + msg); + data = false; + } else { + if (cmd == CMD_PASS) + print("No data to pass"); + } + break; + + case CMD_DISCARD: + if (data) { + print("Discarding data from %s", + source == PARTY_QEMU ? "qemu" : "libvirt"); + } else { + print("No data to discard"); + } + data_len = control_len = 0; + data = false; + break; + + case CMD_SEND_QEMU: + case CMD_SEND_LIBVIRT: + msg->hdr.msg_iov[0].iov_len = cmd_len; + msg->hdr.msg_iov[0].iov_base = cmd_buf; + send_message(cmd == CMD_SEND_QEMU ? PARTY_QEMU : PARTY_LIBVIRT, + msg); + break; + } + } + + return 0; +} + + +static int +process_data(struct message *msg, bool *interactive) +{ + output(msg->source == PARTY_QEMU ? LABEL_QEMU : LABEL_LIBVIRT, msg); + + if (!*interactive) { + send_message(msg->source == PARTY_QEMU ? PARTY_LIBVIRT : PARTY_QEMU, + msg); + } else { + return process_commands(msg, interactive); + } + + return 0; +} + + +static int +proxy(bool *interactive, int libvirt, int qemu) +{ + fd_set fds; + int maxfd; + struct message msg; + int ret = -1; + ssize_t count; + + print("New monitor session"); + if (*interactive) + print("Entering interactive mode"); + + if (init_message(&msg, qemu, libvirt) < 0) { + perror("Cannot initialize msghdr"); + goto cleanup; + } + + maxfd = libvirt > qemu ? libvirt : qemu; + while (1) { + FD_ZERO(&fds); + FD_SET(STDIN_FILENO, &fds); + FD_SET(libvirt, &fds); + FD_SET(qemu, &fds); + + if (select(maxfd + 1, &fds, NULL, NULL, NULL) == -1) { + perror("select() failed"); + goto cleanup; + } + + if (FD_ISSET(qemu, &fds)) { + if ((count = receive_message(PARTY_QEMU, &msg)) <= 0) { + if (count < 0) + perror("Cannot receive message from qemu"); + else + print("Qemu closed connection"); + goto cleanup; + } + if (process_data(&msg, interactive) < 0) + goto cleanup; + } + + if (FD_ISSET(libvirt, &fds)) { + if ((count = receive_message(PARTY_LIBVIRT, &msg) <= 0)) { + if (count < 0) { + perror("Cannot receive message from libvirt"); + } else { + print("Libvirt closed connection"); + ret = 0; + } + goto cleanup; + } + if (process_data(&msg, interactive) < 0) + goto cleanup; + } + + if (!*interactive && FD_ISSET(STDIN_FILENO, &fds)) { + print("Entering interactive mode"); + *interactive = true; + } + + if (*interactive && process_commands(&msg, interactive) < 0) + goto cleanup; + } + +cleanup: + print("Session closed"); + + close(libvirt); + close(qemu); + return ret; +} + + +int +main(int argc, char **argv) +{ + int sock = -1; + int libvirt = -1; + bool interactive = false; + + if (argc < 2) + usage(2); + + if (strcmp(argv[1], "-i") == 0) { + interactive = true; + if (argc < 3) + usage(2); + monitor = argv[2]; + } else { + monitor = argv[1]; + } + + signal(SIGHUP, signal_handler); + signal(SIGINT, signal_handler); + signal(SIGQUIT, signal_handler); + signal(SIGABRT, signal_handler); + signal(SIGKILL, signal_handler); + signal(SIGPIPE, SIG_IGN); + signal(SIGTERM, signal_handler); + + if (move_socket() < 0) + goto error; + + buffer = malloc(SIZE); + control = malloc(SIZE); + commands = malloc(SIZE); + if (!buffer || !control || !commands) + goto error; + + if ((sock = listen_libvirt()) == -1) + goto error; + + while ((libvirt = accept_libvirt(sock)) != -1) { + int qemu; + + if ((qemu = open_qemu()) != -1) { + if (proxy(&interactive, libvirt, qemu) < 0) + break; + } else { + close(libvirt); + } + } + +error: + restore_socket(); + close(sock); + return 1; +} -- 1.7.6