On Mon, Nov 12, 2012 at 03:02:24PM +0800, Gao feng wrote:

this patch addes fuse support for libvirt lxc. we can use fuse filesystem to generate sysinfo dynamically, So we can isolate /proc/meminfo,cpuinfo and so on through fuse filesystem.

we mount fuse filesystem for every container. the mount name is libvirt,mount point is localstatedir/run/libvirt/lxc/containername.

Signed-off-by: Gao feng <gaofeng@cn.fujitsu.com> --- po/POTFILES.in | 1 + src/Makefile.am | 9 ++- src/lxc/lxc_cgroup.c | 1 + src/lxc/lxc_container.h | 3 + src/lxc/lxc_controller.c | 20 +++++ src/lxc/lxc_fuse.c | 202 ++++++++++++++++++++++++++++++++++++++++++++++ src/lxc/lxc_fuse.h | 50 +++++++++++ src/lxc/lxc_process.c | 1 + 8 files changed, 285 insertions(+), 2 deletions(-) create mode 100644 src/lxc/lxc_fuse.c create mode 100644 src/lxc/lxc_fuse.h

diff --git a/po/POTFILES.in b/po/POTFILES.in index 9768528..0199d6f 100644 --- a/po/POTFILES.in +++ b/po/POTFILES.in @@ -50,6 +50,7 @@ src/locking/lock_driver_sanlock.c src/locking/lock_manager.c src/locking/sanlock_helper.c src/lxc/lxc_cgroup.c +src/lxc/lxc_fuse.c src/lxc/lxc_container.c src/lxc/lxc_conf.c src/lxc/lxc_controller.c diff --git a/src/Makefile.am b/src/Makefile.am index 1f32263..4b8e6c0 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -413,6 +413,7 @@ LXC_DRIVER_SOURCES = \ lxc/lxc_domain.c lxc/lxc_domain.h \ lxc/lxc_monitor.c lxc/lxc_monitor.h \ lxc/lxc_process.c lxc/lxc_process.h \ + lxc/lxc_fuse.c lxc/lxc_fuse.h \ lxc/lxc_driver.c lxc/lxc_driver.h

LXC_CONTROLLER_SOURCES = \ @@ -421,6 +422,7 @@ LXC_CONTROLLER_SOURCES = \ lxc/lxc_conf.c lxc/lxc_conf.h \ lxc/lxc_container.c lxc/lxc_container.h \ lxc/lxc_cgroup.c lxc/lxc_cgroup.h \ + lxc/lxc_fuse.c lxc/lxc_fuse.h \ lxc/lxc_controller.c

SECURITY_DRIVER_APPARMOR_HELPER_SOURCES = \ @@ -911,8 +913,9 @@ endif

libvirt_driver_lxc_impl_la_CFLAGS = \ $(LIBNL_CFLAGS) \ + $(FUSE_CFLAGS) \ -I$(top_srcdir)/src/conf $(AM_CFLAGS) -libvirt_driver_lxc_impl_la_LIBADD = $(CAPNG_LIBS) $(LIBNL_LIBS) +libvirt_driver_lxc_impl_la_LIBADD = $(CAPNG_LIBS) $(LIBNL_LIBS) $(FUSE_LIBS) if HAVE_LIBBLKID libvirt_driver_lxc_impl_la_CFLAGS += $(BLKID_CFLAGS) libvirt_driver_lxc_impl_la_LIBADD += $(BLKID_LIBS) @@ -1678,6 +1681,7 @@ libvirt_lxc_SOURCES = \ libvirt_lxc_LDFLAGS = $(WARN_CFLAGS) $(AM_LDFLAGS) libvirt_lxc_LDADD = \ $(NUMACTL_LIBS) \ + $(FUSE_LIBS) \ libvirt-net-rpc-server.la \ libvirt-net-rpc.la \ libvirt_security_manager.la \ @@ -1696,7 +1700,8 @@ endif libvirt_lxc_CFLAGS = \ -I$(top_srcdir)/src/conf \ $(AM_CFLAGS) \ - $(LIBNL_CFLAGS) + $(LIBNL_CFLAGS) \ + $(FUSE_CFLAGS) if HAVE_LIBBLKID libvirt_lxc_CFLAGS += $(BLKID_CFLAGS) libvirt_lxc_LDADD += $(BLKID_LIBS) diff --git a/src/lxc/lxc_cgroup.c b/src/lxc/lxc_cgroup.c index bdfaa54..9a5ba1a 100644 --- a/src/lxc/lxc_cgroup.c +++ b/src/lxc/lxc_cgroup.c @@ -163,6 +163,7 @@ static int virLXCCgroupSetupDeviceACL(virDomainDefPtr def, {'c', LXC_DEV_MAJ_MEMORY, LXC_DEV_MIN_URANDOM}, {'c', LXC_DEV_MAJ_TTY, LXC_DEV_MIN_TTY}, {'c', LXC_DEV_MAJ_TTY, LXC_DEV_MIN_PTMX}, + {'c', LXC_DEV_MAJ_FUSE, LXC_DEV_MIN_FUSE}, {0, 0, 0}};

rc = virCgroupDenyAllDevices(cgroup); diff --git a/src/lxc/lxc_container.h b/src/lxc/lxc_container.h index b1adacb..c8c70e0 100644 --- a/src/lxc/lxc_container.h +++ b/src/lxc/lxc_container.h @@ -46,6 +46,9 @@ enum {

# define LXC_DEV_MAJ_PTY 136

+# define LXC_DEV_MAJ_FUSE 10 +# define LXC_DEV_MIN_FUSE 229 + int lxcContainerSendContinue(int control); int lxcContainerWaitForContinue(int control);

diff --git a/src/lxc/lxc_controller.c b/src/lxc/lxc_controller.c index a41c903..b993fe5 100644 --- a/src/lxc/lxc_controller.c +++ b/src/lxc/lxc_controller.c @@ -60,6 +60,7 @@ #include "lxc_container.h" #include "lxc_cgroup.h" #include "lxc_protocol.h" +#include "lxc_fuse.h" #include "virnetdev.h" #include "virnetdevveth.h" #include "memory.h" @@ -127,6 +128,8 @@ struct _virLXCController { virNetServerProgramPtr prog; bool inShutdown; int timerShutdown; + + virLXCFusePtr fuse; };

#include "lxc_controller_dispatch.h" @@ -236,6 +239,13 @@ static void virLXCControllerConsoleClose(virLXCControllerConsolePtr console) }

+static void +virLXCControllerFreeFuse(virLXCControllerPtr ctrl) +{ + return lxcFreeFuse(&ctrl->fuse); +} + + static void virLXCControllerFree(virLXCControllerPtr ctrl) { size_t i; @@ -266,6 +276,7 @@ static void virLXCControllerFree(virLXCControllerPtr ctrl) virEventRemoveTimeout(ctrl->timerShutdown);

virObjectUnref(ctrl->server); + virLXCControllerFreeFuse(ctrl);

VIR_FREE(ctrl); } @@ -1247,6 +1258,12 @@ cleanup:

static int +virLXCControllerSetupFuse(virLXCControllerPtr ctrl) +{ + return lxcSetupFuse(&ctrl->fuse, ctrl->def); +} + +static int virLXCControllerSetupConsoles(virLXCControllerPtr ctrl, char **containerTTYPaths) { @@ -1388,6 +1405,9 @@ virLXCControllerRun(virLXCControllerPtr ctrl) if (virLXCControllerSetupDevPTS(ctrl) < 0) goto cleanup;

+ if (virLXCControllerSetupFuse(ctrl) < 0) + goto cleanup; + if (virLXCControllerSetupConsoles(ctrl, containerTTYPaths) < 0) goto cleanup;

diff --git a/src/lxc/lxc_fuse.c b/src/lxc/lxc_fuse.c new file mode 100644 index 0000000..6d621ee --- /dev/null +++ b/src/lxc/lxc_fuse.c @@ -0,0 +1,202 @@ +/* + * Copyright (C) 2012 Fujitsu Limited. + * + * lxc_fuse.c: fuse filesystem support for libvirt lxc + * + * Authors: + * Gao feng <gaofeng at cn.fujitsu.com> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library. If not, see + * <http://www.gnu.org/licenses/>. + */ + +#include <config.h> +#include <fcntl.h> +#include <stdio.h> +#include <errno.h> +#include <string.h> +#include <sys/mount.h> +#include <mntent.h> + +#include "lxc_fuse.h" +#include "virterror_internal.h" +#include "logging.h" + +#define VIR_FROM_THIS VIR_FROM_LXC + +#if HAVE_FUSE + +static int lxcProcGetattr(const char *path, struct stat *stbuf) +{ + int res = 0; + + memset(stbuf, 0, sizeof(struct stat)); + + if (STREQ(path, "/")) { + stbuf->st_mode = S_IFDIR | 0755; + stbuf->st_nlink = 2; + } else { + res = -ENOENT; + } + + return res; +} + +static int lxcProcReaddir(const char *path, void *buf, + fuse_fill_dir_t filler, + off_t offset ATTRIBUTE_UNUSED, + struct fuse_file_info *fi ATTRIBUTE_UNUSED) +{ + if (!STREQ(path, "/")) + return -ENOENT; + + filler(buf, ".", NULL, 0); + filler(buf, "..", NULL, 0); + + return 0; +} + +static int lxcProcOpen(const char *path ATTRIBUTE_UNUSED, + struct fuse_file_info *fi ATTRIBUTE_UNUSED) +{ + return -ENOENT; +} + +static int lxcProcRead(const char *path ATTRIBUTE_UNUSED, + char *buf ATTRIBUTE_UNUSED, + size_t size ATTRIBUTE_UNUSED, + off_t offset ATTRIBUTE_UNUSED, + struct fuse_file_info *fi ATTRIBUTE_UNUSED) +{ + return -ENOENT; +} + +static struct fuse_operations lxcProcOper = { + .getattr = lxcProcGetattr, + .readdir = lxcProcReaddir, + .open = lxcProcOpen, + .read = lxcProcRead, +}; + +static void lxcFuseDestroy(virLXCFusePtr fuse) +{ + virMutexLock(&fuse->lock); + fuse_unmount(fuse->mountpoint, fuse->ch); + fuse_destroy(fuse->fuse); + fuse->fuse = NULL; + virMutexUnlock(&fuse->lock); +} + +static void lxcFuseRun(void *opaque) +{ + virLXCFusePtr fuse = opaque; + + if (fuse_loop(fuse->fuse) < 0) + virReportError(VIR_ERR_INTERNAL_ERROR, + _("fuse_loop failed")); + + lxcFuseDestroy(fuse); +} + +int lxcSetupFuse(virLXCFusePtr *f, virDomainDefPtr def) +{ + int ret = -1; + struct fuse_args args = FUSE_ARGS_INIT(0, NULL); + virLXCFusePtr fuse = NULL; + + if (VIR_ALLOC(fuse) < 0) + goto cleanup; + + fuse->def = def; + + if (virMutexInit(&fuse->lock) < 0) + goto cleanup2; + + if (virAsprintf(&fuse->mountpoint, "%s/%s/", LXC_STATE_DIR, + def->name) < 0) { + virReportOOMError(); + goto cleanup1; + } + + if (virFileMakePath(fuse->mountpoint) < 0) { + virReportSystemError(errno, _("Cannot create %s"), + fuse->mountpoint); + goto cleanup1; + } + + /* process name is libvirt_lxc */ + if (fuse_opt_add_arg(&args, "libvirt_lxc") == -1 || + fuse_opt_add_arg(&args, "-odirect_io") == -1 || + fuse_opt_add_arg(&args, "-ofsname=libvirt") == -1) + goto cleanup1; + + fuse->ch = fuse_mount(fuse->mountpoint, &args); + if (fuse->ch == NULL) + goto cleanup1; + + fuse->fuse = fuse_new(fuse->ch, &args, &lxcProcOper, + sizeof(lxcProcOper), fuse->def); + if (fuse->fuse == NULL) { + fuse_unmount(fuse->mountpoint, fuse->ch); + goto cleanup1; + } + + if (virThreadCreate(&fuse->thread, true, lxcFuseRun, + (void *)fuse) < 0) { + lxcFuseDestroy(fuse); + goto cleanup1; + } + + ret = 0; +cleanup: + fuse_opt_free_args(&args); + *f = fuse; + return ret; +cleanup1: + VIR_FREE(fuse->mountpoint); + virMutexDestroy(&fuse->lock); +cleanup2: + VIR_FREE(fuse); + goto cleanup; +} + +void lxcFreeFuse(virLXCFusePtr *f) +{ + virLXCFusePtr fuse = *f; + /* lxcFuseRun thread create success */ + if (fuse) { + /* exit fuse_loop, lxcFuseRun thread may try to destroy + * fuse->fuse at the same time,so add a lock here. */ + virMutexLock(&fuse->lock); + if (fuse->fuse) + fuse_exit(fuse->fuse); + virMutexUnlock(&fuse->lock); + + virThreadJoin(&fuse->thread); + + VIR_FREE(fuse->mountpoint); + VIR_FREE(*f); + } +} +#else +int lxcSetupFuse(virLXCFusePtr *f ATTRIBUTE_UNUSED, + virDomainDefPtr def ATTRIBUTE_UNUSED) +{ + return 0; +} + +void lxcFreeFuse(virLXCFusePtr *f ATTRIBUTE_UNUSED) +{ +} +#endif diff --git a/src/lxc/lxc_fuse.h b/src/lxc/lxc_fuse.h new file mode 100644 index 0000000..f2d2cbb --- /dev/null +++ b/src/lxc/lxc_fuse.h @@ -0,0 +1,50 @@ +/* + * Copyright (C) 2012 Fujitsu Limited. + * + * lxc_fuse.c: fuse filesystem support for libvirt lxc + * + * Authors: + * Gao feng <gaofeng at cn.fujitsu.com> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library. If not, see + * <http://www.gnu.org/licenses/>. + */ + +#ifndef LXC_FUSE_H +#define LXC_FUSE_H + +#define FUSE_USE_VERSION 26 + +#include <config.h> +#if HAVE_FUSE +#include <fuse.h> +#endif + +#include "lxc_conf.h" +#include "util.h" +#include "memory.h" + +struct virLXCFuse { + virDomainDefPtr def; + virThread thread; + char *mountpoint; + struct fuse *fuse; + struct fuse_chan *ch; + virMutex lock; +}; +typedef struct virLXCFuse *virLXCFusePtr; + +extern int lxcSetupFuse(virLXCFusePtr *f, virDomainDefPtr def); +extern void lxcFreeFuse(virLXCFusePtr *f); +#endif diff --git a/src/lxc/lxc_process.c b/src/lxc/lxc_process.c index 079bc3a..a9a0984 100644 --- a/src/lxc/lxc_process.c +++ b/src/lxc/lxc_process.c @@ -28,6 +28,7 @@ #include "lxc_process.h" #include "lxc_domain.h" #include "lxc_container.h" +#include "lxc_fuse.h" #include "datatypes.h" #include "virfile.h" #include "virpidfile.h" -- 1.7.7.6

Yes, this seems better than having the busy wait. ACK. Rich. -- Richard Jones, Virtualization Group, Red Hat http://people.redhat.com/~rjones libguestfs lets you edit virtual machines. Supports shell scripting, bindings from many languages. http://libguestfs.org

this patch addes fuse support for libvirt lxc. we can use fuse filesystem to generate sysinfo dynamically, So we can isolate /proc/meminfo,cpuinfo and so on through fuse filesystem. we mount fuse filesystem for every container. the mount name is libvirt,mount point is localstatedir/run/libvirt/lxc/containername. Signed-off-by: Gao feng <gaofeng@cn.fujitsu.com> --- po/POTFILES.in | 1 + src/Makefile.am | 9 ++- src/lxc/lxc_cgroup.c | 1 + src/lxc/lxc_container.h | 3 + src/lxc/lxc_controller.c | 20 +++++ src/lxc/lxc_fuse.c | 202 ++++++++++++++++++++++++++++++++++++++++++++++ src/lxc/lxc_fuse.h | 50 +++++++++++ src/lxc/lxc_process.c | 1 + 8 files changed, 285 insertions(+), 2 deletions(-) create mode 100644 src/lxc/lxc_fuse.c create mode 100644 src/lxc/lxc_fuse.h diff --git a/po/POTFILES.in b/po/POTFILES.in index 9768528..0199d6f 100644 --- a/po/POTFILES.in +++ b/po/POTFILES.in @@ -50,6 +50,7 @@ src/locking/lock_driver_sanlock.c src/locking/lock_manager.c src/locking/sanlock_helper.c src/lxc/lxc_cgroup.c +src/lxc/lxc_fuse.c src/lxc/lxc_container.c src/lxc/lxc_conf.c src/lxc/lxc_controller.c diff --git a/src/Makefile.am b/src/Makefile.am index 1f32263..4b8e6c0 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -413,6 +413,7 @@ LXC_DRIVER_SOURCES = \ lxc/lxc_domain.c lxc/lxc_domain.h \ lxc/lxc_monitor.c lxc/lxc_monitor.h \ lxc/lxc_process.c lxc/lxc_process.h \ + lxc/lxc_fuse.c lxc/lxc_fuse.h \ lxc/lxc_driver.c lxc/lxc_driver.h LXC_CONTROLLER_SOURCES = \ @@ -421,6 +422,7 @@ LXC_CONTROLLER_SOURCES = \ lxc/lxc_conf.c lxc/lxc_conf.h \ lxc/lxc_container.c lxc/lxc_container.h \ lxc/lxc_cgroup.c lxc/lxc_cgroup.h \ + lxc/lxc_fuse.c lxc/lxc_fuse.h \ lxc/lxc_controller.c SECURITY_DRIVER_APPARMOR_HELPER_SOURCES = \ @@ -911,8 +913,9 @@ endif libvirt_driver_lxc_impl_la_CFLAGS = \ $(LIBNL_CFLAGS) \ + $(FUSE_CFLAGS) \ -I$(top_srcdir)/src/conf $(AM_CFLAGS) -libvirt_driver_lxc_impl_la_LIBADD = $(CAPNG_LIBS) $(LIBNL_LIBS) +libvirt_driver_lxc_impl_la_LIBADD = $(CAPNG_LIBS) $(LIBNL_LIBS) $(FUSE_LIBS) if HAVE_LIBBLKID libvirt_driver_lxc_impl_la_CFLAGS += $(BLKID_CFLAGS) libvirt_driver_lxc_impl_la_LIBADD += $(BLKID_LIBS) @@ -1678,6 +1681,7 @@ libvirt_lxc_SOURCES = \ libvirt_lxc_LDFLAGS = $(WARN_CFLAGS) $(AM_LDFLAGS) libvirt_lxc_LDADD = \ $(NUMACTL_LIBS) \ + $(FUSE_LIBS) \ libvirt-net-rpc-server.la \ libvirt-net-rpc.la \ libvirt_security_manager.la \ @@ -1696,7 +1700,8 @@ endif libvirt_lxc_CFLAGS = \ -I$(top_srcdir)/src/conf \ $(AM_CFLAGS) \ - $(LIBNL_CFLAGS) + $(LIBNL_CFLAGS) \ + $(FUSE_CFLAGS) if HAVE_LIBBLKID libvirt_lxc_CFLAGS += $(BLKID_CFLAGS) libvirt_lxc_LDADD += $(BLKID_LIBS) diff --git a/src/lxc/lxc_cgroup.c b/src/lxc/lxc_cgroup.c index bdfaa54..9a5ba1a 100644 --- a/src/lxc/lxc_cgroup.c +++ b/src/lxc/lxc_cgroup.c @@ -163,6 +163,7 @@ static int virLXCCgroupSetupDeviceACL(virDomainDefPtr def, {'c', LXC_DEV_MAJ_MEMORY, LXC_DEV_MIN_URANDOM}, {'c', LXC_DEV_MAJ_TTY, LXC_DEV_MIN_TTY}, {'c', LXC_DEV_MAJ_TTY, LXC_DEV_MIN_PTMX}, + {'c', LXC_DEV_MAJ_FUSE, LXC_DEV_MIN_FUSE}, {0, 0, 0}}; rc = virCgroupDenyAllDevices(cgroup); diff --git a/src/lxc/lxc_container.h b/src/lxc/lxc_container.h index b1adacb..c8c70e0 100644 --- a/src/lxc/lxc_container.h +++ b/src/lxc/lxc_container.h @@ -46,6 +46,9 @@ enum { # define LXC_DEV_MAJ_PTY 136 +# define LXC_DEV_MAJ_FUSE 10 +# define LXC_DEV_MIN_FUSE 229 + int lxcContainerSendContinue(int control); int lxcContainerWaitForContinue(int control); diff --git a/src/lxc/lxc_controller.c b/src/lxc/lxc_controller.c index a41c903..b993fe5 100644 --- a/src/lxc/lxc_controller.c +++ b/src/lxc/lxc_controller.c @@ -60,6 +60,7 @@ #include "lxc_container.h" #include "lxc_cgroup.h" #include "lxc_protocol.h" +#include "lxc_fuse.h" #include "virnetdev.h" #include "virnetdevveth.h" #include "memory.h" @@ -127,6 +128,8 @@ struct _virLXCController { virNetServerProgramPtr prog; bool inShutdown; int timerShutdown; + + virLXCFusePtr fuse; }; #include "lxc_controller_dispatch.h" @@ -236,6 +239,13 @@ static void virLXCControllerConsoleClose(virLXCControllerConsolePtr console) } +static void +virLXCControllerFreeFuse(virLXCControllerPtr ctrl) +{ + return lxcFreeFuse(&ctrl->fuse); +} + + static void virLXCControllerFree(virLXCControllerPtr ctrl) { size_t i; @@ -266,6 +276,7 @@ static void virLXCControllerFree(virLXCControllerPtr ctrl) virEventRemoveTimeout(ctrl->timerShutdown); virObjectUnref(ctrl->server); + virLXCControllerFreeFuse(ctrl); VIR_FREE(ctrl); } @@ -1247,6 +1258,12 @@ cleanup: static int +virLXCControllerSetupFuse(virLXCControllerPtr ctrl) +{ + return lxcSetupFuse(&ctrl->fuse, ctrl->def); +} + +static int virLXCControllerSetupConsoles(virLXCControllerPtr ctrl, char **containerTTYPaths) { @@ -1388,6 +1405,9 @@ virLXCControllerRun(virLXCControllerPtr ctrl) if (virLXCControllerSetupDevPTS(ctrl) < 0) goto cleanup; + if (virLXCControllerSetupFuse(ctrl) < 0) + goto cleanup; + if (virLXCControllerSetupConsoles(ctrl, containerTTYPaths) < 0) goto cleanup; diff --git a/src/lxc/lxc_fuse.c b/src/lxc/lxc_fuse.c new file mode 100644 index 0000000..8ed5ea1 --- /dev/null +++ b/src/lxc/lxc_fuse.c @@ -0,0 +1,202 @@ +/* + * Copyright (C) 2012 Fujitsu Limited. + * + * lxc_fuse.c: fuse filesystem support for libvirt lxc + * + * Authors: + * Gao feng <gaofeng at cn.fujitsu.com> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library. If not, see + * <http://www.gnu.org/licenses/>. + */ + +#include <config.h> +#include <fcntl.h> +#include <stdio.h> +#include <errno.h> +#include <string.h> +#include <sys/mount.h> +#include <mntent.h> + +#include "lxc_fuse.h" +#include "virterror_internal.h" +#include "logging.h" + +#define VIR_FROM_THIS VIR_FROM_LXC + +#if HAVE_FUSE + +static int lxcProcGetattr(const char *path, struct stat *stbuf) +{ + int res = 0; + + memset(stbuf, 0, sizeof(struct stat)); + + if (STREQ(path, "/")) { + stbuf->st_mode = S_IFDIR | 0755; + stbuf->st_nlink = 2; + } else { + res = -ENOENT; + } + + return res; +} + +static int lxcProcReaddir(const char *path, void *buf, + fuse_fill_dir_t filler, + off_t offset ATTRIBUTE_UNUSED, + struct fuse_file_info *fi ATTRIBUTE_UNUSED) +{ + if (!STREQ(path, "/")) + return -ENOENT; + + filler(buf, ".", NULL, 0); + filler(buf, "..", NULL, 0); + + return 0; +} + +static int lxcProcOpen(const char *path ATTRIBUTE_UNUSED, + struct fuse_file_info *fi ATTRIBUTE_UNUSED) +{ + return -ENOENT; +} + +static int lxcProcRead(const char *path ATTRIBUTE_UNUSED, + char *buf ATTRIBUTE_UNUSED, + size_t size ATTRIBUTE_UNUSED, + off_t offset ATTRIBUTE_UNUSED, + struct fuse_file_info *fi ATTRIBUTE_UNUSED) +{ + return -ENOENT; +} + +static struct fuse_operations lxcProcOper = { + .getattr = lxcProcGetattr, + .readdir = lxcProcReaddir, + .open = lxcProcOpen, + .read = lxcProcRead, +}; + +static void lxcFuseDestroy(virLXCFusePtr fuse) +{ + virMutexLock(&fuse->lock); + fuse_unmount(fuse->mountpoint, fuse->ch); + fuse_destroy(fuse->fuse); + fuse->fuse = NULL; + virMutexUnlock(&fuse->lock); +} + +static void lxcFuseRun(void *opaque) +{ + virLXCFusePtr fuse = opaque; + + if (fuse_loop(fuse->fuse) < 0) + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("fuse_loop failed")); + + lxcFuseDestroy(fuse); +} + +int lxcSetupFuse(virLXCFusePtr *f, virDomainDefPtr def) +{ + int ret = -1; + struct fuse_args args = FUSE_ARGS_INIT(0, NULL); + virLXCFusePtr fuse = NULL; + + if (VIR_ALLOC(fuse) < 0) + goto cleanup; + + fuse->def = def; + + if (virMutexInit(&fuse->lock) < 0) + goto cleanup2; + + if (virAsprintf(&fuse->mountpoint, "%s/%s/", LXC_STATE_DIR, + def->name) < 0) { + virReportOOMError(); + goto cleanup1; + } + + if (virFileMakePath(fuse->mountpoint) < 0) { + virReportSystemError(errno, _("Cannot create %s"), + fuse->mountpoint); + goto cleanup1; + } + + /* process name is libvirt_lxc */ + if (fuse_opt_add_arg(&args, "libvirt_lxc") == -1 || + fuse_opt_add_arg(&args, "-odirect_io") == -1 || + fuse_opt_add_arg(&args, "-ofsname=libvirt") == -1) + goto cleanup1; + + fuse->ch = fuse_mount(fuse->mountpoint, &args); + if (fuse->ch == NULL) + goto cleanup1; + + fuse->fuse = fuse_new(fuse->ch, &args, &lxcProcOper, + sizeof(lxcProcOper), fuse->def); + if (fuse->fuse == NULL) { + fuse_unmount(fuse->mountpoint, fuse->ch); + goto cleanup1; + } + + if (virThreadCreate(&fuse->thread, true, lxcFuseRun, + (void *)fuse) < 0) { + lxcFuseDestroy(fuse); + goto cleanup1; + } + + ret = 0; +cleanup: + fuse_opt_free_args(&args); + *f = fuse; + return ret; +cleanup1: + VIR_FREE(fuse->mountpoint); + virMutexDestroy(&fuse->lock); +cleanup2: + VIR_FREE(fuse); + goto cleanup; +} + +void lxcFreeFuse(virLXCFusePtr *f) +{ + virLXCFusePtr fuse = *f; + /* lxcFuseRun thread create success */ + if (fuse) { + /* exit fuse_loop, lxcFuseRun thread may try to destroy + * fuse->fuse at the same time,so add a lock here. */ + virMutexLock(&fuse->lock); + if (fuse->fuse) + fuse_exit(fuse->fuse); + virMutexUnlock(&fuse->lock); + + virThreadJoin(&fuse->thread); + + VIR_FREE(fuse->mountpoint); + VIR_FREE(*f); + } +} +#else +int lxcSetupFuse(virLXCFusePtr *f ATTRIBUTE_UNUSED, + virDomainDefPtr def ATTRIBUTE_UNUSED) +{ + return 0; +} + +void lxcFreeFuse(virLXCFusePtr *f ATTRIBUTE_UNUSED) +{ +} +#endif diff --git a/src/lxc/lxc_fuse.h b/src/lxc/lxc_fuse.h new file mode 100644 index 0000000..f2d2cbb --- /dev/null +++ b/src/lxc/lxc_fuse.h @@ -0,0 +1,50 @@ +/* + * Copyright (C) 2012 Fujitsu Limited. + * + * lxc_fuse.c: fuse filesystem support for libvirt lxc + * + * Authors: + * Gao feng <gaofeng at cn.fujitsu.com> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library. If not, see + * <http://www.gnu.org/licenses/>. + */ + +#ifndef LXC_FUSE_H +#define LXC_FUSE_H + +#define FUSE_USE_VERSION 26 + +#include <config.h> +#if HAVE_FUSE +#include <fuse.h> +#endif + +#include "lxc_conf.h" +#include "util.h" +#include "memory.h" + +struct virLXCFuse { + virDomainDefPtr def; + virThread thread; + char *mountpoint; + struct fuse *fuse; + struct fuse_chan *ch; + virMutex lock; +}; +typedef struct virLXCFuse *virLXCFusePtr; + +extern int lxcSetupFuse(virLXCFusePtr *f, virDomainDefPtr def); +extern void lxcFreeFuse(virLXCFusePtr *f); +#endif diff --git a/src/lxc/lxc_process.c b/src/lxc/lxc_process.c index 079bc3a..a9a0984 100644 --- a/src/lxc/lxc_process.c +++ b/src/lxc/lxc_process.c @@ -28,6 +28,7 @@ #include "lxc_process.h" #include "lxc_domain.h" #include "lxc_container.h" +#include "lxc_fuse.h" #include "datatypes.h" #include "virfile.h" #include "virpidfile.h" -- 1.7.7.6

On Mon, Nov 12, 2012 at 03:02:25PM +0800, Gao feng wrote:

virCgroupGetMemSwapUsage is used to get container's swap usage, with this interface,we can get swap usage in fuse filesystem.

Signed-off-by: Gao feng <gaofeng@cn.fujitsu.com> --- src/libvirt_private.syms | 1 + src/util/cgroup.c | 20 ++++++++++++++++++++ src/util/cgroup.h | 1 + 3 files changed, 22 insertions(+), 0 deletions(-)

diff --git a/src/libvirt_private.syms b/src/libvirt_private.syms index e94b478..2608072 100644 --- a/src/libvirt_private.syms +++ b/src/libvirt_private.syms @@ -98,6 +98,7 @@ virCgroupGetCpuacctUsage; virCgroupGetCpusetCpus; virCgroupGetCpusetMems; virCgroupGetFreezerState; +virCgroupGetMemSwapUsage; virCgroupGetMemSwapHardLimit; virCgroupGetMemoryHardLimit; virCgroupGetMemorySoftLimit; diff --git a/src/util/cgroup.c b/src/util/cgroup.c index 3f7b5f7..4aae70d 100644 --- a/src/util/cgroup.c +++ b/src/util/cgroup.c @@ -1371,6 +1371,26 @@ int virCgroupGetMemSwapHardLimit(virCgroupPtr group, unsigned long long *kb) }

/** + * virCgroupGetMemSwapUsage: + * + * @group: The cgroup to get mem+swap usage for + * @kb: The mem+swap amount in kilobytes + * + * Returns: 0 on success + */ +int virCgroupGetMemSwapUsage(virCgroupPtr group, unsigned long long *kb) +{ + long long unsigned int usage_in_bytes; + int ret; + ret = virCgroupGetValueU64(group, + VIR_CGROUP_CONTROLLER_MEMORY, + "memory.memsw.usage_in_bytes", &usage_in_bytes); + if (ret == 0) + *kb = usage_in_bytes >> 10; + return ret; +} + +/** * virCgroupSetCpusetMems: * * @group: The cgroup to set cpuset.mems for diff --git a/src/util/cgroup.h b/src/util/cgroup.h index 38fa4b7..0c11eb1 100644 --- a/src/util/cgroup.h +++ b/src/util/cgroup.h @@ -92,6 +92,7 @@ int virCgroupSetMemorySoftLimit(virCgroupPtr group, unsigned long long kb); int virCgroupGetMemorySoftLimit(virCgroupPtr group, unsigned long long *kb); int virCgroupSetMemSwapHardLimit(virCgroupPtr group, unsigned long long kb); int virCgroupGetMemSwapHardLimit(virCgroupPtr group, unsigned long long *kb); +int virCgroupGetMemSwapUsage(virCgroupPtr group, unsigned long long *kb);

enum { VIR_CGROUP_DEVICE_READ = 1, -- 1.7.7.6

ACK. Rich. -- Richard Jones, Virtualization Group, Red Hat http://people.redhat.com/~rjones virt-df lists disk usage of guests without needing to install any software inside the virtual machine. Supports Linux and Windows. http://et.redhat.com/~rjones/virt-df/

On Mon, Nov 12, 2012 at 03:02:26PM +0800, Gao feng wrote:

because libvirt_lxc's cgroup mountpoint is what it shown in /proc/self/cgroup.

we can get container's cgroup through virCgroupNew("/", &group), add interface virCgroupGetAppRoot to help container to get it's cgroup.

Signed-off-by: Gao feng <gaofeng@cn.fujitsu.com> --- src/libvirt_private.syms | 1 + src/util/cgroup.c | 15 +++++++++++++++ src/util/cgroup.h | 2 ++ 3 files changed, 18 insertions(+), 0 deletions(-)

diff --git a/src/libvirt_private.syms b/src/libvirt_private.syms index 2608072..dd7ede2 100644 --- a/src/libvirt_private.syms +++ b/src/libvirt_private.syms @@ -85,6 +85,7 @@ virCgroupDenyDeviceMajor; virCgroupDenyDevicePath; virCgroupForDomain; virCgroupForDriver; +virCgroupGetAppRoot; virCgroupForEmulator; virCgroupForVcpu; virCgroupFree; diff --git a/src/util/cgroup.c b/src/util/cgroup.c index 4aae70d..9777ad5 100644 --- a/src/util/cgroup.c +++ b/src/util/cgroup.c @@ -965,6 +965,21 @@ int virCgroupForDriver(const char *name ATTRIBUTE_UNUSED, } #endif

+/** +* virCgroupGetAppRoot: +* +* @group: Pointer to returned virCgroupPtr +* +* Returns 0 on success +*/ +int virCgroupGetAppRoot(virCgroupPtr *group) +{ +#if defined HAVE_MNTENT_H && defined HAVE_GETMNTENT_R + return virCgroupNew("/", group); +#else + return -ENXIO; +#endif +}

/** * virCgroupForDomain: diff --git a/src/util/cgroup.h b/src/util/cgroup.h index 0c11eb1..e26f65f 100644 --- a/src/util/cgroup.h +++ b/src/util/cgroup.h @@ -47,6 +47,8 @@ int virCgroupForDriver(const char *name, int privileged, int create);

+int virCgroupGetAppRoot(virCgroupPtr *group); + int virCgroupForDomain(virCgroupPtr driver, const char *name, virCgroupPtr *group, -- 1.7.7.6

I've no idea if this function will work or not, but the patch doesn't appear to contain any problems so ACK. Rich. -- Richard Jones, Virtualization Group, Red Hat http://people.redhat.com/~rjones virt-df lists disk usage of guests without needing to install any software inside the virtual machine. Supports Linux and Windows. http://et.redhat.com/~rjones/virt-df/

with this patch,container's meminfo will be shown based on containers' mem cgroup. Right now,it's impossible to virtualize all values in meminfo, I collect some values such as MemTotal,MemFree,Cached,Active, Inactive,Active(anon),Inactive(anon),Active(file),Inactive(anon), Active(file),Inactive(file),Unevictable,SwapTotal,SwapFree. if I miss something, please let me know. Signed-off-by: Gao feng <gaofeng@cn.fujitsu.com> --- src/lxc/lxc_cgroup.c | 143 ++++++++++++++++++++++++++++++++++++++++ src/lxc/lxc_cgroup.h | 3 +- src/lxc/lxc_fuse.c | 178 ++++++++++++++++++++++++++++++++++++++++++++++++-- src/lxc/lxc_fuse.h | 14 ++++ 4 files changed, 332 insertions(+), 6 deletions(-) diff --git a/src/lxc/lxc_cgroup.c b/src/lxc/lxc_cgroup.c index 9a5ba1a..4c32368 100644 --- a/src/lxc/lxc_cgroup.c +++ b/src/lxc/lxc_cgroup.c @@ -23,6 +23,7 @@ #include "lxc_cgroup.h" #include "lxc_container.h" +#include "virfile.h" #include "virterror_internal.h" #include "logging.h" #include "memory.h" @@ -138,6 +139,148 @@ cleanup: } +static int virLXCCgroupGetMemSwapUsage(virCgroupPtr cgroup, + virLXCMeminfoPtr meminfo) +{ + return virCgroupGetMemSwapUsage(cgroup, &meminfo->swapusage); +} + + +static int virLXCCgroupGetMemSwapTotal(virCgroupPtr cgroup, + virLXCMeminfoPtr meminfo) +{ + return virCgroupGetMemSwapHardLimit(cgroup, &meminfo->swaptotal); +} + + +static int virLXCCgroupGetMemUsage(virCgroupPtr cgroup, + virLXCMeminfoPtr meminfo) +{ + int ret; + unsigned long memUsage; + + ret = virCgroupGetMemoryUsage(cgroup, &memUsage); + meminfo->memusage = (unsigned long long) memUsage; + + return ret; +} + + +static int virLXCCgroupGetMemTotal(virCgroupPtr cgroup, + virLXCMeminfoPtr meminfo) +{ + return virCgroupGetMemoryHardLimit(cgroup, &meminfo->memtotal); +} + + +static int virLXCCgroupGetMemStat(virCgroupPtr cgroup, + virLXCMeminfoPtr meminfo) +{ + int ret = 0; + FILE *statfd = NULL; + char *statFile = NULL; + char *line = NULL; + size_t n; + + ret = virCgroupPathOfController(cgroup, VIR_CGROUP_CONTROLLER_MEMORY, + "memory.stat", &statFile); + if (ret != 0) { + virReportSystemError(-ret, "%s", + _("cannot get the path of MEMORY cgroup controller")); + return ret; + } + + statfd = fopen(statFile, "r"); + if (statfd == NULL) { + ret = -errno; + goto cleanup; + } + + while (getline(&line, &n, statfd) > 0) { + + char *value = strchr(line, ' '); + char *nl = value ? strchr(line, '\n') : NULL; + unsigned long long stat_value; + + if (!value) + continue; + + if (nl) + *nl = '\0'; + + *value = '\0'; + + if (virStrToLong_ull(value + 1, NULL, 10, &stat_value) < 0) { + ret = -EINVAL; + goto cleanup; + } + if (STREQ(line, "cache")) + meminfo->cached = stat_value >> 10; + else if (STREQ(line, "inactive_anon")) + meminfo->inactive_anon = stat_value >> 10; + else if (STREQ(line, "active_anon")) + meminfo->active_anon = stat_value >> 10; + else if (STREQ(line, "inactive_file")) + meminfo->inactive_file = stat_value >> 10; + else if (STREQ(line, "active_file")) + meminfo->active_file = stat_value >> 10; + else if (STREQ(line, "unevictable")) + meminfo->unevictable = stat_value >> 10; + } + ret = 0; + +cleanup: + VIR_FREE(line); + VIR_FREE(statFile); + VIR_FORCE_FCLOSE(statfd); + return ret; +} + + +int virLXCCgroupGetMeminfo(virLXCMeminfoPtr meminfo) +{ + int ret; + virCgroupPtr cgroup; + + ret = virCgroupGetAppRoot(&cgroup); + if (ret < 0) { + virReportSystemError(-ret, "%s", + _("Unable to get cgroup for container")); + return ret; + } + + ret = virLXCCgroupGetMemStat(cgroup, meminfo); + if (ret < 0) { + virReportSystemError(-ret, "%s", + _("Unable to get memory cgroup stat info")); + goto cleanup; + } + + ret = virLXCCgroupGetMemTotal(cgroup, meminfo); + if (ret < 0) { + virReportSystemError(-ret, "%s", + _("Unable to get memory cgroup total")); + goto cleanup; + } + + ret = virLXCCgroupGetMemUsage(cgroup, meminfo); + if (ret < 0) { + virReportSystemError(-ret, "%s", + _("Unable to get memory cgroup stat usage")); + goto cleanup; + } + + virLXCCgroupGetMemSwapTotal(cgroup, meminfo); + virLXCCgroupGetMemSwapUsage(cgroup, meminfo); + + ret = 0; +cleanup: + virCgroupFree(&cgroup); + return ret; +} + + + typedef struct _virLXCCgroupDevicePolicy virLXCCgroupDevicePolicy; typedef virLXCCgroupDevicePolicy *virLXCCgroupDevicePolicyPtr; diff --git a/src/lxc/lxc_cgroup.h b/src/lxc/lxc_cgroup.h index 6fc68df..6961943 100644 --- a/src/lxc/lxc_cgroup.h +++ b/src/lxc/lxc_cgroup.h @@ -23,7 +23,8 @@ # define __VIR_LXC_CGROUP_H__ # include "domain_conf.h" +# include "lxc_fuse.h" int virLXCCgroupSetup(virDomainDefPtr def); - +int virLXCCgroupGetMeminfo(virLXCMeminfoPtr meminfo); #endif /* __VIR_LXC_CGROUP_H__ */ diff --git a/src/lxc/lxc_fuse.c b/src/lxc/lxc_fuse.c index 6d621ee..5a514d6 100644 --- a/src/lxc/lxc_fuse.c +++ b/src/lxc/lxc_fuse.c @@ -30,26 +30,54 @@ #include <mntent.h> #include "lxc_fuse.h" +#include "lxc_cgroup.h" #include "virterror_internal.h" #include "logging.h" +#include "virfile.h" +#include "buf.h" #define VIR_FROM_THIS VIR_FROM_LXC #if HAVE_FUSE +static const char *fuse_meminfo_path = "/meminfo"; + static int lxcProcGetattr(const char *path, struct stat *stbuf) { - int res = 0; + int res; + char *mempath = NULL; + struct stat sb; memset(stbuf, 0, sizeof(struct stat)); + if (virAsprintf(&mempath, "/proc/%s", path) < 0) { + virReportOOMError(); + return -errno; + } + + res = 0; if (STREQ(path, "/")) { stbuf->st_mode = S_IFDIR | 0755; stbuf->st_nlink = 2; - } else { + } else if (STREQ(path, fuse_meminfo_path)) { + if (stat(mempath, &sb) < 0) { + res = -errno; + goto cleanup; + } + + stbuf->st_mode = sb.st_mode; + stbuf->st_nlink = 1; + stbuf->st_blksize = sb.st_blksize; + stbuf->st_blocks = sb.st_blocks; + stbuf->st_size = sb.st_size; + stbuf->st_atime = sb.st_atime; + stbuf->st_ctime = sb.st_ctime; + stbuf->st_mtime = sb.st_mtime; + } else res = -ENOENT; - } +cleanup: + VIR_FREE(mempath); return res; } @@ -63,6 +91,7 @@ static int lxcProcReaddir(const char *path, void *buf, filler(buf, ".", NULL, 0); filler(buf, "..", NULL, 0); + filler(buf, fuse_meminfo_path + 1, NULL, 0); return 0; } @@ -70,7 +99,127 @@ static int lxcProcReaddir(const char *path, void *buf, static int lxcProcOpen(const char *path ATTRIBUTE_UNUSED, struct fuse_file_info *fi ATTRIBUTE_UNUSED) { - return -ENOENT; + if (!STREQ(path, fuse_meminfo_path)) + return -ENOENT; + + if ((fi->flags & 3) != O_RDONLY) + return -EACCES; + + return 0; +} + +static int lxcProcHostRead(char *path, char *buf, size_t size, off_t offset) +{ + int fd; + int res; + + fd = open(path, O_RDONLY); + if (fd == -1) + return -errno; + + if ((res = pread(fd, buf, size, offset)) < 0) + res = -errno; + + VIR_FORCE_CLOSE(fd); + return res; +} + +static int lxcProcReadMeminfo(char *hostpath, virDomainDefPtr def, + char *buf, size_t size, off_t offset) +{ + int copied = 0; + int res; + FILE *fd = NULL; + char *line = NULL; + size_t n; + struct virLXCMeminfo meminfo; + virBuffer buffer = VIR_BUFFER_INITIALIZER; + virBufferPtr new_meminfo = &buffer; + + if ((res = virLXCCgroupGetMeminfo(&meminfo)) < 0) + return res; + + fd = fopen(hostpath, "r"); + if (fd == NULL) { + virReportSystemError(errno, _("Cannot open %s"), hostpath); + res = -errno; + goto cleanup; + } + + if (fseek(fd, offset, SEEK_SET) < 0) { + virReportSystemError(errno, "%s", _("fseek failed")); + res = -errno; + goto cleanup; + } + + res = -1; + while (copied < size && getline(&line, &n, fd) > 0) { + char *ptr = strchr(line, ':'); + if (ptr) { + *ptr = '\0'; + + if (STREQ(line, "MemTotal") && + (def->mem.hard_limit || def->mem.max_balloon)) + virBufferAsprintf(new_meminfo, "MemTotal: %8llu KB\n", + meminfo.memtotal); + else if (STREQ(line, "MemFree") && + (def->mem.hard_limit || def->mem.max_balloon)) + virBufferAsprintf(new_meminfo, "MemFree: %8llu KB\n", + (meminfo.memtotal - meminfo.memusage)); + else if (STREQ(line, "Buffers")) + virBufferAsprintf(new_meminfo, "Buffers: %8d KB\n", 0); + else if (STREQ(line, "Cached")) + virBufferAsprintf(new_meminfo, "Cached: %8llu KB\n", + meminfo.cached); + else if (STREQ(line, "Active")) + virBufferAsprintf(new_meminfo, "Active: %8llu KB\n", + (meminfo.active_anon + meminfo.active_file)); + else if (STREQ(line, "Inactive")) + virBufferAsprintf(new_meminfo, "Inactive: %8llu KB\n", + (meminfo.inactive_anon + meminfo.inactive_file)); + else if (STREQ(line, "Active(anon)")) + virBufferAsprintf(new_meminfo, "Active(anon): %8llu KB\n", + meminfo.active_anon); + else if (STREQ(line, "Inactive(anon)")) + virBufferAsprintf(new_meminfo, "Inactive(anon): %8llu KB\n", + meminfo.inactive_anon); + else if (STREQ(line, "Active(file)")) + virBufferAsprintf(new_meminfo, "Active(file): %8llu KB\n", + meminfo.active_file); + else if (STREQ(line, "Inactive(file)")) + virBufferAsprintf(new_meminfo, "Inactive(file): %8llu KB\n", + meminfo.inactive_file); + else if (STREQ(line, "Unevictable")) + virBufferAsprintf(new_meminfo, "Unevictable: %8llu KB\n", + meminfo.unevictable); + else if (STREQ(line, "SwapTotal") && def->mem.swap_hard_limit) + virBufferAsprintf(new_meminfo, "SwapTotal: %8llu KB\n", + (meminfo.swaptotal - meminfo.memtotal)); + else if (STREQ(line, "SwapFree") && def->mem.swap_hard_limit) + virBufferAsprintf(new_meminfo, "SwapFree: %8llu KB\n", + (meminfo.swaptotal - meminfo.memtotal - + meminfo.swapusage + meminfo.memusage)); + else { + *ptr = ':'; + virBufferAdd(new_meminfo, line, -1); + } + + if (virBufferError(new_meminfo)) + goto cleanup; + + copied += strlen(line); + if (copied > size) + copied = size; + } + } + res = copied; + memcpy(buf, virBufferCurrentContent(new_meminfo), copied); + +cleanup: + VIR_FREE(line); + virBufferFreeAndReset(new_meminfo); + VIR_FORCE_FCLOSE(fd); + return res; } static int lxcProcRead(const char *path ATTRIBUTE_UNUSED, @@ -79,7 +228,26 @@ static int lxcProcRead(const char *path ATTRIBUTE_UNUSED, off_t offset ATTRIBUTE_UNUSED, struct fuse_file_info *fi ATTRIBUTE_UNUSED) { - return -ENOENT; + int res = -ENOENT; + char *hostpath = NULL; + struct fuse_context *context = NULL; + virDomainDefPtr def = NULL; + + if (virAsprintf(&hostpath, "/proc/%s", path) < 0) { + virReportOOMError(); + return -errno; + } + + context = fuse_get_context(); + def = (virDomainDefPtr)context->private_data; + + if (STREQ(path, fuse_meminfo_path)) { + if ((res = lxcProcReadMeminfo(hostpath, def, buf, size, offset)) < 0) + res = lxcProcHostRead(hostpath, buf, size, offset); + } + + VIR_FREE(hostpath); + return res; } static struct fuse_operations lxcProcOper = { diff --git a/src/lxc/lxc_fuse.h b/src/lxc/lxc_fuse.h index f2d2cbb..93a5373 100644 --- a/src/lxc/lxc_fuse.h +++ b/src/lxc/lxc_fuse.h @@ -35,6 +35,20 @@ #include "util.h" #include "memory.h" +struct virLXCMeminfo { + unsigned long long memtotal; + unsigned long long memusage; + unsigned long long cached; + unsigned long long active_anon; + unsigned long long inactive_anon; + unsigned long long active_file; + unsigned long long inactive_file; + unsigned long long unevictable; + unsigned long long swaptotal; + unsigned long long swapusage; +}; +typedef struct virLXCMeminfo *virLXCMeminfoPtr; + struct virLXCFuse { virDomainDefPtr def; virThread thread; -- 1.7.7.6

于 2012年11月12日 20:05, Richard W.M. Jones 写道:

On Mon, Nov 12, 2012 at 07:52:01PM +0800, Gao feng wrote:

...

with this patch,container's meminfo will be shown based on containers' mem cgroup.

Right now,it's impossible to virtualize all values in meminfo, I collect some values such as MemTotal,MemFree,Cached,Active, Inactive,Active(anon),Inactive(anon),Active(file),Inactive(anon), Active(file),Inactive(file),Unevictable,SwapTotal,SwapFree.

if I miss something, please let me know.

Signed-off-by: Gao feng <gaofeng@cn.fujitsu.com> --- src/lxc/lxc_cgroup.c | 143 ++++++++++++++++++++++++++++++++++++++++ src/lxc/lxc_cgroup.h | 3 +- src/lxc/lxc_fuse.c | 178 ++++++++++++++++++++++++++++++++++++++++++++++++-- src/lxc/lxc_fuse.h | 14 ++++ 4 files changed, 332 insertions(+), 6 deletions(-)

diff --git a/src/lxc/lxc_cgroup.c b/src/lxc/lxc_cgroup.c index 9a5ba1a..4c32368 100644 --- a/src/lxc/lxc_cgroup.c +++ b/src/lxc/lxc_cgroup.c @@ -23,6 +23,7 @@

#include "lxc_cgroup.h" #include "lxc_container.h" +#include "virfile.h" #include "virterror_internal.h" #include "logging.h" #include "memory.h" @@ -138,6 +139,148 @@ cleanup: }

+static int virLXCCgroupGetMemSwapUsage(virCgroupPtr cgroup, + virLXCMeminfoPtr meminfo) +{ + return virCgroupGetMemSwapUsage(cgroup, &meminfo->swapusage); +} + + +static int virLXCCgroupGetMemSwapTotal(virCgroupPtr cgroup, + virLXCMeminfoPtr meminfo) +{ + return virCgroupGetMemSwapHardLimit(cgroup, &meminfo->swaptotal); +} + + +static int virLXCCgroupGetMemUsage(virCgroupPtr cgroup, + virLXCMeminfoPtr meminfo) +{ + int ret; + unsigned long memUsage; + + ret = virCgroupGetMemoryUsage(cgroup, &memUsage); + meminfo->memusage = (unsigned long long) memUsage; + + return ret; +} + + +static int virLXCCgroupGetMemTotal(virCgroupPtr cgroup, + virLXCMeminfoPtr meminfo) +{ + return virCgroupGetMemoryHardLimit(cgroup, &meminfo->memtotal); +} + + +static int virLXCCgroupGetMemStat(virCgroupPtr cgroup, + virLXCMeminfoPtr meminfo) +{ + int ret = 0; + FILE *statfd = NULL; + char *statFile = NULL; + char *line = NULL; + size_t n; + + ret = virCgroupPathOfController(cgroup, VIR_CGROUP_CONTROLLER_MEMORY, + "memory.stat", &statFile); + if (ret != 0) { + virReportSystemError(-ret, "%s", + _("cannot get the path of MEMORY cgroup controller")); + return ret; + } + + statfd = fopen(statFile, "r"); + if (statfd == NULL) { + ret = -errno; + goto cleanup; + } + + while (getline(&line, &n, statfd) > 0) { + + char *value = strchr(line, ' '); + char *nl = value ? strchr(line, '\n') : NULL; + unsigned long long stat_value; + + if (!value) + continue; + + if (nl) + *nl = '\0'; + + *value = '\0'; + + if (virStrToLong_ull(value + 1, NULL, 10, &stat_value) < 0) { + ret = -EINVAL; + goto cleanup; + } + if (STREQ(line, "cache")) + meminfo->cached = stat_value >> 10; + else if (STREQ(line, "inactive_anon")) + meminfo->inactive_anon = stat_value >> 10; + else if (STREQ(line, "active_anon")) + meminfo->active_anon = stat_value >> 10; + else if (STREQ(line, "inactive_file")) + meminfo->inactive_file = stat_value >> 10; + else if (STREQ(line, "active_file")) + meminfo->active_file = stat_value >> 10; + else if (STREQ(line, "unevictable")) + meminfo->unevictable = stat_value >> 10; + } + ret = 0; + +cleanup: + VIR_FREE(line); + VIR_FREE(statFile); + VIR_FORCE_FCLOSE(statfd); + return ret; +} + + +int virLXCCgroupGetMeminfo(virLXCMeminfoPtr meminfo) +{ + int ret; + virCgroupPtr cgroup; + + ret = virCgroupGetAppRoot(&cgroup); + if (ret < 0) { + virReportSystemError(-ret, "%s", + _("Unable to get cgroup for container")); + return ret; + } + + ret = virLXCCgroupGetMemStat(cgroup, meminfo); + if (ret < 0) { + virReportSystemError(-ret, "%s", + _("Unable to get memory cgroup stat info")); + goto cleanup; + } + + ret = virLXCCgroupGetMemTotal(cgroup, meminfo); + if (ret < 0) { + virReportSystemError(-ret, "%s", + _("Unable to get memory cgroup total")); + goto cleanup; + } + + ret = virLXCCgroupGetMemUsage(cgroup, meminfo); + if (ret < 0) { + virReportSystemError(-ret, "%s", + _("Unable to get memory cgroup stat usage")); + goto cleanup; + } + + virLXCCgroupGetMemSwapTotal(cgroup, meminfo); + virLXCCgroupGetMemSwapUsage(cgroup, meminfo); + + ret = 0; +cleanup: + virCgroupFree(&cgroup); + return ret; +} + + + typedef struct _virLXCCgroupDevicePolicy virLXCCgroupDevicePolicy; typedef virLXCCgroupDevicePolicy *virLXCCgroupDevicePolicyPtr;

diff --git a/src/lxc/lxc_cgroup.h b/src/lxc/lxc_cgroup.h index 6fc68df..6961943 100644 --- a/src/lxc/lxc_cgroup.h +++ b/src/lxc/lxc_cgroup.h @@ -23,7 +23,8 @@ # define __VIR_LXC_CGROUP_H__

# include "domain_conf.h" +# include "lxc_fuse.h"

int virLXCCgroupSetup(virDomainDefPtr def); - +int virLXCCgroupGetMeminfo(virLXCMeminfoPtr meminfo); #endif /* __VIR_LXC_CGROUP_H__ */ diff --git a/src/lxc/lxc_fuse.c b/src/lxc/lxc_fuse.c index 6d621ee..5a514d6 100644 --- a/src/lxc/lxc_fuse.c +++ b/src/lxc/lxc_fuse.c @@ -30,26 +30,54 @@ #include <mntent.h>

#include "lxc_fuse.h" +#include "lxc_cgroup.h" #include "virterror_internal.h" #include "logging.h" +#include "virfile.h" +#include "buf.h"

#define VIR_FROM_THIS VIR_FROM_LXC

#if HAVE_FUSE

+static const char *fuse_meminfo_path = "/meminfo"; + static int lxcProcGetattr(const char *path, struct stat *stbuf) { - int res = 0; + int res; + char *mempath = NULL; + struct stat sb;

memset(stbuf, 0, sizeof(struct stat)); + if (virAsprintf(&mempath, "/proc/%s", path) < 0) { + virReportOOMError(); + return -errno; + } + + res = 0;

if (STREQ(path, "/")) { stbuf->st_mode = S_IFDIR | 0755; stbuf->st_nlink = 2; - } else { + } else if (STREQ(path, fuse_meminfo_path)) { + if (stat(mempath, &sb) < 0) { + res = -errno; + goto cleanup; + } + + stbuf->st_mode = sb.st_mode; + stbuf->st_nlink = 1; + stbuf->st_blksize = sb.st_blksize; + stbuf->st_blocks = sb.st_blocks; + stbuf->st_size = sb.st_size; + stbuf->st_atime = sb.st_atime; + stbuf->st_ctime = sb.st_ctime; + stbuf->st_mtime = sb.st_mtime; + } else res = -ENOENT; - }

+cleanup: + VIR_FREE(mempath); return res; }

@@ -63,6 +91,7 @@ static int lxcProcReaddir(const char *path, void *buf,

filler(buf, ".", NULL, 0); filler(buf, "..", NULL, 0); + filler(buf, fuse_meminfo_path + 1, NULL, 0);

return 0; } @@ -70,7 +99,127 @@ static int lxcProcReaddir(const char *path, void *buf, static int lxcProcOpen(const char *path ATTRIBUTE_UNUSED, struct fuse_file_info *fi ATTRIBUTE_UNUSED) { - return -ENOENT; + if (!STREQ(path, fuse_meminfo_path)) + return -ENOENT; + + if ((fi->flags & 3) != O_RDONLY) + return -EACCES; + + return 0; +} + +static int lxcProcHostRead(char *path, char *buf, size_t size, off_t offset) +{ + int fd; + int res; + + fd = open(path, O_RDONLY); + if (fd == -1) + return -errno; + + if ((res = pread(fd, buf, size, offset)) < 0) + res = -errno; + + VIR_FORCE_CLOSE(fd); + return res; +} + +static int lxcProcReadMeminfo(char *hostpath, virDomainDefPtr def, + char *buf, size_t size, off_t offset) +{ + int copied = 0; + int res; + FILE *fd = NULL; + char *line = NULL; + size_t n; + struct virLXCMeminfo meminfo; + virBuffer buffer = VIR_BUFFER_INITIALIZER; + virBufferPtr new_meminfo = &buffer; + + if ((res = virLXCCgroupGetMeminfo(&meminfo)) < 0) + return res; + + fd = fopen(hostpath, "r"); + if (fd == NULL) { + virReportSystemError(errno, _("Cannot open %s"), hostpath); + res = -errno; + goto cleanup; + } + + if (fseek(fd, offset, SEEK_SET) < 0) { + virReportSystemError(errno, "%s", _("fseek failed")); + res = -errno; + goto cleanup; + } + + res = -1; + while (copied < size && getline(&line, &n, fd) > 0) { + char *ptr = strchr(line, ':'); + if (ptr) { + *ptr = '\0'; + + if (STREQ(line, "MemTotal") && + (def->mem.hard_limit || def->mem.max_balloon)) + virBufferAsprintf(new_meminfo, "MemTotal: %8llu KB\n", + meminfo.memtotal); + else if (STREQ(line, "MemFree") && + (def->mem.hard_limit || def->mem.max_balloon)) + virBufferAsprintf(new_meminfo, "MemFree: %8llu KB\n", + (meminfo.memtotal - meminfo.memusage)); + else if (STREQ(line, "Buffers")) + virBufferAsprintf(new_meminfo, "Buffers: %8d KB\n", 0); + else if (STREQ(line, "Cached")) + virBufferAsprintf(new_meminfo, "Cached: %8llu KB\n", + meminfo.cached); + else if (STREQ(line, "Active")) + virBufferAsprintf(new_meminfo, "Active: %8llu KB\n", + (meminfo.active_anon + meminfo.active_file)); + else if (STREQ(line, "Inactive")) + virBufferAsprintf(new_meminfo, "Inactive: %8llu KB\n", + (meminfo.inactive_anon + meminfo.inactive_file)); + else if (STREQ(line, "Active(anon)")) + virBufferAsprintf(new_meminfo, "Active(anon): %8llu KB\n", + meminfo.active_anon); + else if (STREQ(line, "Inactive(anon)")) + virBufferAsprintf(new_meminfo, "Inactive(anon): %8llu KB\n", + meminfo.inactive_anon); + else if (STREQ(line, "Active(file)")) + virBufferAsprintf(new_meminfo, "Active(file): %8llu KB\n", + meminfo.active_file); + else if (STREQ(line, "Inactive(file)")) + virBufferAsprintf(new_meminfo, "Inactive(file): %8llu KB\n", + meminfo.inactive_file); + else if (STREQ(line, "Unevictable")) + virBufferAsprintf(new_meminfo, "Unevictable: %8llu KB\n", + meminfo.unevictable); + else if (STREQ(line, "SwapTotal") && def->mem.swap_hard_limit) + virBufferAsprintf(new_meminfo, "SwapTotal: %8llu KB\n", + (meminfo.swaptotal - meminfo.memtotal)); + else if (STREQ(line, "SwapFree") && def->mem.swap_hard_limit) + virBufferAsprintf(new_meminfo, "SwapFree: %8llu KB\n", + (meminfo.swaptotal - meminfo.memtotal - + meminfo.swapusage + meminfo.memusage)); + else { + *ptr = ':'; + virBufferAdd(new_meminfo, line, -1); + } + + if (virBufferError(new_meminfo)) + goto cleanup; + + copied += strlen(line); + if (copied > size) + copied = size; + } + } + res = copied; + memcpy(buf, virBufferCurrentContent(new_meminfo), copied); + +cleanup: + VIR_FREE(line); + virBufferFreeAndReset(new_meminfo); + VIR_FORCE_FCLOSE(fd); + return res; }

static int lxcProcRead(const char *path ATTRIBUTE_UNUSED, @@ -79,7 +228,26 @@ static int lxcProcRead(const char *path ATTRIBUTE_UNUSED, off_t offset ATTRIBUTE_UNUSED, struct fuse_file_info *fi ATTRIBUTE_UNUSED) { - return -ENOENT; + int res = -ENOENT; + char *hostpath = NULL; + struct fuse_context *context = NULL; + virDomainDefPtr def = NULL; + + if (virAsprintf(&hostpath, "/proc/%s", path) < 0) { + virReportOOMError(); + return -errno; + } + + context = fuse_get_context(); + def = (virDomainDefPtr)context->private_data; + + if (STREQ(path, fuse_meminfo_path)) { + if ((res = lxcProcReadMeminfo(hostpath, def, buf, size, offset)) < 0) + res = lxcProcHostRead(hostpath, buf, size, offset); + } + + VIR_FREE(hostpath); + return res; }

static struct fuse_operations lxcProcOper = { diff --git a/src/lxc/lxc_fuse.h b/src/lxc/lxc_fuse.h index f2d2cbb..93a5373 100644 --- a/src/lxc/lxc_fuse.h +++ b/src/lxc/lxc_fuse.h @@ -35,6 +35,20 @@ #include "util.h" #include "memory.h"

+struct virLXCMeminfo { + unsigned long long memtotal; + unsigned long long memusage; + unsigned long long cached; + unsigned long long active_anon; + unsigned long long inactive_anon; + unsigned long long active_file; + unsigned long long inactive_file; + unsigned long long unevictable; + unsigned long long swaptotal; + unsigned long long swapusage; +}; +typedef struct virLXCMeminfo *virLXCMeminfoPtr; + struct virLXCFuse { virDomainDefPtr def; virThread thread; -- 1.7.7.6

This seems better from a pure coding point of view. I can't really comment on whether the code actually works since I'm not using LXC over here.

Hi Rich thanks for your help :). with my test,it works well. without this patchset,in container, the free shows: sh-4.2# free -l total used free shared buffers cached Mem: 5947936 4514904 1433032 0 112688 1931716 Low: 5947936 4514904 1433032 High: 0 0 0 -/+ buffers/cache: 2470500 3477436 Swap: 8191996 273436 7918560 with this patchset applied,in container,the free shows: sh-4.2# free -l total used free shared buffers cached Mem: 102400 4820 97580 0 0 4036 Low: 102400 4820 97580 High: 0 0 0 -/+ buffers/cache: 784 101616 Swap: 8191996 263480 7928516 It works just like what we expect. Thanks!