Each on_crash or watchdog-triggered dump writes a full memory dump into auto_dump_path. A guest that keeps crashing and restarting (or crashing and getting destroyed, then respawned by the mgmt app) can fill the disk one dump at a time, with nothing to stop it. Add auto_dump_max_size (qemu.conf), parsed via virConfGetValueBytes() so it takes a plain byte count or a size with a unit suffix (e.g. "10GiB"). After a dump is written, the oldest dumps under auto_dump_path are removed until the total fits the configured quota. The dump that was just written is always kept by identity, not by sort position: mtime is only second-granularity, so two dumps written the same second would otherwise make the eviction order between them arbitrary and could delete the one just written instead of an older one. Defaults to 0, which keeps every dump forever, as before. Only files named the way getAutoDumpPath() names them count toward the quota, and nothing is pruned when the dump itself failed. libvirtd_qemu.aug gains a matching entry; its quoted branch requires a unit suffix, as one matching a bare number would overlap the integer branch. Signed-off-by: Denis V. Lunev <den@openvz.org> --- NEWS.rst | 7 ++ src/qemu/libvirtd_qemu.aug | 4 + src/qemu/qemu.conf.in | 17 ++++ src/qemu/qemu_conf.c | 2 + src/qemu/qemu_conf.h | 1 + src/qemu/qemu_driver.c | 132 +++++++++++++++++++++++++++++ src/qemu/test_libvirtd_qemu.aug.in | 1 + 7 files changed, 164 insertions(+) diff --git a/NEWS.rst b/NEWS.rst index 357f9cda10..c333d6287b 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -17,6 +17,13 @@ v12.8.0 (unreleased) * **New features** + * qemu: Add ``auto_dump_max_size`` to cap the size of auto-triggered dumps + + A guest that keeps crashing fills ``auto_dump_path`` one dump at a time. + The new ``auto_dump_max_size`` setting in qemu.conf bounds the total size + those dumps may occupy, pruning the oldest after each new one. It takes a + byte count or a size such as ``"10GiB"``; 0, the default, keeps them all. + * **Improvements** * qemu: Select the win-dmp format for Windows guest crash dumps diff --git a/src/qemu/libvirtd_qemu.aug b/src/qemu/libvirtd_qemu.aug index 311992e441..dbcbbb04c0 100644 --- a/src/qemu/libvirtd_qemu.aug +++ b/src/qemu/libvirtd_qemu.aug @@ -25,6 +25,9 @@ module Libvirtd_qemu = let unlimited_val = del /\"/ "\"" . store /unlimited/ . del /\"/ "\"" let limits_entry (kw:string) = [ key kw . value_sep . unlimited_val ] | [ key kw . value_sep . int_val ] + let bytes_val = del /\"/ "\"" . store /[0-9]+[a-zA-Z]+/ . del /\"/ "\"" + let bytes_entry (kw:string) = [ key kw . value_sep . bytes_val ] | [ key kw . value_sep . int_val ] + (* Config entry grouped by function - same order as example config *) let default_tls_entry = str_entry "default_tls_x509_cert_dir" @@ -109,6 +112,7 @@ module Libvirtd_qemu = | str_entry "snapshot_image_format" | str_entry "auto_dump_path" | bool_entry "auto_dump_bypass_cache" + | bytes_entry "auto_dump_max_size" | bool_entry "auto_start_bypass_cache" | int_entry "auto_start_delay" | str_entry "auto_shutdown_try_save" diff --git a/src/qemu/qemu.conf.in b/src/qemu/qemu.conf.in index 97b0141cf6..de14db6cde 100644 --- a/src/qemu/qemu.conf.in +++ b/src/qemu/qemu.conf.in @@ -688,6 +688,23 @@ #auto_dump_bypass_cache = 0 +# Total size that auto-triggered dumps (from on_crash and watchdog +# handling) are allowed to occupy under auto_dump_path. After each new +# dump is written, the oldest dumps are removed until the total fits +# the quota again. The dump that was just written is never removed by +# this, even if it alone exceeds the quota. Only the dumps libvirt +# itself wrote are counted and removed. +# +# The value is a plain byte count, or a byte count followed by a unit +# suffix: bytes/b, KB/k/KiB, MB/M/MiB, GB/G/GiB, TB/T/TiB, PB/P/PiB, or +# EB/E/EiB (decimal 'B' suffixes scale by 1000, binary 'iB' suffixes, +# and their bare single-letter equivalents, scale by 1024). +# +# Defaults to 0, which disables the quota and keeps every dump forever. +# +#auto_dump_max_size = "10GiB" + + # When a domain is configured to be auto-started, enabling this flag # has the same effect as using the VIR_DOMAIN_START_BYPASS_CACHE flag # with the virDomainCreateWithFlags API. That is, the system will diff --git a/src/qemu/qemu_conf.c b/src/qemu/qemu_conf.c index e30b146634..6d67939b6e 100644 --- a/src/qemu/qemu_conf.c +++ b/src/qemu/qemu_conf.c @@ -748,6 +748,8 @@ virQEMUDriverConfigLoadSaveEntry(virQEMUDriverConfig *cfg, return -1; if (virConfGetValueBool(conf, "auto_dump_bypass_cache", &cfg->autoDumpBypassCache) < 0) return -1; + if (virConfGetValueBytes(conf, "auto_dump_max_size", &cfg->autoDumpMaxSize) < 0) + return -1; if (virConfGetValueBool(conf, "auto_start_bypass_cache", &cfg->autoStartBypassCache) < 0) return -1; if (virConfGetValueUInt(conf, "auto_start_delay", &cfg->autoStartDelayMS) < 0) diff --git a/src/qemu/qemu_conf.h b/src/qemu/qemu_conf.h index 1d29f35c5d..9faf6db206 100644 --- a/src/qemu/qemu_conf.h +++ b/src/qemu/qemu_conf.h @@ -226,6 +226,7 @@ struct _virQEMUDriverConfig { char *autoDumpPath; bool autoDumpBypassCache; + unsigned long long autoDumpMaxSize; bool autoStartBypassCache; unsigned int autoStartDelayMS; virDomainDriverAutoShutdownConfig autoShutdown; diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c index 4766c39cbd..cb144407c4 100644 --- a/src/qemu/qemu_driver.c +++ b/src/qemu/qemu_driver.c @@ -3560,6 +3560,134 @@ qemuDomainGetAutoDumpFormat(virDomainObj *vm) } +typedef struct _qemuAutoDumpFile qemuAutoDumpFile; +struct _qemuAutoDumpFile { + char *path; + unsigned long long size; + long long mtime; +}; + +static void +qemuAutoDumpFileFree(void *opaque) +{ + qemuAutoDumpFile *file = opaque; + + g_free(file->path); + g_free(file); +} + + +static gint +qemuAutoDumpFileCompare(gconstpointer a, + gconstpointer b) +{ + qemuAutoDumpFile *fa = *(qemuAutoDumpFile **) a; + qemuAutoDumpFile *fb = *(qemuAutoDumpFile **) b; + + return fa->mtime < fb->mtime ? -1 : fa->mtime > fb->mtime; +} + + +static bool +qemuIsAutoDumpFileName(const char *name) +{ + /* "-YYYY-MM-DD-HH:MM:SS" as built by getAutoDumpPath(), 'd' any digit */ + static const char shape[] = "-dddd-dd-dd-dd:dd:dd"; + size_t shapelen = sizeof(shape) - 1; + size_t len = strlen(name); + const char *stamp; + size_t i; + + if (len <= shapelen) + return false; + + stamp = name + len - shapelen; + + for (i = 0; i < shapelen; i++) { + if (shape[i] == 'd') { + if (!g_ascii_isdigit(stamp[i])) + return false; + } else if (stamp[i] != shape[i]) { + return false; + } + } + + return true; +} + + +/* Removes the oldest dumps under autoDumpPath until the total size fits + * autoDumpMaxSize. KEEP (the dump just written) is never removed, even + * alone over quota: mtime alone can't protect it, since it is only + * second-granularity and ties with another dump written the same + * second would make the eviction order among them arbitrary. */ +static void +qemuPruneAutoDumpPath(virQEMUDriverConfig *cfg, + const char *keep) +{ + g_autoptr(DIR) dir = NULL; + struct dirent *entry; + g_autoptr(GPtrArray) files = NULL; + unsigned long long total = 0; + size_t i; + int rc; + + if (cfg->autoDumpMaxSize == 0) + return; + + if (virDirOpenQuiet(&dir, cfg->autoDumpPath) < 0) + return; + + files = g_ptr_array_new_with_free_func(qemuAutoDumpFileFree); + + while ((rc = virDirRead(dir, &entry, NULL)) > 0) { + g_autofree char *path = g_strdup_printf("%s/%s", cfg->autoDumpPath, + entry->d_name); + GStatBuf sb; + qemuAutoDumpFile *file; + unsigned long long alloc; + + if (!qemuIsAutoDumpFileName(entry->d_name)) + continue; + + if (g_stat(path, &sb) < 0 || !S_ISREG(sb.st_mode)) + continue; + + alloc = (unsigned long long)sb.st_blocks * DEV_BSIZE; + total += alloc; + + if (STREQ(path, keep)) + continue; + + file = g_new0(qemuAutoDumpFile, 1); + file->path = g_steal_pointer(&path); + file->size = alloc; + file->mtime = sb.st_mtime; + + g_ptr_array_add(files, file); + } + + if (rc < 0) + return; + + g_ptr_array_sort(files, qemuAutoDumpFileCompare); + + for (i = 0; i < files->len && total > cfg->autoDumpMaxSize; i++) { + qemuAutoDumpFile *file = g_ptr_array_index(files, i); + + if (unlink(file->path) < 0 && errno != ENOENT) { + VIR_WARN("Failed to prune old dump %s: %s", + file->path, g_strerror(errno)); + continue; + } + + VIR_DEBUG("Pruned old dump %s to satisfy auto_dump_max_size quota", + file->path); + total -= file->size; + } +} + + static void processWatchdogEvent(virQEMUDriver *driver, virDomainObj *vm, @@ -3589,6 +3717,8 @@ processWatchdogEvent(virQEMUDriver *driver, qemuDomainGetAutoDumpFormat(vm))) < 0) virReportError(VIR_ERR_OPERATION_FAILED, "%s", _("Dump failed")); + else + qemuPruneAutoDumpPath(cfg, dumpfile); ret = qemuProcessStartCPUs(driver, vm, VIR_DOMAIN_RUNNING_UNPAUSED, @@ -3624,6 +3754,8 @@ doCoreDumpToAutoDumpPath(virQEMUDriver *driver, qemuDomainGetAutoDumpFormat(vm))) < 0) virReportError(VIR_ERR_OPERATION_FAILED, "%s", _("Dump failed")); + else + qemuPruneAutoDumpPath(cfg, dumpfile); return ret; } diff --git a/src/qemu/test_libvirtd_qemu.aug.in b/src/qemu/test_libvirtd_qemu.aug.in index c4cf9cf634..c97d8d7081 100644 --- a/src/qemu/test_libvirtd_qemu.aug.in +++ b/src/qemu/test_libvirtd_qemu.aug.in @@ -83,6 +83,7 @@ module Test_libvirtd_qemu = { "snapshot_image_format" = "raw" } { "auto_dump_path" = "/var/lib/libvirt/qemu/dump" } { "auto_dump_bypass_cache" = "0" } +{ "auto_dump_max_size" = "10GiB" } { "auto_start_bypass_cache" = "0" } { "auto_start_delay" = "0" } { "auto_shutdown_try_save" = "persistent" } -- 2.53.0