On Tue, Jul 07, 2026 at 09:48:22AM +0200, Denis V. Lunev via Devel wrote:
Each on_crash or watchdog-triggered dump writes a full memory dump into auto_dump_path. A guest that keeps crashing and restarting (or crashing and getting destroyed, then respawned by the mgmt app) can fill the disk one dump at a time, with nothing to stop it.
Add auto_dump_max_size (qemu.conf), parsed via virConfGetValueBytes() so it takes a plain byte count or a size with a unit suffix (e.g. "10GiB"). After each dump attempt, the oldest files under auto_dump_path are removed until the total fits the configured quota. The dump that was just written is always kept by identity, not by sort position: mtime is only second-granularity, so two dumps written the same second would otherwise make the eviction order between them arbitrary and could delete the one just written instead of an older one. Defaults to 0, which keeps every dump forever, as before.
This is a good idea, however, I wonder if we should also have a global cross-VM limit, as with cloud you might equally see a loop where a VM fails to spawn and automation spins up a new VM with different identity each time. Perhaps also "auto_dump_min_free_space" where we statfs the dump location to figure out current free space, subtract the VM RAM size (as a rough approximation for worst case dump size), and then reject the dump if the result would be under the limit. If you want to do that, it could be a separate followup patch so no need to repost this series just for that.
Signed-off-by: Denis V. Lunev <den@openvz.org> --- src/qemu/qemu.conf.in | 16 +++++++ src/qemu/qemu_conf.c | 2 + src/qemu/qemu_conf.h | 1 + src/qemu/qemu_driver.c | 97 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 116 insertions(+)
diff --git a/src/qemu/qemu.conf.in b/src/qemu/qemu.conf.in index 97b0141cf6..3828af0fab 100644 --- a/src/qemu/qemu.conf.in +++ b/src/qemu/qemu.conf.in @@ -688,6 +688,22 @@ #auto_dump_bypass_cache = 0
+# Total size that auto-triggered dumps (from on_crash and watchdog +# handling) are allowed to occupy under auto_dump_path. After each new +# dump is written, the oldest dumps are removed until the total fits +# the quota again. The dump that was just written is never removed by +# this, even if it alone exceeds the quota. +# +# The value is a plain byte count, or a byte count followed by a unit +# suffix: bytes/b, KB/k/KiB, MB/M/MiB, GB/G/GiB, TB/T/TiB, PB/P/PiB, or +# EB/E/EiB (decimal 'B' suffixes scale by 1000, binary 'iB' suffixes, +# and their bare single-letter equivalents, scale by 1024). +# +# Defaults to 0, which disables the quota and keeps every dump forever. +# +#auto_dump_max_size = "10GiB" + + # When a domain is configured to be auto-started, enabling this flag # has the same effect as using the VIR_DOMAIN_START_BYPASS_CACHE flag # with the virDomainCreateWithFlags API. That is, the system will diff --git a/src/qemu/qemu_conf.c b/src/qemu/qemu_conf.c index e30b146634..6d67939b6e 100644 --- a/src/qemu/qemu_conf.c +++ b/src/qemu/qemu_conf.c @@ -748,6 +748,8 @@ virQEMUDriverConfigLoadSaveEntry(virQEMUDriverConfig *cfg, return -1; if (virConfGetValueBool(conf, "auto_dump_bypass_cache", &cfg->autoDumpBypassCache) < 0) return -1; + if (virConfGetValueBytes(conf, "auto_dump_max_size", &cfg->autoDumpMaxSize) < 0) + return -1; if (virConfGetValueBool(conf, "auto_start_bypass_cache", &cfg->autoStartBypassCache) < 0) return -1; if (virConfGetValueUInt(conf, "auto_start_delay", &cfg->autoStartDelayMS) < 0) diff --git a/src/qemu/qemu_conf.h b/src/qemu/qemu_conf.h index 1d29f35c5d..9faf6db206 100644 --- a/src/qemu/qemu_conf.h +++ b/src/qemu/qemu_conf.h @@ -226,6 +226,7 @@ struct _virQEMUDriverConfig {
char *autoDumpPath; bool autoDumpBypassCache; + unsigned long long autoDumpMaxSize; bool autoStartBypassCache; unsigned int autoStartDelayMS; virDomainDriverAutoShutdownConfig autoShutdown; diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c index b59a714a81..3f021bb7c5 100644 --- a/src/qemu/qemu_driver.c +++ b/src/qemu/qemu_driver.c @@ -3557,6 +3557,101 @@ qemuDomainGetAutoDumpFormat(virDomainObj *vm) }
+typedef struct _qemuAutoDumpFile qemuAutoDumpFile; +struct _qemuAutoDumpFile { + char *path; + unsigned long long size; + long long mtime; +}; + +static void +qemuAutoDumpFileFree(void *opaque) +{ + qemuAutoDumpFile *file = opaque; + + g_free(file->path); + g_free(file); +} + + +static gint +qemuAutoDumpFileCompare(gconstpointer a, + gconstpointer b) +{ + qemuAutoDumpFile *fa = *(qemuAutoDumpFile **) a; + qemuAutoDumpFile *fb = *(qemuAutoDumpFile **) b; + + return fa->mtime < fb->mtime ? -1 : fa->mtime > fb->mtime; +} + + +/* Removes the oldest files under autoDumpPath until the total size fits + * autoDumpMaxSize. KEEP (the dump just written) is never removed, even + * alone over quota: mtime alone can't protect it, since it is only + * second-granularity and ties with another dump written the same + * second would make the eviction order among them arbitrary. */ +static void +qemuPruneAutoDumpPath(virQEMUDriverConfig *cfg, + const char *keep) +{ + g_autoptr(DIR) dir = NULL; + struct dirent *entry; + g_autoptr(GPtrArray) files = NULL; + unsigned long long total = 0; + size_t i; + int rc; + + if (cfg->autoDumpMaxSize == 0) + return; + + if (virDirOpenQuiet(&dir, cfg->autoDumpPath) < 0) + return; + + files = g_ptr_array_new_with_free_func(qemuAutoDumpFileFree); + + while ((rc = virDirRead(dir, &entry, NULL)) > 0) { + g_autofree char *path = g_strdup_printf("%s/%s", cfg->autoDumpPath, + entry->d_name); + GStatBuf sb; + qemuAutoDumpFile *file; + + if (g_stat(path, &sb) < 0 || !S_ISREG(sb.st_mode)) + continue; + + total += sb.st_size; + + if (STREQ(path, keep)) + continue; + + file = g_new0(qemuAutoDumpFile, 1); + file->path = g_steal_pointer(&path); + file->size = sb.st_size; + file->mtime = sb.st_mtime; + + g_ptr_array_add(files, file); + } + + if (rc < 0) + return; + + g_ptr_array_sort(files, qemuAutoDumpFileCompare); + + for (i = 0; i < files->len && total > cfg->autoDumpMaxSize; i++) { + qemuAutoDumpFile *file = g_ptr_array_index(files, i); + + if (unlink(file->path) < 0 && errno != ENOENT) { + VIR_WARN("Failed to prune old dump %s: %s", + file->path, g_strerror(errno)); + continue; + } + + VIR_INFO("Pruned old dump %s to satisfy auto_dump_max_size quota", + file->path); + total -= file->size; + } +} + + static void processWatchdogEvent(virQEMUDriver *driver, virDomainObj *vm, @@ -3586,6 +3681,7 @@ processWatchdogEvent(virQEMUDriver *driver, qemuDomainGetAutoDumpFormat(vm))) < 0) virReportError(VIR_ERR_OPERATION_FAILED, "%s", _("Dump failed")); + qemuPruneAutoDumpPath(cfg, dumpfile);
ret = qemuProcessStartCPUs(driver, vm, VIR_DOMAIN_RUNNING_UNPAUSED, @@ -3621,6 +3717,7 @@ doCoreDumpToAutoDumpPath(virQEMUDriver *driver, qemuDomainGetAutoDumpFormat(vm))) < 0) virReportError(VIR_ERR_OPERATION_FAILED, "%s", _("Dump failed")); + qemuPruneAutoDumpPath(cfg, dumpfile); return ret; }
-- 2.53.0
With regards, Daniel -- |: https://berrange.com ~~ https://hachyderm.io/@berrange :| |: https://libvirt.org ~~ https://entangle-photo.org :| |: https://pixelfed.art/berrange ~~ https://fstop138.berrange.com :|