From: Stefan Kober <stefan.kober@cyberus-technology.de> Make use of the numa configuration the user has configured via libvirt and transform it into the right cloud hypervisor API calls. On-behalf-of: SAP stefan.kober@sap.com On-behalf-of: SAP thomas.prescher@sap.com Signed-off-by: Stefan Kober <stefan.kober@cyberus-technology.de> Signed-off-by: Thomas Prescher <thomas.prescher@cyberus-technology.de> --- src/ch/ch_monitor.c | 140 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 138 insertions(+), 2 deletions(-) diff --git a/src/ch/ch_monitor.c b/src/ch/ch_monitor.c index 52f281cf73..bcfff58995 100644 --- a/src/ch/ch_monitor.c +++ b/src/ch/ch_monitor.c @@ -99,6 +99,73 @@ virCHMonitorBuildCPUJson(virJSONValue *content, virDomainDef *vmdef) return 0; } +/** + * CHV example NUMA cmdline: + *--numa guest_numa_id=0,cpus=[0,1],memory_zones=[fast_mem] \ + *--numa guest_numa_id=1,cpus=[2,3],memory_zones=[bulk_mem] \ + *--memory-zone id=fast_mem,size=2G,host_numa_node=0,hugepages=on,hugepage_size=1G,prefault=on \ + *--memory-zone id=bulk_mem,size=6G,host_numa_node=1,hugepages=on,hugepage_size=2M \ + */ +static int +virCHMonitorBuildNumaJSON(virJSONValue *content, + virDomainDef *def) +{ + size_t ncells = virDomainNumaGetNodeCount(def->numa); + size_t i = 0; + size_t j = 0; + virBitmap *cpus = NULL; + g_auto(virBuffer) buf = VIR_BUFFER_INITIALIZER; + g_autoptr(virJSONValue) numas = virJSONValueNewArray(); + + if (ncells == 0) { + return 0; + } + + for (i = 0; i < ncells; i++) { + ssize_t lastcpu = 0; + g_autoptr(virJSONValue) numa = virJSONValueNewObject(); + g_autoptr(virJSONValue) mem_zones = virJSONValueNewArray(); + char *mem_zone_str = g_strdup_printf("zone%lu", i); + g_autoptr(virJSONValue) mem_zone_id = virJSONValueNewString(mem_zone_str); + g_autoptr(virJSONValue) cpu_arr = virJSONValueNewArray(); + + cpus = virDomainNumaGetNodeCpumask(def->numa, i); + lastcpu = virBitmapLastSetBit(cpus); + + /* + * Go through bitmap and check set bits which correspond to CPUs + * We create an array of vCPU IDs: [1,2,3] of CPUs belonging to the + * respective NUMA node. + */ + for (j = 0; j < lastcpu + 1; j++) { + if (virBitmapIsBitSet(cpus, j)) { + g_autoptr(virJSONValue) cpu_id = virJSONValueNewNumberUint(j); + if (virJSONValueArrayAppend(cpu_arr, &cpu_id) < 0) + return -1; + } + + } + + if (virJSONValueArrayAppend(mem_zones, &mem_zone_id) < 0) + return -1; + + if (virJSONValueObjectAdd(&numa, + "U:guest_numa_id", i, + "a:memory_zones", &mem_zones, + "a:cpus", &cpu_arr, + NULL) < 0) + return -1; + + if (virJSONValueArrayAppend(numas, &numa) < 0) + return -1; + } + + if (virJSONValueObjectAppend(content, "numa", &numas) < 0) + return -1; + + return 0; +} + static int virCHMonitorBuildConsoleJson(virJSONValue *content, virDomainDef *vmdef) @@ -222,16 +289,82 @@ virCHMonitorBuildKernelRelatedJson(virJSONValue *content, virDomainDef *vmdef) return 0; } +static int +virCHMonitorBuildMemoryZonesJSON(virJSONValue *content, + virDomainDef *def) +{ + size_t ncells = virDomainNumaGetNodeCount(def->numa); + size_t i = 0; + g_autoptr(virJSONValue) zones = virJSONValueNewArray(); + + VIR_DEBUG("Creating %zu NUMA nodes for guest VM", ncells); + + for (i = 0; i < ncells; i++) { + g_autofree char *id = g_strdup_printf("zone%zu", i); + /* Memory returned is in KiB, so we multiply by 1024 */ + unsigned long long memsize = virDomainNumaGetNodeMemorySize(def->numa, i) * 1024; + g_autoptr(virJSONValue) zone = virJSONValueNewObject(); + virBitmap *nodes = virDomainNumatuneGetNodeset(def->numa, NULL, i); + g_autofree char *nodeset = virBitmapFormat(nodes); + size_t hostNodeCount = virBitmapCountBits(nodes); + size_t hostNode = virBitmapLastSetBit(nodes); + + if (hostNodeCount > 1) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, + _("%ld host nodes specified but " + "Cloud Hypervisor only supports 1"), + hostNodeCount); + return -1; + } + + if (virJSONValueObjectAdd(&zone, + "s:id", id, + "U:size", memsize, + NULL) < 0) + return -1; + + if (hostNodeCount == 1) { + VIR_DEBUG("Associating guest node %lu with host node %s", i , nodeset); + + if (virJSONValueObjectAdd(&zone, + "U:host_numa_node", hostNode, + NULL) < 0) + return -1; + } + + if (virJSONValueArrayAppend(zones, &zone) < 0) + return -1; + } + + if (virJSONValueObjectAppend(content, "zones", &zones) < 0) + return -1; + + return 0; +} + static int virCHMonitorBuildMemoryJson(virJSONValue *content, virDomainDef *vmdef) { unsigned long long total_memory = virDomainDefGetMemoryInitial(vmdef) * 1024; + size_t ncells = virDomainNumaGetNodeCount(vmdef->numa); if (total_memory != 0) { g_autoptr(virJSONValue) memory = virJSONValueNewObject(); - if (virJSONValueObjectAppendNumberUlong(memory, "size", total_memory) < 0) - return -1; + /* If we have multiple NUMA nodes, then we define memory zones. When + * memory zones are defined, the "size" field in the CHV memory config + * must be 0 */ + if (ncells >= 1) { + if (virCHMonitorBuildMemoryZonesJSON(memory, vmdef) < 0) { + return -1; + } + if (virJSONValueObjectAppendNumberUlong(memory, "size", 0) < 0) { + return -1; + } + } else { + if (virJSONValueObjectAppendNumberUlong(memory, "size", total_memory) < 0) + return -1; + } if (virJSONValueObjectAppend(content, "memory", &memory) < 0) return -1; @@ -560,6 +693,9 @@ virCHMonitorBuildVMJson(virCHDriver *driver, virDomainDef *vmdef, if (virCHMonitorBuildMemoryJson(content, vmdef) < 0) return -1; + if (virCHMonitorBuildNumaJSON(content, vmdef) < 0) + return -1; + if (virBitmapIsBitSet(driver->chCaps, CH_KERNEL_API_DEPRCATED)) { if (virCHMonitorBuildPayloadJson(content, vmdef) < 0) return -1; -- 2.53.0

From: Stefan Kober <stefan.kober@cyberus-technology.de> Parse the user provided cpu topology information and transform it into correct cloud-hypervisor API calls. On-behalf-of: SAP stefan.kober@sap.com On-behalf-of: SAP thomas.prescher@sap.com Signed-off-by: Stefan Kober <stefan.kober@cyberus-technology.de> Signed-off-by: Thomas Prescher <thomas.prescher@cyberus-technology.de> --- src/ch/ch_monitor.c | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/src/ch/ch_monitor.c b/src/ch/ch_monitor.c index bcfff58995..15fded2274 100644 --- a/src/ch/ch_monitor.c +++ b/src/ch/ch_monitor.c @@ -69,13 +69,35 @@ virCHMonitorPut(virCHMonitor *mon, domainLogContext *logCtxt, virJSONValue **answer); +static int +virCHMonitorBuildCPUTopologyJSON(virJSONValue *content, + virDomainDef *vmdef) +{ + int rc; + + if ((rc = virDomainDefGetVcpusTopology(vmdef, NULL)) != 0) + return rc; + + if (virJSONValueObjectAdd(&content, + "u:threads_per_core", vmdef->cpu->threads, + "u:cores_per_die", vmdef->cpu->cores, + "u:dies_per_package", vmdef->cpu->dies, + "u:packages", vmdef->cpu->sockets, + NULL) < 0) + return -1; + + return 0; +} + static int virCHMonitorBuildCPUJson(virJSONValue *content, virDomainDef *vmdef) { g_autoptr(virJSONValue) cpus = NULL; + g_autoptr(virJSONValue) topology = virJSONValueNewObject(); unsigned int maxvcpus = 0; unsigned int nvcpus = 0; virDomainVcpuDef *vcpu; + int rc; size_t i; /* count maximum allowed number vcpus and enabled vcpus when boot.*/ @@ -92,6 +114,17 @@ virCHMonitorBuildCPUJson(virJSONValue *content, virDomainDef *vmdef) return -1; if (virJSONValueObjectAppendNumberInt(cpus, "max_vcpus", vmdef->maxvcpus) < 0) return -1; + if ((rc = virCHMonitorBuildCPUTopologyJSON(topology, vmdef)) < 0) + return -1; + + if (rc == 0) { + VIR_DEBUG("Using CPU topology: %u:%u:%u:%u", + vmdef->cpu->sockets, vmdef->cpu->dies, + vmdef->cpu->cores, vmdef->cpu->threads); + if (virJSONValueObjectAppend(cpus, "topology", &topology) < 0) + return -1; + } + if (virJSONValueObjectAppend(content, "cpus", &cpus) < 0) return -1; } -- 2.53.0