From 181742d43fb8f450746e13148a4ffa7e1df9b64d Mon Sep 17 00:00:00 2001 From: Peter Krempa Date: Mon, 16 Feb 2015 17:28:48 +0100 Subject: [PATCH] conf: Move all NUMA configuration to virDomainNuma For historical reasons data regarding NUMA configuration were split between the CPU definition and numatune. We cannot do anything about the XML still being split, but we certainly can at least store the relevant data in one place. This patch moves the NUMA stuff to the right place. --- src/conf/cpu_conf.c | 26 +++------------- src/conf/cpu_conf.h | 12 ++------ src/conf/domain_conf.c | 8 ++--- src/conf/numa_conf.c | 67 ++++++++++++++++++++--------------------- src/conf/numa_conf.h | 20 ++++++------ src/cpu/cpu.c | 2 +- src/qemu/qemu_command.c | 20 ++++++------ tests/cputest.c | 2 +- tests/testutilsqemu.c | 2 -- 9 files changed, 63 insertions(+), 96 deletions(-) diff --git a/src/conf/cpu_conf.c b/src/conf/cpu_conf.c index 0e3ce26a3d..4e33934698 100644 --- a/src/conf/cpu_conf.c +++ b/src/conf/cpu_conf.c @@ -73,16 +73,11 @@ virCPUDefFreeModel(virCPUDefPtr def) void virCPUDefFree(virCPUDefPtr def) { - size_t i; - if (!def) return; virCPUDefFreeModel(def); - for (i = 0; i < def->ncells; i++) - virBitmapFree(def->cells[i].cpumask); - VIR_FREE(def->cells); VIR_FREE(def->vendor_id); VIR_FREE(def); @@ -126,7 +121,6 @@ virCPUDefPtr virCPUDefCopy(const virCPUDef *cpu) { virCPUDefPtr copy; - size_t i; if (!cpu || VIR_ALLOC(copy) < 0) return NULL; @@ -143,20 +137,6 @@ virCPUDefCopy(const virCPUDef *cpu) if (virCPUDefCopyModel(copy, cpu, false) < 0) goto error; - if (cpu->ncells) { - if (VIR_ALLOC_N(copy->cells, cpu->ncells) < 0) - goto error; - - for (i = 0; i < cpu->ncells; i++) { - copy->cells[i].mem = cpu->cells[i].mem; - - copy->cells[i].cpumask = virBitmapNewCopy(cpu->cells[i].cpumask); - - if (!copy->cells[i].cpumask) - goto error; - } - } - return copy; error: @@ -429,11 +409,12 @@ virCPUDefParseXML(xmlNodePtr node, char * virCPUDefFormat(virCPUDefPtr def, + virDomainNumaPtr numa, bool updateCPU) { virBuffer buf = VIR_BUFFER_INITIALIZER; - if (virCPUDefFormatBufFull(&buf, def, updateCPU) < 0) + if (virCPUDefFormatBufFull(&buf, def, numa, updateCPU) < 0) goto cleanup; if (virBufferCheckError(&buf) < 0) @@ -450,6 +431,7 @@ virCPUDefFormat(virCPUDefPtr def, int virCPUDefFormatBufFull(virBufferPtr buf, virCPUDefPtr def, + virDomainNumaPtr numa, bool updateCPU) { if (!def) @@ -489,7 +471,7 @@ virCPUDefFormatBufFull(virBufferPtr buf, if (virCPUDefFormatBuf(buf, def, updateCPU) < 0) return -1; - if (virDomainNumaDefCPUFormat(buf, def) < 0) + if (virDomainNumaDefCPUFormat(buf, numa) < 0) return -1; virBufferAdjustIndent(buf, -2); diff --git a/src/conf/cpu_conf.h b/src/conf/cpu_conf.h index d2563e2b8b..705ba6d902 100644 --- a/src/conf/cpu_conf.h +++ b/src/conf/cpu_conf.h @@ -29,6 +29,7 @@ # include "virxml.h" # include "virbitmap.h" # include "virarch.h" +# include "numa_conf.h" # define VIR_CPU_VENDOR_ID_LENGTH 12 @@ -90,13 +91,6 @@ struct _virCPUFeatureDef { int policy; /* enum virCPUFeaturePolicy */ }; -typedef struct _virCellDef virCellDef; -typedef virCellDef *virCellDefPtr; -struct _virCellDef { - virBitmapPtr cpumask; /* CPUs that are part of this node */ - unsigned long long mem; /* Node memory in kB */ - int memAccess; /* virNumaMemAccess */ -}; typedef struct _virCPUDef virCPUDef; typedef virCPUDef *virCPUDefPtr; @@ -115,8 +109,6 @@ struct _virCPUDef { size_t nfeatures; size_t nfeatures_max; virCPUFeatureDefPtr features; - size_t ncells; - virCellDefPtr cells; }; @@ -145,6 +137,7 @@ virCPUDefIsEqual(virCPUDefPtr src, char * virCPUDefFormat(virCPUDefPtr def, + virDomainNumaPtr numa, bool updateCPU); int @@ -154,6 +147,7 @@ virCPUDefFormatBuf(virBufferPtr buf, int virCPUDefFormatBufFull(virBufferPtr buf, virCPUDefPtr def, + virDomainNumaPtr numa, bool updateCPU); int diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c index 9ebf01a4b4..d947df7b6d 100644 --- a/src/conf/domain_conf.c +++ b/src/conf/domain_conf.c @@ -13517,11 +13517,10 @@ virDomainDefParseXML(xmlDocPtr xml, } - if (virDomainNumaDefCPUParseXML(def->cpu, ctxt) < 0) + if (virDomainNumaDefCPUParseXML(def->numa, ctxt) < 0) goto error; - if (def->cpu && - virDomainNumaGetCPUCountTotal(def->cpu) > def->maxvcpus) { + if (virDomainNumaGetCPUCountTotal(def->numa) > def->maxvcpus) { virReportError(VIR_ERR_INTERNAL_ERROR, "%s", _("Number of CPUs in exceeds the" " count")); @@ -13531,7 +13530,6 @@ virDomainDefParseXML(xmlDocPtr xml, if (virDomainNumatuneParseXML(def->numa, def->placement_mode == VIR_DOMAIN_CPU_PLACEMENT_MODE_STATIC, - virDomainNumaGetNodeCount(def->cpu), ctxt) < 0) goto error; @@ -20204,7 +20202,7 @@ virDomainDefFormatInternal(virDomainDefPtr def, virBufferAddLit(buf, "\n"); } - if (virCPUDefFormatBufFull(buf, def->cpu, + if (virCPUDefFormatBufFull(buf, def->cpu, def->numa, !!(flags & VIR_DOMAIN_DEF_FORMAT_UPDATE_CPU)) < 0) goto error; diff --git a/src/conf/numa_conf.c b/src/conf/numa_conf.c index 3fb086f4ba..52b9fe9be1 100644 --- a/src/conf/numa_conf.c +++ b/src/conf/numa_conf.c @@ -60,9 +60,12 @@ struct _virDomainNuma { } memory; /* pinning for all the memory */ struct _virDomainNumaNode { - virBitmapPtr nodeset; - virDomainNumatuneMemMode mode; - } *mem_nodes; /* fine tuning per guest node */ + unsigned long long mem; /* memory size in KiB */ + virBitmapPtr cpumask; /* bitmap of vCPUs corresponding to the node */ + virBitmapPtr nodeset; /* host memory nodes where this guest node resides */ + virDomainNumatuneMemMode mode; /* memory mode selection */ + virNumaMemAccess memAccess; /* shared memory access configuration */ + } *mem_nodes; /* guest node configuration */ size_t nmem_nodes; /* Future NUMA tuning related stuff should go here. */ @@ -83,7 +86,6 @@ virDomainNumatuneNodeSpecified(virDomainNumaPtr numatune, static int virDomainNumatuneNodeParseXML(virDomainNumaPtr numa, - size_t ncells, xmlXPathContextPtr ctxt) { char *tmp = NULL; @@ -109,19 +111,13 @@ virDomainNumatuneNodeParseXML(virDomainNumaPtr numa, goto cleanup; } - if (!ncells) { + if (!numa->nmem_nodes) { virReportError(VIR_ERR_XML_ERROR, "%s", _("Element 'memnode' is invalid without " "any guest NUMA cells")); goto cleanup; } - VIR_FREE(numa->mem_nodes); - if (VIR_ALLOC_N(numa->mem_nodes, ncells) < 0) - goto cleanup; - - numa->nmem_nodes = ncells; - for (i = 0; i < n; i++) { int mode = 0; unsigned int cellid = 0; @@ -195,7 +191,6 @@ virDomainNumatuneNodeParseXML(virDomainNumaPtr numa, int virDomainNumatuneParseXML(virDomainNumaPtr numa, bool placement_static, - size_t ncells, xmlXPathContextPtr ctxt) { char *tmp = NULL; @@ -251,7 +246,7 @@ virDomainNumatuneParseXML(virDomainNumaPtr numa, nodeset) < 0) goto cleanup; - if (virDomainNumatuneNodeParseXML(numa, ncells, ctxt) < 0) + if (virDomainNumatuneNodeParseXML(numa, ctxt) < 0) goto cleanup; ret = 0; @@ -332,8 +327,10 @@ virDomainNumaFree(virDomainNumaPtr numa) return; virBitmapFree(numa->memory.nodeset); - for (i = 0; i < numa->nmem_nodes; i++) + for (i = 0; i < numa->nmem_nodes; i++) { + virBitmapFree(numa->mem_nodes[i].cpumask); virBitmapFree(numa->mem_nodes[i].nodeset); + } VIR_FREE(numa->mem_nodes); VIR_FREE(numa); @@ -655,7 +652,7 @@ virDomainNumatuneNodesetIsAvailable(virDomainNumaPtr numatune, int -virDomainNumaDefCPUParseXML(virCPUDefPtr def, +virDomainNumaDefCPUParseXML(virDomainNumaPtr def, xmlXPathContextPtr ctxt) { xmlNodePtr *nodes = NULL; @@ -675,9 +672,9 @@ virDomainNumaDefCPUParseXML(virCPUDefPtr def, goto cleanup; } - if (VIR_ALLOC_N(def->cells, n) < 0) + if (VIR_ALLOC_N(def->mem_nodes, n) < 0) goto cleanup; - def->ncells = n; + def->nmem_nodes = n; for (i = 0; i < n; i++) { int rc; @@ -702,7 +699,7 @@ virDomainNumaDefCPUParseXML(virCPUDefPtr def, } VIR_FREE(tmp); - if (def->cells[cur_cell].cpumask) { + if (def->mem_nodes[cur_cell].cpumask) { virReportError(VIR_ERR_XML_ERROR, _("Duplicate NUMA cell info for cell id '%u'"), cur_cell); @@ -715,11 +712,11 @@ virDomainNumaDefCPUParseXML(virCPUDefPtr def, goto cleanup; } - if (virBitmapParse(tmp, 0, &def->cells[cur_cell].cpumask, + if (virBitmapParse(tmp, 0, &def->mem_nodes[cur_cell].cpumask, VIR_DOMAIN_CPUMASK_LEN) < 0) goto cleanup; - if (virBitmapIsAllClear(def->cells[cur_cell].cpumask)) { + if (virBitmapIsAllClear(def->mem_nodes[cur_cell].cpumask)) { virReportError(VIR_ERR_CONFIG_UNSUPPORTED, _("NUMA cell %d has no vCPUs assigned"), cur_cell); goto cleanup; @@ -728,7 +725,7 @@ virDomainNumaDefCPUParseXML(virCPUDefPtr def, ctxt->node = nodes[i]; if (virDomainParseMemory("./@memory", "./@unit", ctxt, - &def->cells[cur_cell].mem, true, false) < 0) + &def->mem_nodes[cur_cell].mem, true, false) < 0) goto cleanup; if ((tmp = virXMLPropString(nodes[i], "memAccess"))) { @@ -739,7 +736,7 @@ virDomainNumaDefCPUParseXML(virCPUDefPtr def, goto cleanup; } - def->cells[cur_cell].memAccess = rc; + def->mem_nodes[cur_cell].memAccess = rc; VIR_FREE(tmp); } } @@ -756,7 +753,7 @@ virDomainNumaDefCPUParseXML(virCPUDefPtr def, int virDomainNumaDefCPUFormat(virBufferPtr buf, - virCPUDefPtr def) + virDomainNumaPtr def) { virNumaMemAccess memAccess; char *cpustr; @@ -794,12 +791,12 @@ virDomainNumaDefCPUFormat(virBufferPtr buf, unsigned int -virDomainNumaGetCPUCountTotal(virCPUDefPtr numa) +virDomainNumaGetCPUCountTotal(virDomainNumaPtr numa) { size_t i; unsigned int ret = 0; - for (i = 0; i < numa->ncells; i++) + for (i = 0; i < numa->nmem_nodes; i++) ret += virBitmapCountBits(virDomainNumaGetNodeCpumask(numa, i)); return ret; @@ -818,43 +815,43 @@ virDomainNumaNew(void) size_t -virDomainNumaGetNodeCount(virCPUDefPtr numa) +virDomainNumaGetNodeCount(virDomainNumaPtr numa) { if (!numa) return 0; - return numa->ncells; + return numa->nmem_nodes; } virBitmapPtr -virDomainNumaGetNodeCpumask(virCPUDefPtr numa, +virDomainNumaGetNodeCpumask(virDomainNumaPtr numa, size_t node) { - return numa->cells[node].cpumask; + return numa->mem_nodes[node].cpumask; } virNumaMemAccess -virDomainNumaGetNodeMemoryAccessMode(virCPUDefPtr numa, +virDomainNumaGetNodeMemoryAccessMode(virDomainNumaPtr numa, size_t node) { - return numa->cells[node].memAccess; + return numa->mem_nodes[node].memAccess; } unsigned long long -virDomainNumaGetNodeMemorySize(virCPUDefPtr numa, +virDomainNumaGetNodeMemorySize(virDomainNumaPtr numa, size_t node) { - return numa->cells[node].mem; + return numa->mem_nodes[node].mem; } void -virDomainNumaSetNodeMemorySize(virCPUDefPtr numa, +virDomainNumaSetNodeMemorySize(virDomainNumaPtr numa, size_t node, unsigned long long size) { - numa->cells[node].mem = size; + numa->mem_nodes[node].mem = size; } diff --git a/src/conf/numa_conf.h b/src/conf/numa_conf.h index cda850b3d0..0a0a06e7b7 100644 --- a/src/conf/numa_conf.h +++ b/src/conf/numa_conf.h @@ -29,7 +29,6 @@ # include "virutil.h" # include "virbitmap.h" # include "virbuffer.h" -# include "cpu_conf.h" typedef struct _virDomainNuma virDomainNuma; @@ -64,9 +63,8 @@ void virDomainNumaFree(virDomainNumaPtr numa); */ int virDomainNumatuneParseXML(virDomainNumaPtr numa, bool placement_static, - size_t ncells, xmlXPathContextPtr ctxt) - ATTRIBUTE_NONNULL(1) ATTRIBUTE_NONNULL(4); + ATTRIBUTE_NONNULL(1) ATTRIBUTE_NONNULL(3); int virDomainNumatuneFormatXML(virBufferPtr buf, virDomainNumaPtr numatune) ATTRIBUTE_NONNULL(1); @@ -86,15 +84,15 @@ int virDomainNumatuneMaybeGetNodeset(virDomainNumaPtr numatune, virBitmapPtr *retNodeset, int cellid); -size_t virDomainNumaGetNodeCount(virCPUDefPtr numa) +size_t virDomainNumaGetNodeCount(virDomainNumaPtr numa) ATTRIBUTE_NONNULL(1); -virBitmapPtr virDomainNumaGetNodeCpumask(virCPUDefPtr numa, +virBitmapPtr virDomainNumaGetNodeCpumask(virDomainNumaPtr numa, size_t node) ATTRIBUTE_NONNULL(1); -virNumaMemAccess virDomainNumaGetNodeMemoryAccessMode(virCPUDefPtr numa, +virNumaMemAccess virDomainNumaGetNodeMemoryAccessMode(virDomainNumaPtr numa, size_t node) ATTRIBUTE_NONNULL(1); -unsigned long long virDomainNumaGetNodeMemorySize(virCPUDefPtr numa, +unsigned long long virDomainNumaGetNodeMemorySize(virDomainNumaPtr numa, size_t node) ATTRIBUTE_NONNULL(1); @@ -120,7 +118,7 @@ int virDomainNumatuneSet(virDomainNumaPtr numa, virBitmapPtr nodeset) ATTRIBUTE_NONNULL(1); -void virDomainNumaSetNodeMemorySize(virCPUDefPtr numa, +void virDomainNumaSetNodeMemorySize(virDomainNumaPtr numa, size_t node, unsigned long long size) ATTRIBUTE_NONNULL(1); @@ -143,10 +141,10 @@ bool virDomainNumatuneNodesetIsAvailable(virDomainNumaPtr numatune, bool virDomainNumatuneNodeSpecified(virDomainNumaPtr numatune, int cellid); -int virDomainNumaDefCPUParseXML(virCPUDefPtr def, xmlXPathContextPtr ctxt); -int virDomainNumaDefCPUFormat(virBufferPtr buf, virCPUDefPtr def); +int virDomainNumaDefCPUParseXML(virDomainNumaPtr def, xmlXPathContextPtr ctxt); +int virDomainNumaDefCPUFormat(virBufferPtr buf, virDomainNumaPtr def); -unsigned int virDomainNumaGetCPUCountTotal(virCPUDefPtr numa); +unsigned int virDomainNumaGetCPUCountTotal(virDomainNumaPtr numa); #endif /* __NUMA_CONF_H__ */ diff --git a/src/cpu/cpu.c b/src/cpu/cpu.c index 6976b7bbc8..9e67ddd8cc 100644 --- a/src/cpu/cpu.c +++ b/src/cpu/cpu.c @@ -472,7 +472,7 @@ cpuBaselineXML(const char **xmlCPUs, if (!(cpu = cpuBaseline(cpus, ncpus, models, nmodels, flags))) goto error; - cpustr = virCPUDefFormat(cpu, false); + cpustr = virCPUDefFormat(cpu, NULL, false); cleanup: if (cpus) { diff --git a/src/qemu/qemu_command.c b/src/qemu/qemu_command.c index dec7a326a7..7853125c08 100644 --- a/src/qemu/qemu_command.c +++ b/src/qemu/qemu_command.c @@ -4558,7 +4558,7 @@ qemuBuildMemoryBackendStr(unsigned long long size, virDomainHugePagePtr hugepage = NULL; virDomainNumatuneMemMode mode; const long system_page_size = virGetSystemPageSizeKB(); - virNumaMemAccess memAccess = virDomainNumaGetNodeMemoryAccessMode(def->cpu, guestNode); + virNumaMemAccess memAccess = virDomainNumaGetNodeMemoryAccessMode(def->numa, guestNode); size_t i; char *mem_path = NULL; @@ -4747,7 +4747,7 @@ qemuBuildMemoryCellBackendStr(virDomainDefPtr def, if (virAsprintf(&alias, "ram-node%zu", cell) < 0) goto cleanup; - if ((rc = qemuBuildMemoryBackendStr(virDomainNumaGetNodeMemorySize(def->cpu, cell), + if ((rc = qemuBuildMemoryBackendStr(virDomainNumaGetNodeMemorySize(def->numa, cell), 0, cell, NULL, auto_nodeset, def, qemuCaps, cfg, @@ -7123,7 +7123,7 @@ qemuBuildNumaArgStr(virQEMUDriverConfigPtr cfg, bool needBackend = false; int rc; int ret = -1; - size_t ncells = virDomainNumaGetNodeCount(def->cpu); + size_t ncells = virDomainNumaGetNodeCount(def->numa); const long system_page_size = virGetSystemPageSizeKB(); if (virDomainNumatuneHasPerNodeBinding(def->numa) && @@ -7177,8 +7177,8 @@ qemuBuildNumaArgStr(virQEMUDriverConfigPtr cfg, /* using of -numa memdev= cannot be combined with -numa mem=, thus we * need to check which approach to use */ for (i = 0; i < ncells; i++) { - unsigned long long cellmem = virDomainNumaGetNodeMemorySize(def->cpu, i); - virDomainNumaSetNodeMemorySize(def->cpu, i, VIR_ROUND_UP(cellmem, 1024)); + unsigned long long cellmem = virDomainNumaGetNodeMemorySize(def->numa, i); + virDomainNumaSetNodeMemorySize(def->numa, i, VIR_ROUND_UP(cellmem, 1024)); if (virQEMUCapsGet(qemuCaps, QEMU_CAPS_OBJECT_MEMORY_RAM) || virQEMUCapsGet(qemuCaps, QEMU_CAPS_OBJECT_MEMORY_FILE)) { @@ -7190,7 +7190,7 @@ qemuBuildNumaArgStr(virQEMUDriverConfigPtr cfg, if (rc == 0) needBackend = true; } else { - if (virDomainNumaGetNodeMemoryAccessMode(def->cpu, i)) { + if (virDomainNumaGetNodeMemoryAccessMode(def->numa, i)) { virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", _("Shared memory mapping is not supported " "with this QEMU")); @@ -7201,7 +7201,7 @@ qemuBuildNumaArgStr(virQEMUDriverConfigPtr cfg, for (i = 0; i < ncells; i++) { VIR_FREE(cpumask); - if (!(cpumask = virBitmapFormat(virDomainNumaGetNodeCpumask(def->cpu, i)))) + if (!(cpumask = virBitmapFormat(virDomainNumaGetNodeCpumask(def->numa, i)))) goto cleanup; if (strchr(cpumask, ',') && @@ -7229,7 +7229,7 @@ qemuBuildNumaArgStr(virQEMUDriverConfigPtr cfg, virBufferAsprintf(&buf, ",memdev=ram-node%zu", i); else virBufferAsprintf(&buf, ",mem=%llu", - virDomainNumaGetNodeMemorySize(def->cpu, i) / 1024); + virDomainNumaGetNodeMemorySize(def->numa, i) / 1024); virCommandAddArgBuffer(cmd, &buf); } @@ -8321,7 +8321,7 @@ qemuBuildCommandLine(virConnectPtr conn, virCommandAddArg(cmd, "-m"); def->mem.max_balloon = VIR_DIV_UP(def->mem.max_balloon, 1024) * 1024; virCommandAddArgFormat(cmd, "%llu", def->mem.max_balloon / 1024); - if (def->mem.nhugepages && !virDomainNumaGetNodeCount(def->cpu)) { + if (def->mem.nhugepages && !virDomainNumaGetNodeCount(def->numa)) { const long system_page_size = virGetSystemPageSizeKB(); char *mem_path = NULL; @@ -8401,7 +8401,7 @@ qemuBuildCommandLine(virConnectPtr conn, } } - if (virDomainNumaGetNodeCount(def->cpu)) + if (virDomainNumaGetNodeCount(def->numa)) if (qemuBuildNumaArgStr(cfg, def, cmd, qemuCaps, nodeset) < 0) goto error; diff --git a/tests/cputest.c b/tests/cputest.c index e49ae2471a..76cc9d415a 100644 --- a/tests/cputest.c +++ b/tests/cputest.c @@ -167,7 +167,7 @@ cpuTestCompareXML(const char *arch, if (virtTestLoadFile(xml, &expected) < 0) goto cleanup; - if (!(actual = virCPUDefFormat(cpu, updateCPU))) + if (!(actual = virCPUDefFormat(cpu, NULL, updateCPU))) goto cleanup; if (STRNEQ(expected, actual)) { diff --git a/tests/testutilsqemu.c b/tests/testutilsqemu.c index 4a3df8ab22..e3d4503079 100644 --- a/tests/testutilsqemu.c +++ b/tests/testutilsqemu.c @@ -242,8 +242,6 @@ virCapsPtr testQemuCapsInit(void) ARRAY_CARDINALITY(host_cpu_features), /* nfeatures */ ARRAY_CARDINALITY(host_cpu_features), /* nfeatures_max */ host_cpu_features, /* features */ - 0, /* ncells */ - NULL, /* cells */ }; if ((caps = virCapabilitiesNew(host_cpu.arch,