From 2d37d8dbc987d1998b4ad8029ba324b6bfe49799 Mon Sep 17 00:00:00 2001 From: Luyao Zhong Date: Thu, 15 Apr 2021 15:55:04 +0200 Subject: [PATCH] qemu: Add support for 'restrictive' mode in numatune Signed-off-by: Luyao Zhong Signed-off-by: Martin Kletzander Reviewed-by: Daniel Henrique Barboza Reviewed-by: Michal Privoznik --- src/qemu/qemu_command.c | 5 ++- src/qemu/qemu_process.c | 32 ++++++++++++++++ ...emnode-restrictive-mode.x86_64-latest.args | 38 +++++++++++++++++++ tests/qemuxml2argvtest.c | 1 + 4 files changed, 75 insertions(+), 1 deletion(-) create mode 100644 tests/qemuxml2argvdata/numatune-memnode-restrictive-mode.x86_64-latest.args diff --git a/src/qemu/qemu_command.c b/src/qemu/qemu_command.c index 6d40983ce1..278590c2c1 100644 --- a/src/qemu/qemu_command.c +++ b/src/qemu/qemu_command.c @@ -3302,7 +3302,10 @@ qemuBuildMemoryBackendProps(virJSONValue **backendProps, return -1; } - if (nodemask) { + /* If mode is "restrictive", we should only use cgroups setting allowed memory + * nodes, and skip passing the host-nodes and policy parameters to QEMU command + * line which means we will use system default memory policy. */ + if (nodemask && mode != VIR_DOMAIN_NUMATUNE_MEM_RESTRICTIVE) { if (!virNumaNodesetIsAvailable(nodemask)) return -1; if (virJSONValueObjectAdd(props, diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c index 4df33dc0d5..05104e3098 100644 --- a/src/qemu/qemu_process.c +++ b/src/qemu/qemu_process.c @@ -2692,6 +2692,7 @@ qemuProcessSetupPid(virDomainObj *vm, g_autoptr(virBitmap) hostcpumap = NULL; g_autofree char *mem_mask = NULL; int ret = -1; + size_t i; if ((period || quota) && !virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPU)) { @@ -2732,6 +2733,37 @@ qemuProcessSetupPid(virDomainObj *vm, &mem_mask, -1) < 0) goto cleanup; + /* For restrictive numatune mode we need to set cpuset.mems for vCPU + * threads based on the node they are in as there is nothing else uses + * for such restriction (e.g. numa_set_membind). */ + if (nameval == VIR_CGROUP_THREAD_VCPU) { + virDomainNuma *numatune = vm->def->numa; + + /* Look for the guest NUMA node of this vCPU */ + for (i = 0; i < virDomainNumaGetNodeCount(numatune); i++) { + g_autoptr(virBitmap) node_cpus = NULL; + node_cpus = virDomainNumaGetNodeCpumask(numatune, i); + + if (!virBitmapIsBitSet(node_cpus, id)) + continue; + + /* Update the mem_mask for this vCPU if the mode of its node is + * 'restrictive'. */ + if (virDomainNumatuneGetMode(numatune, i, &mem_mode) == 0 && + mem_mode == VIR_DOMAIN_NUMATUNE_MEM_RESTRICTIVE) { + VIR_FREE(mem_mask); + + if (virDomainNumatuneMaybeFormatNodeset(numatune, + priv->autoNodeset, + &mem_mask, i) < 0) { + goto cleanup; + } + } + + break; + } + } + if (virCgroupNewThread(priv->cgroup, nameval, id, true, &cgroup) < 0) goto cleanup; diff --git a/tests/qemuxml2argvdata/numatune-memnode-restrictive-mode.x86_64-latest.args b/tests/qemuxml2argvdata/numatune-memnode-restrictive-mode.x86_64-latest.args new file mode 100644 index 0000000000..6d5e2eb76e --- /dev/null +++ b/tests/qemuxml2argvdata/numatune-memnode-restrictive-mode.x86_64-latest.args @@ -0,0 +1,38 @@ +LC_ALL=C \ +PATH=/bin \ +HOME=/tmp/lib/domain--1-QEMUGuest \ +USER=test \ +LOGNAME=test \ +XDG_DATA_HOME=/tmp/lib/domain--1-QEMUGuest/.local/share \ +XDG_CACHE_HOME=/tmp/lib/domain--1-QEMUGuest/.cache \ +XDG_CONFIG_HOME=/tmp/lib/domain--1-QEMUGuest/.config \ +/usr/bin/qemu-system-x86_64 \ +-name guest=QEMUGuest,debug-threads=on \ +-S \ +-object '{"qom-type":"secret","id":"masterKey0","format":"raw","file":"/tmp/lib/domain--1-QEMUGuest/master-key.aes"}' \ +-machine pc,accel=tcg,usb=off,dump-guest-core=off \ +-cpu qemu64 \ +-m 24105 \ +-overcommit mem-lock=off \ +-smp 32,sockets=32,cores=1,threads=1 \ +-object '{"qom-type":"memory-backend-ram","id":"ram-node0","size":20971520}' \ +-numa node,nodeid=0,cpus=0,memdev=ram-node0 \ +-object '{"qom-type":"memory-backend-ram","id":"ram-node1","size":676331520}' \ +-numa node,nodeid=1,cpus=1-27,cpus=29,memdev=ram-node1 \ +-object '{"qom-type":"memory-backend-ram","id":"ram-node2","size":24578621440}' \ +-numa node,nodeid=2,cpus=28,cpus=30-31,memdev=ram-node2 \ +-uuid 9f4b6512-e73a-4a25-93e8-5307802821ce \ +-display none \ +-no-user-config \ +-nodefaults \ +-chardev socket,id=charmonitor,fd=1729,server=on,wait=off \ +-mon chardev=charmonitor,id=monitor,mode=control \ +-rtc base=utc \ +-no-shutdown \ +-no-acpi \ +-boot strict=on \ +-device piix3-usb-uhci,id=usb,bus=pci.0,addr=0x1.0x2 \ +-audiodev id=audio1,driver=none \ +-device virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x2 \ +-sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny \ +-msg timestamp=on diff --git a/tests/qemuxml2argvtest.c b/tests/qemuxml2argvtest.c index e919bef986..572c7b251a 100644 --- a/tests/qemuxml2argvtest.c +++ b/tests/qemuxml2argvtest.c @@ -2172,6 +2172,7 @@ mymain(void) DO_TEST_CAPS_VER("numatune-memnode", "5.2.0"); DO_TEST_CAPS_LATEST("numatune-memnode"); DO_TEST_PARSE_ERROR("numatune-memnode-invalid-mode", NONE); + DO_TEST_CAPS_LATEST("numatune-memnode-restrictive-mode"); DO_TEST("numatune-memnode-no-memory", QEMU_CAPS_NUMA,