From 0b66196d86ea375234cb0ee99289c486f9921820 Mon Sep 17 00:00:00 2001 From: Michal Privoznik Date: Thu, 8 Oct 2020 16:22:50 +0200 Subject: [PATCH] qemu: Set noqueue qdisc for TAP devices By default, pfifo_fast queueing discipline (qdisc) is set on newly created interfaces (including TAPs). This qdisc has three queues and packets that want to be sent through given NIC are placed into one of the queues based on TOS field. Queues are then emptied based on their priority allowing interactive sessions stay interactive whilst something else is downloading a large file. Obviously, this means that kernel has to be involved and some locking has to happen (when placing packets into queues). If virtualization is taken into account then the above algorithm happens twice - once in the guest and the second time in the host. This is arguably not optimal as it burns host CPU cycles needlessly. Guest already made it choice and sent packets in the order it wants. To resolve this, Linux kernel offers 'noqueue' qdisc which can be applied on virtual interfaces and in fact for 'lo' it is by default: lo: mtu 65536 qdisc noqueue Set it for other TAP devices we create for domains too. With this change I was able to squeeze 1Mbps more from a macvtap attached to a guest and to my 1Gbps LAN (as measured by iperf3). Resolves: https://bugzilla.redhat.com/show_bug.cgi?id=1329644 Signed-off-by: Michal Privoznik Reviewed-by: Daniel Henrique Barboza --- src/qemu/qemu_command.c | 2 ++ src/qemu/qemu_domain.c | 36 ++++++++++++++++++++++++++++++++++++ src/qemu/qemu_domain.h | 4 ++++ src/qemu/qemu_hotplug.c | 2 ++ src/util/virnetdev.h | 3 ++- tests/qemuxml2argvmock.c | 8 ++++++++ 6 files changed, 54 insertions(+), 1 deletion(-) diff --git a/src/qemu/qemu_command.c b/src/qemu/qemu_command.c index 9519861e92..eec860382c 100644 --- a/src/qemu/qemu_command.c +++ b/src/qemu/qemu_command.c @@ -8267,6 +8267,8 @@ qemuBuildInterfaceCommandLine(virQEMUDriverPtr driver, } } + qemuDomainInterfaceSetDefaultQDisc(driver, net); + if (net->mtu && virNetDevSetMTU(net->ifname, net->mtu) < 0) goto cleanup; diff --git a/src/qemu/qemu_domain.c b/src/qemu/qemu_domain.c index 2d7f61f5e9..5e603284be 100644 --- a/src/qemu/qemu_domain.c +++ b/src/qemu/qemu_domain.c @@ -11049,3 +11049,39 @@ qemuDomainFileWrapperFDClose(virDomainObjPtr vm, } return ret; } + + +/** + * qemuDomainInterfaceSetDefaultQDisc: + * @driver: QEMU driver + * @net: domain interface + * + * Set the noqueue qdisc on @net if running as privileged. The + * noqueue qdisc is a lockless transmit and thus faster than the + * default pfifo_fast (at least in theory). But we can modify + * root qdisc only if we have CAP_NET_ADMIN. + * + * Returns: 0 on success, + * -1 otherwise. + */ +int +qemuDomainInterfaceSetDefaultQDisc(virQEMUDriverPtr driver, + virDomainNetDefPtr net) +{ + virDomainNetType actualType = virDomainNetGetActualType(net); + + if (!driver->privileged || !net->ifname) + return 0; + + /* We want only those types which are represented as TAP + * devices in the host. */ + if (actualType == VIR_DOMAIN_NET_TYPE_ETHERNET || + actualType == VIR_DOMAIN_NET_TYPE_NETWORK || + actualType == VIR_DOMAIN_NET_TYPE_BRIDGE || + actualType == VIR_DOMAIN_NET_TYPE_DIRECT) { + if (virNetDevSetRootQDisc(net->ifname, "noqueue") < 0) + return -1; + } + + return 0; +} diff --git a/src/qemu/qemu_domain.h b/src/qemu/qemu_domain.h index 9bf32e16c9..91b3b67cb6 100644 --- a/src/qemu/qemu_domain.h +++ b/src/qemu/qemu_domain.h @@ -1046,3 +1046,7 @@ qemuDomainOpenFile(virQEMUDriverPtr driver, int qemuDomainFileWrapperFDClose(virDomainObjPtr vm, virFileWrapperFdPtr fd); + +int +qemuDomainInterfaceSetDefaultQDisc(virQEMUDriverPtr driver, + virDomainNetDefPtr net); diff --git a/src/qemu/qemu_hotplug.c b/src/qemu/qemu_hotplug.c index 7a54fcb221..2c184b9ba0 100644 --- a/src/qemu/qemu_hotplug.c +++ b/src/qemu/qemu_hotplug.c @@ -1368,6 +1368,8 @@ qemuDomainAttachNetDevice(virQEMUDriverPtr driver, virNetDevSetMTU(net->ifname, net->mtu) < 0) goto cleanup; + qemuDomainInterfaceSetDefaultQDisc(driver, net); + for (i = 0; i < tapfdSize; i++) { if (qemuSecuritySetTapFDLabel(driver->securityManager, vm->def, tapfd[i]) < 0) diff --git a/src/util/virnetdev.h b/src/util/virnetdev.h index 82943b8e08..dfef49938f 100644 --- a/src/util/virnetdev.h +++ b/src/util/virnetdev.h @@ -313,6 +313,7 @@ int virNetDevRunEthernetScript(const char *ifname, const char *script) G_GNUC_NO_INLINE; int virNetDevSetRootQDisc(const char *ifname, - const char *qdisc); + const char *qdisc) + G_GNUC_NO_INLINE; G_DEFINE_AUTOPTR_CLEANUP_FUNC(virNetDevRxFilter, virNetDevRxFilterFree); diff --git a/tests/qemuxml2argvmock.c b/tests/qemuxml2argvmock.c index 9bf4357b66..b9322f4f2a 100644 --- a/tests/qemuxml2argvmock.c +++ b/tests/qemuxml2argvmock.c @@ -286,3 +286,11 @@ qemuBuildTPMOpenBackendFDs(const char *tpmdev G_GNUC_UNUSED, *cancelfd = 1731; return 0; } + + +int +virNetDevSetRootQDisc(const char *ifname G_GNUC_UNUSED, + const char *qdisc G_GNUC_UNUSED) +{ + return 0; +}