From c7fc151eec74c7bbb380f78268a45d0d1e559d52 Mon Sep 17 00:00:00 2001 From: Laine Stump Date: Sat, 13 Aug 2016 18:10:41 -0400 Subject: [PATCH] qemu: assign virtio devices to PCIe slot when appropriate libvirt previously assigned nearly all devices to a "hotpluggable" legacy PCI slot even on machines with a PCIe root bus (and even though most such machines don't even support hotplug on legacy PCI slots!) Forcing all devices onto legacy PCI slots means that the domain will need a dmi-to-pci-bridge (to convert from PCIe to legacy PCI) and a pci-bridge (to provide hotpluggable legacy PCI slots which, again, usually aren't hotpluggable anyway). To help reduce the need for these legacy controllers, this patch tries to assign virtio-1.0-capable devices to PCIe slots whenever possible, by setting appropriate connectFlags in virDomainCalculateDevicePCIConnectFlags(). Happily, when that function was written (just a few commits ago) it was created with a "virtioFlags" argument, set by both of its callers, which is the proper connectFlags to set for any virtio-*-pci device - depending on the arch/machinetype of the domain, and whether or not the qemu binary supports virtio-1.0, that flag will have either been set to PCI or PCIe. This patch merely enables the functionality by setting the flags for the device to whatever is in virtioFlags if the device is a virtio-*-pci device. NB: the first virtio video device will be placed directly on bus 0 slot 1 rather than on a pcie-root-port due to the override for primary video devices in qemuDomainValidateDevicePCISlotsQ35(). Whether or not to change that is a topic of discussion, but this patch doesn't change that particular behavior. NB2: since the slot must be hotpluggable, and pcie-root (the PCIe root complex) does *not* support hotplug, this means that suitable controllers must also be in the config (i.e. either pcie-root-port, or pcie-downstream-port). For now, libvirt doesn't add those automatically, so if you put virtio devices in a config for a qemu that has PCIe-capable virtio devices, you'll need to add extra pcie-root-ports yourself. That requirement will be eliminated in a future patch, but for now, it's simple to do this: ... Partially Resolves: https://bugzilla.redhat.com/show_bug.cgi?id=1330024 --- src/qemu/qemu_domain_address.c | 42 ++++- .../qemuxml2argv-q35-pcie.args | 58 +++++++ .../qemuxml2argv-q35-pcie.xml | 61 +++++++ .../qemuxml2argv-q35-virtio-pci.args | 58 +++++++ .../qemuxml2argv-q35-virtio-pci.xml | 1 + tests/qemuxml2argvtest.c | 48 ++++++ .../qemuxml2xmlout-q35-pcie.xml | 154 ++++++++++++++++++ .../qemuxml2xmlout-q35-virtio-pci.xml | 154 ++++++++++++++++++ tests/qemuxml2xmltest.c | 43 +++++ 9 files changed, 610 insertions(+), 9 deletions(-) create mode 100644 tests/qemuxml2argvdata/qemuxml2argv-q35-pcie.args create mode 100644 tests/qemuxml2argvdata/qemuxml2argv-q35-pcie.xml create mode 100644 tests/qemuxml2argvdata/qemuxml2argv-q35-virtio-pci.args create mode 120000 tests/qemuxml2argvdata/qemuxml2argv-q35-virtio-pci.xml create mode 100644 tests/qemuxml2xmloutdata/qemuxml2xmlout-q35-pcie.xml create mode 100644 tests/qemuxml2xmloutdata/qemuxml2xmlout-q35-virtio-pci.xml diff --git a/src/qemu/qemu_domain_address.c b/src/qemu/qemu_domain_address.c index 4c688b1898..72b8c88307 100644 --- a/src/qemu/qemu_domain_address.c +++ b/src/qemu/qemu_domain_address.c @@ -430,8 +430,7 @@ static virDomainPCIConnectFlags qemuDomainDeviceCalculatePCIConnectFlags(virDomainDeviceDefPtr dev, virDomainPCIConnectFlags pcieFlags ATTRIBUTE_UNUSED, - virDomainPCIConnectFlags virtioFlags - ATTRIBUTE_UNUSED) + virDomainPCIConnectFlags virtioFlags) { virDomainPCIConnectFlags pciFlags = (VIR_PCI_CONNECT_TYPE_PCI_DEVICE | VIR_PCI_CONNECT_HOTPLUGGABLE); @@ -471,10 +470,29 @@ qemuDomainDeviceCalculatePCIConnectFlags(virDomainDeviceDefPtr dev, } case VIR_DOMAIN_CONTROLLER_TYPE_IDE: - case VIR_DOMAIN_CONTROLLER_TYPE_SCSI: - case VIR_DOMAIN_CONTROLLER_TYPE_VIRTIO_SERIAL: return pciFlags; + case VIR_DOMAIN_CONTROLLER_TYPE_SCSI: + switch ((virDomainControllerModelSCSI) cont->model) { + case VIR_DOMAIN_CONTROLLER_MODEL_SCSI_VIRTIO_SCSI: + return virtioFlags; + + case VIR_DOMAIN_CONTROLLER_MODEL_SCSI_AUTO: + case VIR_DOMAIN_CONTROLLER_MODEL_SCSI_BUSLOGIC: + case VIR_DOMAIN_CONTROLLER_MODEL_SCSI_LSILOGIC: + case VIR_DOMAIN_CONTROLLER_MODEL_SCSI_LSISAS1068: + case VIR_DOMAIN_CONTROLLER_MODEL_SCSI_VMPVSCSI: + case VIR_DOMAIN_CONTROLLER_MODEL_SCSI_IBMVSCSI: + case VIR_DOMAIN_CONTROLLER_MODEL_SCSI_LSISAS1078: + return pciFlags; + + case VIR_DOMAIN_CONTROLLER_MODEL_SCSI_LAST: + return 0; + } + + case VIR_DOMAIN_CONTROLLER_TYPE_VIRTIO_SERIAL: + return virtioFlags; + case VIR_DOMAIN_CONTROLLER_TYPE_FDC: case VIR_DOMAIN_CONTROLLER_TYPE_CCID: case VIR_DOMAIN_CONTROLLER_TYPE_LAST: @@ -485,7 +503,7 @@ qemuDomainDeviceCalculatePCIConnectFlags(virDomainDeviceDefPtr dev, case VIR_DOMAIN_DEVICE_FS: /* the only type of filesystem so far is virtio-9p-pci */ - return pciFlags; + return virtioFlags; case VIR_DOMAIN_DEVICE_NET: { virDomainNetDefPtr net = dev->data.net; @@ -498,6 +516,10 @@ qemuDomainDeviceCalculatePCIConnectFlags(virDomainDeviceDefPtr dev, STREQ(net->model, "usb-net")) { return 0; } + + if (STREQ(net->model, "virtio")) + return virtioFlags; + return pciFlags; } @@ -519,7 +541,7 @@ qemuDomainDeviceCalculatePCIConnectFlags(virDomainDeviceDefPtr dev, case VIR_DOMAIN_DEVICE_DISK: switch ((virDomainDiskBus) dev->data.disk->bus) { case VIR_DOMAIN_DISK_BUS_VIRTIO: - return pciFlags; /* only virtio disks use PCI */ + return virtioFlags; /* only virtio disks use PCI */ case VIR_DOMAIN_DISK_BUS_IDE: case VIR_DOMAIN_DISK_BUS_FDC: @@ -539,7 +561,7 @@ qemuDomainDeviceCalculatePCIConnectFlags(virDomainDeviceDefPtr dev, case VIR_DOMAIN_DEVICE_MEMBALLOON: switch ((virDomainMemballoonModel) dev->data.memballoon->model) { case VIR_DOMAIN_MEMBALLOON_MODEL_VIRTIO: - return pciFlags; + return virtioFlags; case VIR_DOMAIN_MEMBALLOON_MODEL_XEN: case VIR_DOMAIN_MEMBALLOON_MODEL_NONE: @@ -550,7 +572,7 @@ qemuDomainDeviceCalculatePCIConnectFlags(virDomainDeviceDefPtr dev, case VIR_DOMAIN_DEVICE_RNG: switch ((virDomainRNGModel) dev->data.rng->model) { case VIR_DOMAIN_RNG_MODEL_VIRTIO: - return pciFlags; + return virtioFlags; case VIR_DOMAIN_RNG_MODEL_LAST: return 0; @@ -571,6 +593,8 @@ qemuDomainDeviceCalculatePCIConnectFlags(virDomainDeviceDefPtr dev, case VIR_DOMAIN_DEVICE_VIDEO: switch ((virDomainVideoType) dev->data.video->type) { case VIR_DOMAIN_VIDEO_TYPE_VIRTIO: + return virtioFlags; + case VIR_DOMAIN_VIDEO_TYPE_VGA: case VIR_DOMAIN_VIDEO_TYPE_CIRRUS: case VIR_DOMAIN_VIDEO_TYPE_VMVGA: @@ -590,7 +614,7 @@ qemuDomainDeviceCalculatePCIConnectFlags(virDomainDeviceDefPtr dev, case VIR_DOMAIN_DEVICE_INPUT: switch ((virDomainInputBus) dev->data.input->bus) { case VIR_DOMAIN_INPUT_BUS_VIRTIO: - return pciFlags; + return virtioFlags; case VIR_DOMAIN_INPUT_BUS_PS2: case VIR_DOMAIN_INPUT_BUS_USB: diff --git a/tests/qemuxml2argvdata/qemuxml2argv-q35-pcie.args b/tests/qemuxml2argvdata/qemuxml2argv-q35-pcie.args new file mode 100644 index 0000000000..f07c085867 --- /dev/null +++ b/tests/qemuxml2argvdata/qemuxml2argv-q35-pcie.args @@ -0,0 +1,58 @@ +LC_ALL=C \ +PATH=/bin \ +HOME=/home/test \ +USER=test \ +LOGNAME=test \ +QEMU_AUDIO_DRV=none \ +/usr/libexec/qemu-kvm \ +-name q35-test \ +-S \ +-M q35 \ +-m 2048 \ +-smp 2,sockets=2,cores=1,threads=1 \ +-uuid 11dbdcdd-4c3b-482b-8903-9bdb8c0a2774 \ +-nographic \ +-nodefaults \ +-monitor unix:/tmp/lib/domain--1-q35-test/monitor.sock,server,nowait \ +-no-acpi \ +-boot c \ +-device i82801b11-bridge,id=pci.1,bus=pcie.0,addr=0x1e \ +-device pci-bridge,chassis_nr=2,id=pci.2,bus=pci.1,addr=0x0 \ +-device ioh3420,port=0x10,chassis=3,id=pci.3,bus=pcie.0,addr=0x2 \ +-device ioh3420,port=0x18,chassis=4,id=pci.4,bus=pcie.0,addr=0x3 \ +-device ioh3420,port=0x20,chassis=5,id=pci.5,bus=pcie.0,addr=0x4 \ +-device ioh3420,port=0x28,chassis=6,id=pci.6,bus=pcie.0,addr=0x5 \ +-device ioh3420,port=0x30,chassis=7,id=pci.7,bus=pcie.0,addr=0x6 \ +-device ioh3420,port=0x38,chassis=8,id=pci.8,bus=pcie.0,addr=0x7 \ +-device ioh3420,port=0x40,chassis=9,id=pci.9,bus=pcie.0,addr=0x8 \ +-device ioh3420,port=0x48,chassis=10,id=pci.10,bus=pcie.0,addr=0x9 \ +-device ioh3420,port=0x50,chassis=11,id=pci.11,bus=pcie.0,addr=0xa \ +-device ioh3420,port=0x58,chassis=12,id=pci.12,bus=pcie.0,addr=0xb \ +-device ioh3420,port=0x60,chassis=13,id=pci.13,bus=pcie.0,addr=0xc \ +-device ioh3420,port=0x68,chassis=14,id=pci.14,bus=pcie.0,addr=0xd \ +-device ich9-usb-ehci1,id=usb,bus=pcie.0,addr=0x1d.0x7 \ +-device ich9-usb-uhci1,masterbus=usb.0,firstport=0,bus=pcie.0,multifunction=on,\ +addr=0x1d \ +-device ich9-usb-uhci2,masterbus=usb.0,firstport=2,bus=pcie.0,addr=0x1d.0x1 \ +-device ich9-usb-uhci3,masterbus=usb.0,firstport=4,bus=pcie.0,addr=0x1d.0x2 \ +-device virtio-scsi-pci,id=scsi0,bus=pci.6,addr=0x0 \ +-device virtio-serial-pci,id=virtio-serial0,bus=pci.5,addr=0x0 \ +-drive file=/dev/HostVG/QEMUGuest1,format=raw,if=none,id=drive-virtio-disk1 \ +-device virtio-blk-pci,bus=pci.7,addr=0x0,drive=drive-virtio-disk1,\ +id=virtio-disk1 \ +-fsdev local,security_model=passthrough,id=fsdev-fs0,path=/export/to/guest \ +-device virtio-9p-pci,id=fs0,fsdev=fsdev-fs0,mount_tag=/import/from/host,\ +bus=pci.3,addr=0x0 \ +-netdev user,id=hostnet0 \ +-device virtio-net-pci,netdev=hostnet0,id=net0,mac=00:11:22:33:44:55,bus=pci.4,\ +addr=0x0 \ +-device virtio-input-host-pci,id=input0,evdev=/dev/input/event1234,bus=pci.10,\ +addr=0x0 \ +-device virtio-mouse-pci,id=input1,bus=pci.11,addr=0x0 \ +-device virtio-keyboard-pci,id=input2,bus=pci.12,addr=0x0 \ +-device virtio-tablet-pci,id=input3,bus=pci.13,addr=0x0 \ +-device virtio-gpu-pci,id=video0,bus=pcie.0,addr=0x1 \ +-device virtio-balloon-pci,id=balloon0,bus=pci.8,addr=0x0 \ +-object rng-random,id=objrng0,filename=/dev/urandom \ +-device virtio-rng-pci,rng=objrng0,id=rng0,max-bytes=123,period=1234,bus=pci.9,\ +addr=0x0 diff --git a/tests/qemuxml2argvdata/qemuxml2argv-q35-pcie.xml b/tests/qemuxml2argvdata/qemuxml2argv-q35-pcie.xml new file mode 100644 index 0000000000..be2439efb5 --- /dev/null +++ b/tests/qemuxml2argvdata/qemuxml2argv-q35-pcie.xml @@ -0,0 +1,61 @@ + + q35-test + 11dbdcdd-4c3b-482b-8903-9bdb8c0a2774 + 2097152 + 2097152 + 2 + + hvm + + + + destroy + restart + destroy + + /usr/libexec/qemu-kvm + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + /dev/urandom + + + + + + + + + diff --git a/tests/qemuxml2argvdata/qemuxml2argv-q35-virtio-pci.args b/tests/qemuxml2argvdata/qemuxml2argv-q35-virtio-pci.args new file mode 100644 index 0000000000..98284a217a --- /dev/null +++ b/tests/qemuxml2argvdata/qemuxml2argv-q35-virtio-pci.args @@ -0,0 +1,58 @@ +LC_ALL=C \ +PATH=/bin \ +HOME=/home/test \ +USER=test \ +LOGNAME=test \ +QEMU_AUDIO_DRV=none \ +/usr/libexec/qemu-kvm \ +-name q35-test \ +-S \ +-M q35 \ +-m 2048 \ +-smp 2,sockets=2,cores=1,threads=1 \ +-uuid 11dbdcdd-4c3b-482b-8903-9bdb8c0a2774 \ +-nographic \ +-nodefaults \ +-monitor unix:/tmp/lib/domain--1-q35-test/monitor.sock,server,nowait \ +-no-acpi \ +-boot c \ +-device i82801b11-bridge,id=pci.1,bus=pcie.0,addr=0x1e \ +-device pci-bridge,chassis_nr=2,id=pci.2,bus=pci.1,addr=0x0 \ +-device ioh3420,port=0x10,chassis=3,id=pci.3,bus=pcie.0,addr=0x2 \ +-device ioh3420,port=0x18,chassis=4,id=pci.4,bus=pcie.0,addr=0x3 \ +-device ioh3420,port=0x20,chassis=5,id=pci.5,bus=pcie.0,addr=0x4 \ +-device ioh3420,port=0x28,chassis=6,id=pci.6,bus=pcie.0,addr=0x5 \ +-device ioh3420,port=0x30,chassis=7,id=pci.7,bus=pcie.0,addr=0x6 \ +-device ioh3420,port=0x38,chassis=8,id=pci.8,bus=pcie.0,addr=0x7 \ +-device ioh3420,port=0x40,chassis=9,id=pci.9,bus=pcie.0,addr=0x8 \ +-device ioh3420,port=0x48,chassis=10,id=pci.10,bus=pcie.0,addr=0x9 \ +-device ioh3420,port=0x50,chassis=11,id=pci.11,bus=pcie.0,addr=0xa \ +-device ioh3420,port=0x58,chassis=12,id=pci.12,bus=pcie.0,addr=0xb \ +-device ioh3420,port=0x60,chassis=13,id=pci.13,bus=pcie.0,addr=0xc \ +-device ioh3420,port=0x68,chassis=14,id=pci.14,bus=pcie.0,addr=0xd \ +-device ich9-usb-ehci1,id=usb,bus=pcie.0,addr=0x1d.0x7 \ +-device ich9-usb-uhci1,masterbus=usb.0,firstport=0,bus=pcie.0,multifunction=on,\ +addr=0x1d \ +-device ich9-usb-uhci2,masterbus=usb.0,firstport=2,bus=pcie.0,addr=0x1d.0x1 \ +-device ich9-usb-uhci3,masterbus=usb.0,firstport=4,bus=pcie.0,addr=0x1d.0x2 \ +-device virtio-scsi-pci,id=scsi0,bus=pci.2,addr=0x4 \ +-device virtio-serial-pci,id=virtio-serial0,bus=pci.2,addr=0x3 \ +-drive file=/dev/HostVG/QEMUGuest1,format=raw,if=none,id=drive-virtio-disk1 \ +-device virtio-blk-pci,bus=pci.2,addr=0x5,drive=drive-virtio-disk1,\ +id=virtio-disk1 \ +-fsdev local,security_model=passthrough,id=fsdev-fs0,path=/export/to/guest \ +-device virtio-9p-pci,id=fs0,fsdev=fsdev-fs0,mount_tag=/import/from/host,\ +bus=pci.2,addr=0x1 \ +-netdev user,id=hostnet0 \ +-device virtio-net-pci,netdev=hostnet0,id=net0,mac=00:11:22:33:44:55,bus=pci.2,\ +addr=0x2 \ +-device virtio-input-host-pci,id=input0,evdev=/dev/input/event1234,bus=pci.2,\ +addr=0x8 \ +-device virtio-mouse-pci,id=input1,bus=pci.2,addr=0x9 \ +-device virtio-keyboard-pci,id=input2,bus=pci.2,addr=0xa \ +-device virtio-tablet-pci,id=input3,bus=pci.2,addr=0xb \ +-device virtio-gpu-pci,id=video0,bus=pcie.0,addr=0x1 \ +-device virtio-balloon-pci,id=balloon0,bus=pci.2,addr=0x6 \ +-object rng-random,id=objrng0,filename=/dev/urandom \ +-device virtio-rng-pci,rng=objrng0,id=rng0,max-bytes=123,period=1234,bus=pci.2,\ +addr=0x7 diff --git a/tests/qemuxml2argvdata/qemuxml2argv-q35-virtio-pci.xml b/tests/qemuxml2argvdata/qemuxml2argv-q35-virtio-pci.xml new file mode 120000 index 0000000000..fc8c0aded9 --- /dev/null +++ b/tests/qemuxml2argvdata/qemuxml2argv-q35-virtio-pci.xml @@ -0,0 +1 @@ +qemuxml2argv-q35-pcie.xml \ No newline at end of file diff --git a/tests/qemuxml2argvtest.c b/tests/qemuxml2argvtest.c index 1ee84025ea..dec3056af6 100644 --- a/tests/qemuxml2argvtest.c +++ b/tests/qemuxml2argvtest.c @@ -1770,6 +1770,54 @@ mymain(void) QEMU_CAPS_PCI_MULTIFUNCTION, QEMU_CAPS_ICH9_USB_EHCI1, QEMU_CAPS_DEVICE_VIDEO_PRIMARY, QEMU_CAPS_DEVICE_QXL); + /* verify that devices with pcie capability are assigned to a pcie slot */ + DO_TEST("q35-pcie", + QEMU_CAPS_VIRTIO_PCI_DISABLE_LEGACY, + QEMU_CAPS_DEVICE_VIRTIO_RNG, + QEMU_CAPS_OBJECT_RNG_RANDOM, + QEMU_CAPS_NETDEV, + QEMU_CAPS_DEVICE_VIRTIO_NET, + QEMU_CAPS_DEVICE_VIRTIO_GPU, + QEMU_CAPS_VIRTIO_GPU_VIRGL, + QEMU_CAPS_VIRTIO_KEYBOARD, + QEMU_CAPS_VIRTIO_MOUSE, + QEMU_CAPS_VIRTIO_TABLET, + QEMU_CAPS_VIRTIO_INPUT_HOST, + QEMU_CAPS_VIRTIO_SCSI, + QEMU_CAPS_FSDEV, + QEMU_CAPS_FSDEV_WRITEOUT, + QEMU_CAPS_DEVICE_PCI_BRIDGE, + QEMU_CAPS_DEVICE_DMI_TO_PCI_BRIDGE, + QEMU_CAPS_DEVICE_IOH3420, + QEMU_CAPS_ICH9_AHCI, + QEMU_CAPS_PCI_MULTIFUNCTION, + QEMU_CAPS_ICH9_USB_EHCI1, + QEMU_CAPS_DEVICE_VIDEO_PRIMARY); + /* same XML as q35-pcie, but don't set + * QEMU_CAPS_VIRTIO_PCI_DISABLE_LEGACY, so virtio devices should + * be assigned to legacy pci slots + */ + DO_TEST("q35-virtio-pci", + QEMU_CAPS_DEVICE_VIRTIO_RNG, + QEMU_CAPS_OBJECT_RNG_RANDOM, + QEMU_CAPS_NETDEV, + QEMU_CAPS_DEVICE_VIRTIO_NET, + QEMU_CAPS_DEVICE_VIRTIO_GPU, + QEMU_CAPS_VIRTIO_GPU_VIRGL, + QEMU_CAPS_VIRTIO_KEYBOARD, + QEMU_CAPS_VIRTIO_MOUSE, + QEMU_CAPS_VIRTIO_TABLET, + QEMU_CAPS_VIRTIO_INPUT_HOST, + QEMU_CAPS_VIRTIO_SCSI, + QEMU_CAPS_FSDEV, + QEMU_CAPS_FSDEV_WRITEOUT, + QEMU_CAPS_DEVICE_PCI_BRIDGE, + QEMU_CAPS_DEVICE_DMI_TO_PCI_BRIDGE, + QEMU_CAPS_DEVICE_IOH3420, + QEMU_CAPS_ICH9_AHCI, + QEMU_CAPS_PCI_MULTIFUNCTION, + QEMU_CAPS_ICH9_USB_EHCI1, + QEMU_CAPS_DEVICE_VIDEO_PRIMARY); DO_TEST("pcie-root-port", QEMU_CAPS_DEVICE_PCI_BRIDGE, QEMU_CAPS_DEVICE_DMI_TO_PCI_BRIDGE, diff --git a/tests/qemuxml2xmloutdata/qemuxml2xmlout-q35-pcie.xml b/tests/qemuxml2xmloutdata/qemuxml2xmlout-q35-pcie.xml new file mode 100644 index 0000000000..48e9b02ffa --- /dev/null +++ b/tests/qemuxml2xmloutdata/qemuxml2xmlout-q35-pcie.xml @@ -0,0 +1,154 @@ + + q35-test + 11dbdcdd-4c3b-482b-8903-9bdb8c0a2774 + 2097152 + 2097152 + 2 + + hvm + + + + destroy + restart + destroy + + /usr/libexec/qemu-kvm + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + +
+ + +
+ + +
+ + + +
+ + + +
+ + + +
+ + +
+ + + + +
+ + + + +
+ + + +
+ + +
+ + +
+ + +
+ + + +