From 63c30a6e79c844647d628aecfed589627824e877 Mon Sep 17 00:00:00 2001 From: Sebastien Boeuf Date: Mon, 7 Oct 2019 22:05:08 -0700 Subject: [PATCH] vmm: Build and set the list of external mappings for VFIO When VFIO devices are created and if the device is attached to the virtual IOMMU, the ExternalDmaMapping trait implementation is created and associated with the device. The idea is to build a hash map of device IDs with their associated trait implementation. This hash map is provided to the virtual IOMMU device so that it knows how to properly trigger external mappings associated with VFIO devices. Signed-off-by: Sebastien Boeuf --- src/main.rs | 6 +- .../clear/openstack/latest/user_data | 4 +- vmm/src/device_manager.rs | 74 ++++++++++--------- 3 files changed, 44 insertions(+), 40 deletions(-) diff --git a/src/main.rs b/src/main.rs index 8ea37eba8..27a3b9a37 100755 --- a/src/main.rs +++ b/src/main.rs @@ -2904,7 +2904,7 @@ mod tests { .ssh_command("ls /sys/kernel/iommu_groups/0/devices") .unwrap() .trim(), - "0000:00:03.0" + "0000:00:02.0" ); // Verify the second disk is located under IOMMU group 1. @@ -2914,7 +2914,7 @@ mod tests { .ssh_command("ls /sys/kernel/iommu_groups/1/devices") .unwrap() .trim(), - "0000:00:04.0" + "0000:00:03.0" ); // Verify the network card is located under IOMMU group 2. @@ -2924,7 +2924,7 @@ mod tests { .ssh_command("ls /sys/kernel/iommu_groups/2/devices") .unwrap() .trim(), - "0000:00:05.0" + "0000:00:04.0" ); guest.ssh_command("sudo shutdown -h now")?; diff --git a/test_data/cloud-init/clear/openstack/latest/user_data b/test_data/cloud-init/clear/openstack/latest/user_data index fc88fae1d..e15e3f0e4 100644 --- a/test_data/cloud-init/clear/openstack/latest/user_data +++ b/test_data/cloud-init/clear/openstack/latest/user_data @@ -48,7 +48,7 @@ write_files: #!/bin/bash mount -t virtio_fs virtiofs /mnt -o rootmode=040000,user_id=0,group_id=0,dax - bash -c "echo 0000:00:06.0 > /sys/bus/pci/devices/0000\:00\:06.0/driver/unbind" + bash -c "echo 0000:00:05.0 > /sys/bus/pci/devices/0000\:00\:05.0/driver/unbind" bash -c "echo 1af4 1041 > /sys/bus/pci/drivers/vfio-pci/new_id" - /mnt/cloud-hypervisor --kernel /mnt/vmlinux --cmdline "console=hvc0 reboot=k panic=1 nomodules i8042.noaux i8042.nomux i8042.nopnp i8042.dumbkbd root=/dev/vda2 VFIOTAG" --disk path=/mnt/clear-cloudguest.img path=/mnt/cloudinit.img --cpus 1 --memory size=512M --rng --device path=/sys/bus/pci/devices/0000:00:06.0/ + /mnt/cloud-hypervisor --kernel /mnt/vmlinux --cmdline "console=hvc0 reboot=k panic=1 nomodules i8042.noaux i8042.nomux i8042.nopnp i8042.dumbkbd root=/dev/vda2 VFIOTAG" --disk path=/mnt/clear-cloudguest.img path=/mnt/cloudinit.img --cpus 1 --memory size=512M --rng --device path=/sys/bus/pci/devices/0000:00:05.0/ diff --git a/vmm/src/device_manager.rs b/vmm/src/device_manager.rs index e59387266..88270e4cf 100644 --- a/vmm/src/device_manager.rs +++ b/vmm/src/device_manager.rs @@ -463,32 +463,10 @@ impl DeviceManager { let pci_root = PciRoot::new(None); let mut pci_bus = PciBus::new(pci_root); - let (iommu_mapping, iommu_id) = if vm_info.vm_cfg.iommu { - let (iommu_device, mapping) = + let (mut iommu_device, iommu_mapping) = if vm_info.vm_cfg.iommu { + let (device, mapping) = vm_virtio::Iommu::new().map_err(DeviceManagerError::CreateVirtioIommu)?; - - // We need to shift the device id since the 3 first bits - // are dedicated to the PCI function, and we know we don't - // do multifunction. Also, because we only support one PCI - // bus, the bus 0, we don't need to add anything to the - // global device ID. - let iommu_id = pci_bus.next_device_id() << 3; - - // Because we determined the virtio-iommu b/d/f, we have to - // add the device to the PCI topology now. Otherwise, the - // b/d/f won't match the virtio-iommu device as expected. - DeviceManager::add_virtio_pci_device( - Box::new(iommu_device), - vm_info.memory, - allocator, - vm_info.vm_fd, - &mut pci_bus, - &mut buses, - &interrupt_info, - &None, - )?; - - (Some(mapping), Some(iommu_id)) + (Some(device), Some(mapping)) } else { (None, None) }; @@ -518,17 +496,39 @@ impl DeviceManager { } } - let mut iommu_attached_vfio_devices = DeviceManager::add_vfio_devices( + let mut vfio_iommu_device_ids = DeviceManager::add_vfio_devices( vm_info, allocator, &mut pci_bus, &mut buses, mem_slots, + &mut iommu_device, )?; - iommu_attached_devices.append(&mut iommu_attached_vfio_devices); + iommu_attached_devices.append(&mut vfio_iommu_device_ids); + + if let Some(iommu_device) = iommu_device { + // We need to shift the device id since the 3 first bits + // are dedicated to the PCI function, and we know we don't + // do multifunction. Also, because we only support one PCI + // bus, the bus 0, we don't need to add anything to the + // global device ID. + let iommu_id = pci_bus.next_device_id() << 3; + + // Because we determined the virtio-iommu b/d/f, we have to + // add the device to the PCI topology now. Otherwise, the + // b/d/f won't match the virtio-iommu device as expected. + DeviceManager::add_virtio_pci_device( + Box::new(iommu_device), + vm_info.memory, + allocator, + vm_info.vm_fd, + &mut pci_bus, + &mut buses, + &interrupt_info, + &None, + )?; - if let Some(iommu_id) = iommu_id { virt_iommu = Some((iommu_id, iommu_attached_devices)); } @@ -982,9 +982,10 @@ impl DeviceManager { pci: &mut PciBus, buses: &mut BusInfo, mem_slots: u32, + iommu_device: &mut Option, ) -> DeviceManagerResult> { let mut mem_slot = mem_slots; - let mut iommu_attached_list = Vec::new(); + let mut iommu_attached_device_ids = Vec::new(); if let Some(device_list_cfg) = &vm_info.vm_cfg.devices { // Create the KVM VFIO device let device_fd = DeviceManager::create_kvm_device(vm_info.vm_fd)?; @@ -1003,12 +1004,15 @@ impl DeviceManager { .map_err(DeviceManagerError::VfioCreate)?; if device_cfg.iommu { - let _vfio_mapping = VfioDmaMapping::new( - vfio_device.get_container(), - Arc::clone(vm_info.memory), - ); + if let Some(iommu) = iommu_device { + let vfio_mapping = Arc::new(VfioDmaMapping::new( + vfio_device.get_container(), + Arc::clone(vm_info.memory), + )); - iommu_attached_list.push(device_id); + iommu_attached_device_ids.push(device_id); + iommu.add_external_mapping(device_id, vfio_mapping); + } } let mut vfio_pci_device = VfioPciDevice::new(vm_info.vm_fd, allocator, vfio_device) @@ -1031,7 +1035,7 @@ impl DeviceManager { .map_err(DeviceManagerError::AddPciDevice)?; } } - Ok(iommu_attached_list) + Ok(iommu_attached_device_ids) } #[cfg(feature = "pci_support")]