diff --git a/src/main.rs b/src/main.rs index 8ea37eba8..27a3b9a37 100755 --- a/src/main.rs +++ b/src/main.rs @@ -2904,7 +2904,7 @@ mod tests { .ssh_command("ls /sys/kernel/iommu_groups/0/devices") .unwrap() .trim(), - "0000:00:03.0" + "0000:00:02.0" ); // Verify the second disk is located under IOMMU group 1. @@ -2914,7 +2914,7 @@ mod tests { .ssh_command("ls /sys/kernel/iommu_groups/1/devices") .unwrap() .trim(), - "0000:00:04.0" + "0000:00:03.0" ); // Verify the network card is located under IOMMU group 2. @@ -2924,7 +2924,7 @@ mod tests { .ssh_command("ls /sys/kernel/iommu_groups/2/devices") .unwrap() .trim(), - "0000:00:05.0" + "0000:00:04.0" ); guest.ssh_command("sudo shutdown -h now")?; diff --git a/test_data/cloud-init/clear/openstack/latest/user_data b/test_data/cloud-init/clear/openstack/latest/user_data index fc88fae1d..e15e3f0e4 100644 --- a/test_data/cloud-init/clear/openstack/latest/user_data +++ b/test_data/cloud-init/clear/openstack/latest/user_data @@ -48,7 +48,7 @@ write_files: #!/bin/bash mount -t virtio_fs virtiofs /mnt -o rootmode=040000,user_id=0,group_id=0,dax - bash -c "echo 0000:00:06.0 > /sys/bus/pci/devices/0000\:00\:06.0/driver/unbind" + bash -c "echo 0000:00:05.0 > /sys/bus/pci/devices/0000\:00\:05.0/driver/unbind" bash -c "echo 1af4 1041 > /sys/bus/pci/drivers/vfio-pci/new_id" - /mnt/cloud-hypervisor --kernel /mnt/vmlinux --cmdline "console=hvc0 reboot=k panic=1 nomodules i8042.noaux i8042.nomux i8042.nopnp i8042.dumbkbd root=/dev/vda2 VFIOTAG" --disk path=/mnt/clear-cloudguest.img path=/mnt/cloudinit.img --cpus 1 --memory size=512M --rng --device path=/sys/bus/pci/devices/0000:00:06.0/ + /mnt/cloud-hypervisor --kernel /mnt/vmlinux --cmdline "console=hvc0 reboot=k panic=1 nomodules i8042.noaux i8042.nomux i8042.nopnp i8042.dumbkbd root=/dev/vda2 VFIOTAG" --disk path=/mnt/clear-cloudguest.img path=/mnt/cloudinit.img --cpus 1 --memory size=512M --rng --device path=/sys/bus/pci/devices/0000:00:05.0/ diff --git a/vmm/src/device_manager.rs b/vmm/src/device_manager.rs index e59387266..88270e4cf 100644 --- a/vmm/src/device_manager.rs +++ b/vmm/src/device_manager.rs @@ -463,32 +463,10 @@ impl DeviceManager { let pci_root = PciRoot::new(None); let mut pci_bus = PciBus::new(pci_root); - let (iommu_mapping, iommu_id) = if vm_info.vm_cfg.iommu { - let (iommu_device, mapping) = + let (mut iommu_device, iommu_mapping) = if vm_info.vm_cfg.iommu { + let (device, mapping) = vm_virtio::Iommu::new().map_err(DeviceManagerError::CreateVirtioIommu)?; - - // We need to shift the device id since the 3 first bits - // are dedicated to the PCI function, and we know we don't - // do multifunction. Also, because we only support one PCI - // bus, the bus 0, we don't need to add anything to the - // global device ID. - let iommu_id = pci_bus.next_device_id() << 3; - - // Because we determined the virtio-iommu b/d/f, we have to - // add the device to the PCI topology now. Otherwise, the - // b/d/f won't match the virtio-iommu device as expected. - DeviceManager::add_virtio_pci_device( - Box::new(iommu_device), - vm_info.memory, - allocator, - vm_info.vm_fd, - &mut pci_bus, - &mut buses, - &interrupt_info, - &None, - )?; - - (Some(mapping), Some(iommu_id)) + (Some(device), Some(mapping)) } else { (None, None) }; @@ -518,17 +496,39 @@ impl DeviceManager { } } - let mut iommu_attached_vfio_devices = DeviceManager::add_vfio_devices( + let mut vfio_iommu_device_ids = DeviceManager::add_vfio_devices( vm_info, allocator, &mut pci_bus, &mut buses, mem_slots, + &mut iommu_device, )?; - iommu_attached_devices.append(&mut iommu_attached_vfio_devices); + iommu_attached_devices.append(&mut vfio_iommu_device_ids); + + if let Some(iommu_device) = iommu_device { + // We need to shift the device id since the 3 first bits + // are dedicated to the PCI function, and we know we don't + // do multifunction. Also, because we only support one PCI + // bus, the bus 0, we don't need to add anything to the + // global device ID. + let iommu_id = pci_bus.next_device_id() << 3; + + // Because we determined the virtio-iommu b/d/f, we have to + // add the device to the PCI topology now. Otherwise, the + // b/d/f won't match the virtio-iommu device as expected. + DeviceManager::add_virtio_pci_device( + Box::new(iommu_device), + vm_info.memory, + allocator, + vm_info.vm_fd, + &mut pci_bus, + &mut buses, + &interrupt_info, + &None, + )?; - if let Some(iommu_id) = iommu_id { virt_iommu = Some((iommu_id, iommu_attached_devices)); } @@ -982,9 +982,10 @@ impl DeviceManager { pci: &mut PciBus, buses: &mut BusInfo, mem_slots: u32, + iommu_device: &mut Option, ) -> DeviceManagerResult> { let mut mem_slot = mem_slots; - let mut iommu_attached_list = Vec::new(); + let mut iommu_attached_device_ids = Vec::new(); if let Some(device_list_cfg) = &vm_info.vm_cfg.devices { // Create the KVM VFIO device let device_fd = DeviceManager::create_kvm_device(vm_info.vm_fd)?; @@ -1003,12 +1004,15 @@ impl DeviceManager { .map_err(DeviceManagerError::VfioCreate)?; if device_cfg.iommu { - let _vfio_mapping = VfioDmaMapping::new( - vfio_device.get_container(), - Arc::clone(vm_info.memory), - ); + if let Some(iommu) = iommu_device { + let vfio_mapping = Arc::new(VfioDmaMapping::new( + vfio_device.get_container(), + Arc::clone(vm_info.memory), + )); - iommu_attached_list.push(device_id); + iommu_attached_device_ids.push(device_id); + iommu.add_external_mapping(device_id, vfio_mapping); + } } let mut vfio_pci_device = VfioPciDevice::new(vm_info.vm_fd, allocator, vfio_device) @@ -1031,7 +1035,7 @@ impl DeviceManager { .map_err(DeviceManagerError::AddPciDevice)?; } } - Ok(iommu_attached_list) + Ok(iommu_attached_device_ids) } #[cfg(feature = "pci_support")]