virtio-devices: Map mmio over virtio-iommu

Add infrastructure to lookup the host address for mmio regions on
external dma mapping requests. This specifically resolves vfio
passthrough for virtio-iommu, allowing for nested virtualization to pass
external devices through.

Fixes #6110

Signed-off-by: Andrew Carp <acarp@crusoeenergy.com>

pci/src/lib.rs

@@ -26,7 +26,7 @@ pub use self::device::{
 };
 pub use self::msi::{msi_num_enabled_vectors, MsiCap, MsiConfig};
 pub use self::msix::{MsixCap, MsixConfig, MsixTableEntry, MSIX_CONFIG_ID, MSIX_TABLE_ENTRY_SIZE};
-pub use self::vfio::{VfioDmaMapping, VfioPciDevice, VfioPciError};
+pub use self::vfio::{MmioRegion, VfioDmaMapping, VfioPciDevice, VfioPciError};
 pub use self::vfio_user::{VfioUserDmaMapping, VfioUserPciDevice, VfioUserPciDeviceError};
 use serde::de::Visitor;
 use std::fmt::{self, Display};

pci/src/vfio.rs

@@ -275,6 +275,48 @@ pub struct MmioRegion {
     pub(crate) index: u32,
     pub(crate) user_memory_regions: Vec<UserMemoryRegion>,
 }
+
+trait MmioRegionRange {
+    fn check_range(&self, guest_addr: u64, size: u64) -> bool;
+    fn find_user_address(&self, guest_addr: u64) -> Result<u64, io::Error>;
+}
+
+impl MmioRegionRange for Vec<MmioRegion> {
+    // Check if a guest address is within the range of mmio regions
+    fn check_range(&self, guest_addr: u64, size: u64) -> bool {
+        for region in self.iter() {
+            let Some(guest_addr_end) = guest_addr.checked_add(size) else {
+                return false;
+            };
+            let Some(region_end) = region.start.raw_value().checked_add(region.length) else {
+                return false;
+            };
+            if guest_addr >= region.start.raw_value() && guest_addr_end <= region_end {
+                return true;
+            }
+        }
+        false
+    }
+
+    // Locate the user region address for a guest address within all mmio regions
+    fn find_user_address(&self, guest_addr: u64) -> Result<u64, io::Error> {
+        for region in self.iter() {
+            for user_region in region.user_memory_regions.iter() {
+                if guest_addr >= user_region.start
+                    && guest_addr < user_region.start + user_region.size
+                {
+                    return Ok(user_region.host_addr + (guest_addr - user_region.start));
+                }
+            }
+        }
+
+        Err(io::Error::new(
+            io::ErrorKind::Other,
+            format!("unable to find user address: 0x{guest_addr:x}"),
+        ))
+    }
+}
+
 #[derive(Debug, Error)]
 pub enum VfioError {
     #[error("Kernel VFIO error: {0}")]
@@ -1893,16 +1935,25 @@ impl Migratable for VfioPciDevice {}
 pub struct VfioDmaMapping<M: GuestAddressSpace> {
     container: Arc<VfioContainer>,
     memory: Arc<M>,
+    mmio_regions: Arc<Mutex<Vec<MmioRegion>>>,
 }
 
 impl<M: GuestAddressSpace> VfioDmaMapping<M> {
     /// Create a DmaMapping object.
-    ///
     /// # Parameters
     /// * `container`: VFIO container object.
-    /// * `memory·: guest memory to mmap.
+    /// * `memory`: guest memory to mmap.
-    pub fn new(container: Arc<VfioContainer>, memory: Arc<M>) -> Self {
+    /// * `mmio_regions`: mmio_regions to mmap.
-        VfioDmaMapping { container, memory }
+    pub fn new(
+        container: Arc<VfioContainer>,
+        memory: Arc<M>,
+        mmio_regions: Arc<Mutex<Vec<MmioRegion>>>,
+    ) -> Self {
+        VfioDmaMapping {
+            container,
+            memory,
+            mmio_regions,
+        }
     }
 }
 
@@ -1911,14 +1962,21 @@ impl<M: GuestAddressSpace + Sync + Send> ExternalDmaMapping for VfioDmaMapping<M
         let mem = self.memory.memory();
         let guest_addr = GuestAddress(gpa);
         let user_addr = if mem.check_range(guest_addr, size as usize) {
-            mem.get_host_address(guest_addr).unwrap() as u64
+            match mem.get_host_address(guest_addr) {
+                Ok(t) => t as u64,
+                Err(e) => {
+                    return Err(io::Error::new(
+                        io::ErrorKind::Other,
+                        format!("unable to retrieve user address for gpa 0x{gpa:x} from guest memory region: {e}")
+                    ));
+                }
+            }
+        } else if self.mmio_regions.lock().unwrap().check_range(gpa, size) {
+            self.mmio_regions.lock().unwrap().find_user_address(gpa)?
         } else {
             return Err(io::Error::new(
                 io::ErrorKind::Other,
-                format!(
+                format!("failed to locate guest address 0x{gpa:x} in guest memory"),
-                    "failed to convert guest address 0x{gpa:x} into \
-                     host user virtual address"
-                ),
             ));
         };
 

vmm/src/device_manager.rs

@@ -58,7 +58,7 @@ use libc::{
     O_TMPFILE, PROT_READ, PROT_WRITE, TCSANOW,
 };
 use pci::{
-    DeviceRelocation, PciBarRegionType, PciBdf, PciDevice, VfioDmaMapping,
+    DeviceRelocation, MmioRegion, PciBarRegionType, PciBdf, PciDevice, VfioDmaMapping,
     VfioPciDevice, VfioUserDmaMapping, VfioUserPciDevice, VfioUserPciDeviceError,
 };
 use rate_limiter::group::RateLimiterGroup;
@@ -965,6 +965,8 @@ pub struct DeviceManager {
     snapshot: Option<Snapshot>,
 
     rate_limit_groups: HashMap<String, Arc<RateLimiterGroup>>,
+
+    mmio_regions: Arc<Mutex<Vec<MmioRegion>>>,
 }
 
 impl DeviceManager {
@@ -1195,6 +1197,7 @@ impl DeviceManager {
             acpi_platform_addresses: AcpiPlatformAddresses::default(),
             snapshot,
             rate_limit_groups,
+            mmio_regions: Arc::new(Mutex::new(Vec::new())),
         };
 
         let device_manager = Arc::new(Mutex::new(device_manager));
@@ -3423,6 +3426,7 @@ impl DeviceManager {
             let vfio_mapping = Arc::new(VfioDmaMapping::new(
                 Arc::clone(&vfio_container),
                 Arc::new(self.memory_manager.lock().unwrap().guest_memory()),
+                Arc::clone(&self.mmio_regions),
             ));
 
             if let Some(iommu) = &self.iommu_device {
@@ -3467,6 +3471,7 @@ impl DeviceManager {
             let vfio_mapping = Arc::new(VfioDmaMapping::new(
                 Arc::clone(&vfio_container),
                 Arc::new(self.memory_manager.lock().unwrap().guest_memory()),
+                Arc::clone(&self.mmio_regions),
             ));
 
             for virtio_mem_device in self.virtio_mem_devices.iter() {
@@ -3529,6 +3534,10 @@ impl DeviceManager {
             .map_mmio_regions()
             .map_err(DeviceManagerError::VfioMapRegion)?;
 
+        for mmio_region in vfio_pci_device.lock().unwrap().mmio_regions() {
+            self.mmio_regions.lock().unwrap().push(mmio_region);
+        }
+
         let mut node = device_node!(vfio_name, vfio_pci_device);
 
         // Update the device tree with correct resource information.
@@ -4200,12 +4209,21 @@ impl DeviceManager {
 
         let (pci_device, bus_device, virtio_device, remove_dma_handler) = match pci_device_handle {
             // No need to remove any virtio-mem mapping here as the container outlives all devices
-            PciDeviceHandle::Vfio(vfio_pci_device) => (
+            PciDeviceHandle::Vfio(vfio_pci_device) => {
-                Arc::clone(&vfio_pci_device) as Arc<Mutex<dyn PciDevice>>,
+                for mmio_region in vfio_pci_device.lock().unwrap().mmio_regions() {
-                Arc::clone(&vfio_pci_device) as Arc<Mutex<dyn BusDevice>>,
+                    self.mmio_regions
-                None as Option<Arc<Mutex<dyn virtio_devices::VirtioDevice>>>,
+                        .lock()
-                false,
+                        .unwrap()
-            ),
+                        .retain(|x| x.start != mmio_region.start)
+                }
+
+                (
+                    Arc::clone(&vfio_pci_device) as Arc<Mutex<dyn PciDevice>>,
+                    Arc::clone(&vfio_pci_device) as Arc<Mutex<dyn BusDevice>>,
+                    None as Option<Arc<Mutex<dyn virtio_devices::VirtioDevice>>>,
+                    false,
+                )
+            }
             PciDeviceHandle::Virtio(virtio_pci_device) => {
                 let dev = virtio_pci_device.lock().unwrap();
                 let bar_addr = dev.config_bar_addr();