vmm: Use new Resource type PciBar

Instead of defining some very generic resources as PioAddressRange or
MmioAddressRange for each PCI BAR, let's move to the new Resource type
PciBar in order to make things clearer. This allows the code for being
more readable, but also removes the need for hard assumptions about the
MMIO and PIO ranges. PioAddressRange and MmioAddressRange types can be
used to describe everything except PCI BARs. BARs are very special as
they can be relocated and have special information we want to carry
along with them.

Signed-off-by: Sebastien Boeuf <sebastien.boeuf@intel.com>

pci/src/configuration.rs

@@ -9,6 +9,7 @@ use std::fmt::{self, Display};
 use std::sync::{Arc, Mutex};
 use versionize::{VersionMap, Versionize, VersionizeError, VersionizeResult};
 use versionize_derive::Versionize;
+use vm_device::PciBarType;
 use vm_migration::{MigratableError, Pausable, Snapshot, Snapshottable, VersionMapped};
 
 // The number of 32bit registers in the config space, 4096 bytes.
@@ -327,12 +328,43 @@ pub enum PciBarRegionType {
     Memory64BitRegion = 0x04,
 }
 
+impl From<PciBarType> for PciBarRegionType {
+    fn from(type_: PciBarType) -> Self {
+        match type_ {
+            PciBarType::Io => PciBarRegionType::IoRegion,
+            PciBarType::Mmio32 => PciBarRegionType::Memory32BitRegion,
+            PciBarType::Mmio64 => PciBarRegionType::Memory64BitRegion,
+        }
+    }
+}
+
+#[allow(clippy::from_over_into)]
+impl Into<PciBarType> for PciBarRegionType {
+    fn into(self) -> PciBarType {
+        match self {
+            PciBarRegionType::IoRegion => PciBarType::Io,
+            PciBarRegionType::Memory32BitRegion => PciBarType::Mmio32,
+            PciBarRegionType::Memory64BitRegion => PciBarType::Mmio64,
+        }
+    }
+}
+
 #[derive(Copy, Clone)]
 pub enum PciBarPrefetchable {
     NotPrefetchable = 0,
     Prefetchable = 0x08,
 }
 
+#[allow(clippy::from_over_into)]
+impl Into<bool> for PciBarPrefetchable {
+    fn into(self) -> bool {
+        match self {
+            PciBarPrefetchable::NotPrefetchable => false,
+            PciBarPrefetchable::Prefetchable => true,
+        }
+    }
+}
+
 #[derive(Copy, Clone)]
 pub struct PciBarConfiguration {
     addr: u64,
@@ -982,6 +1014,10 @@ impl PciBarConfiguration {
     pub fn region_type(&self) -> PciBarRegionType {
         self.region_type
     }
+
+    pub fn prefetchable(&self) -> PciBarPrefetchable {
+        self.prefetchable
+    }
 }
 
 #[cfg(test)]

pci/src/device.rs

@@ -19,10 +19,8 @@ pub enum Error {
     IoAllocationFailed(u64),
     /// Registering an IO BAR failed.
     IoRegistrationFailed(u64, configuration::Error),
-    /// Not enough resources
+    /// Expected resource not found.
     MissingResource,
-    /// Invalid type of resource
-    InvalidResourceType,
 }
 pub type Result<T> = std::result::Result<T, Error>;
 
@@ -38,8 +36,7 @@ impl Display for Error {
             IoRegistrationFailed(addr, e) => {
                 write!(f, "failed to register an IO BAR, addr={} err={}", addr, e)
             }
-            MissingResource => write!(f, "Missing resource"),
+            MissingResource => write!(f, "failed to find expected resource"),
-            InvalidResourceType => write!(f, "Invalid type of resource"),
         }
     }
 }

pci/src/vfio.rs

@@ -376,18 +376,17 @@ impl VfioCommon {
             let region_size: u64;
             let bar_addr: GuestAddress;
 
-            let restored_bar_addr = if let Some(resources) = &resources {
+            let mut restored_bar_addr = None;
-                match resources
+            if let Some(resources) = &resources {
-                    .get(bars.len())
+                for resource in resources {
-                    .ok_or(PciDeviceError::MissingResource)?
+                    if let Resource::PciBar { index, base, .. } = resource {
-                {
+                        if *index == bar_id as usize {
-                    Resource::MmioAddressRange { base, .. } => Some(GuestAddress(*base)),
+                            restored_bar_addr = Some(GuestAddress(*base));
-                    Resource::PioAddressRange { base, .. } => Some(GuestAddress(*base as u64)),
+                            break;
-                    _ => return Err(PciDeviceError::InvalidResourceType),
+                        }
+                    }
+                }
             }
-            } else {
-                None
-            };
 
             let bar_offset = if bar_id == VFIO_PCI_ROM_REGION_INDEX {
                 (PCI_ROM_EXP_BAR_INDEX * 4) as u32

virtio-devices/src/transport/pci_device.rs

@@ -846,17 +846,22 @@ impl PciDevice for VirtioPciDevice {
         let mut bars = Vec::new();
         let device_clone = self.device.clone();
         let device = device_clone.lock().unwrap();
-        let settings_bar_addr = if let Some(resources) = &resources {
+
-            if resources.is_empty() {
+        let mut settings_bar_addr = None;
+        if let Some(resources) = &resources {
+            for resource in resources {
+                if let Resource::PciBar { index, base, .. } = resource {
+                    if *index == VIRTIO_COMMON_BAR_INDEX {
+                        settings_bar_addr = Some(GuestAddress(*base));
+                        break;
+                    }
+                }
+            }
+            // Error out if no resource was matching the BAR id.
+            if settings_bar_addr.is_none() {
                 return Err(PciDeviceError::MissingResource);
             }
-            match resources[VIRTIO_COMMON_BAR_INDEX] {
-                Resource::MmioAddressRange { base, .. } => Some(GuestAddress(base)),
-                _ => return Err(PciDeviceError::InvalidResourceType),
         }
-        } else {
-            None
-        };
 
         // Allocate the virtio-pci capability BAR.
         // See http://docs.oasis-open.org/virtio/virtio/v1.0/cs04/virtio-v1.0-cs04.html#x1-740004

vm-device/src/lib.rs

@@ -23,6 +23,13 @@ pub enum MsiIrqType {
     GenericMsi,
 }
 
+#[derive(Copy, Clone, PartialEq, Serialize, Deserialize, Debug)]
+pub enum PciBarType {
+    Io,
+    Mmio32,
+    Mmio64,
+}
+
 /// Enumeration for device resources.
 #[allow(missing_docs)]
 #[derive(Clone, Debug, Serialize, Deserialize)]
@@ -31,6 +38,14 @@ pub enum Resource {
     PioAddressRange { base: u16, size: u16 },
     /// Memory Mapped IO address range.
     MmioAddressRange { base: u64, size: u64 },
+    /// PCI BAR
+    PciBar {
+        index: usize,
+        base: u64,
+        size: u64,
+        type_: PciBarType,
+        prefetchable: bool,
+    },
     /// Legacy IRQ number.
     LegacyIrq(u32),
     /// Message Signaled Interrupt

vmm/src/device_manager.rs

@@ -665,28 +665,14 @@ impl DeviceRelocation for AddressManager {
             if let Some(node) = self.device_tree.lock().unwrap().get_mut(&id) {
                 let mut resource_updated = false;
                 for resource in node.resources.iter_mut() {
-                    match region_type {
+                    if let Resource::PciBar { base, type_, .. } = resource {
-                        PciBarRegionType::IoRegion => {
+                        if PciBarRegionType::from(*type_) == region_type && *base == old_base {
-                            if let Resource::PioAddressRange { base, .. } = resource {
-                                if *base as u64 == old_base {
-                                    *base = new_base as u16;
-                                    resource_updated = true;
-                                    break;
-                                }
-                            }
-                        }
-                        PciBarRegionType::Memory32BitRegion
-                        | PciBarRegionType::Memory64BitRegion => {
-                            if let Resource::MmioAddressRange { base, .. } = resource {
-                                if *base == old_base {
                             *base = new_base;
                             resource_updated = true;
                             break;
                         }
                     }
                 }
-                    }
-                }
 
                 if !resource_updated {
                     return Err(io::Error::new(
@@ -3222,13 +3208,6 @@ impl DeviceManager {
 
         let mut node = device_node!(vfio_name);
 
-        for region in vfio_pci_device.lock().unwrap().mmio_regions() {
-            node.resources.push(Resource::MmioAddressRange {
-                base: region.start.0,
-                size: region.length as u64,
-            });
-        }
-
         // Update the device tree with correct resource information.
         node.resources = new_resources;
         node.pci_bdf = Some(pci_device_bdf);
@@ -3286,18 +3265,13 @@ impl DeviceManager {
 
         let mut new_resources = Vec::new();
         for bar in bars {
-            match bar.region_type() {
+            new_resources.push(Resource::PciBar {
-                PciBarRegionType::IoRegion => new_resources.push(Resource::PioAddressRange {
+                index: bar.idx(),
-                    base: bar.addr() as u16,
-                    size: bar.size() as u16,
-                }),
-                PciBarRegionType::Memory32BitRegion | PciBarRegionType::Memory64BitRegion => {
-                    new_resources.push(Resource::MmioAddressRange {
                 base: bar.addr(),
-                        size: bar.size() as u64,
+                size: bar.size(),
-                    })
+                type_: bar.region_type().into(),
-                }
+                prefetchable: bar.prefetchable().into(),
-            }
+            });
         }
 
         Ok(new_resources)