msix: Add SystemAllocator to MsixConfig

The point here is to let MsixConfig take care of the GSI allocation,
which means the SystemAllocator must be passed from the vmm crate all
the way down to the pci crate.

Once this is done, the GSI allocation and irq_fd creation is performed
by MsixConfig directly.

Signed-off-by: Sebastien Boeuf <sebastien.boeuf@intel.com>

pci/src/msix.rs

@@ -9,8 +9,9 @@ extern crate vm_memory;
 use std::sync::Arc;
 
 use crate::device::InterruptParameters;
-use crate::{InterruptDelivery, PciCapability, PciCapabilityID};
+use crate::{InterruptDelivery, InterruptRoute, PciCapability, PciCapabilityID};
 use byteorder::{ByteOrder, LittleEndian};
+use vm_allocator::SystemAllocator;
 use vm_memory::ByteValued;
 
 const MAX_MSIX_VECTORS_PER_DEVICE: u16 = 2048;
@@ -51,13 +52,14 @@ impl Default for MsixTableEntry {
 pub struct MsixConfig {
     pub table_entries: Vec<MsixTableEntry>,
     pub pba_entries: Vec<u64>,
+    pub irq_routes: Vec<InterruptRoute>,
     interrupt_cb: Option<Arc<InterruptDelivery>>,
     masked: bool,
     enabled: bool,
 }
 
 impl MsixConfig {
-    pub fn new(msix_vectors: u16) -> Self {
+    pub fn new(msix_vectors: u16, allocator: &mut SystemAllocator) -> Self {
         assert!(msix_vectors <= MAX_MSIX_VECTORS_PER_DEVICE);
 
         let mut table_entries: Vec<MsixTableEntry> = Vec::new();
@@ -66,9 +68,15 @@ impl MsixConfig {
         let num_pba_entries: usize = ((msix_vectors as usize) / BITS_PER_PBA_ENTRY) + 1;
         pba_entries.resize_with(num_pba_entries, Default::default);
 
+        let mut irq_routes: Vec<InterruptRoute> = Vec::new();
+        for _ in 0..msix_vectors {
+            irq_routes.push(InterruptRoute::new(allocator).unwrap());
+        }
+
         MsixConfig {
             table_entries,
             pba_entries,
+            irq_routes,
             interrupt_cb: None,
             masked: false,
             enabled: false,

vfio/src/vfio_pci.rs

@@ -340,7 +340,7 @@ impl VfioPciDevice {
             gsi_msi_routes,
         };
 
-        vfio_pci_device.parse_capabilities();
+        vfio_pci_device.parse_capabilities(allocator);
 
         // Allocate temporary interrupt routes for now.
         // The MSI vectors will be filled when the guest driver programs the device.
@@ -394,7 +394,7 @@ impl VfioPciDevice {
             .map_err(VfioPciError::SetGsiRouting)
     }
 
-    fn parse_msix_capabilities(&mut self, cap: u8) {
+    fn parse_msix_capabilities(&mut self, cap: u8, allocator: &mut SystemAllocator) {
         let msg_ctl = self
             .vfio_pci_configuration
             .read_config_word((cap + 2).into());
@@ -412,7 +412,7 @@ impl VfioPciDevice {
             table,
             pba,
         };
-        let msix_config = MsixConfig::new(msix_cap.table_size());
+        let msix_config = MsixConfig::new(msix_cap.table_size(), allocator);
 
         self.interrupt.msix = Some(VfioMsix {
             bar: msix_config,
@@ -435,7 +435,7 @@ impl VfioPciDevice {
         });
     }
 
-    fn parse_capabilities(&mut self) {
+    fn parse_capabilities(&mut self, allocator: &mut SystemAllocator) {
         let mut cap_next = self
             .vfio_pci_configuration
             .read_config_byte(PCI_CONFIG_CAPABILITY_OFFSET);
@@ -450,7 +450,7 @@ impl VfioPciDevice {
                     self.parse_msi_capabilities(cap_next);
                 }
                 PciCapabilityID::MSIX => {
-                    self.parse_msix_capabilities(cap_next);
+                    self.parse_msix_capabilities(cap_next, allocator);
                 }
                 _ => {}
             };

vm-virtio/src/transport/pci_device.rs

@@ -256,6 +256,7 @@ impl VirtioPciDevice {
         device: Arc<Mutex<dyn VirtioDevice>>,
         msix_num: u16,
         iommu_mapping_cb: Option<Arc<VirtioIommuRemapping>>,
+        allocator: &mut SystemAllocator,
     ) -> Result<Self> {
         let device_clone = device.clone();
         let locked_device = device_clone.lock().unwrap();
@@ -276,7 +277,7 @@ impl VirtioPciDevice {
         let pci_device_id = VIRTIO_PCI_DEVICE_ID_BASE + locked_device.device_type() as u16;
 
         let (msix_config, msix_config_clone) = if msix_num > 0 {
-            let msix_config = Arc::new(Mutex::new(MsixConfig::new(msix_num)));
+            let msix_config = Arc::new(Mutex::new(MsixConfig::new(msix_num, allocator)));
             let msix_config_clone = msix_config.clone();
             (Some(msix_config), Some(msix_config_clone))
         } else {

vmm/src/device_manager.rs

@@ -1465,12 +1465,17 @@ impl DeviceManager {
                 None
             };
 
-        let mut virtio_pci_device =
-            VirtioPciDevice::new(memory.clone(), virtio_device, msix_num, iommu_mapping_cb)
-                .map_err(DeviceManagerError::VirtioDevice)?;
-
         let mut allocator = address_manager.allocator.lock().unwrap();
 
+        let mut virtio_pci_device = VirtioPciDevice::new(
+            memory.clone(),
+            virtio_device,
+            msix_num,
+            iommu_mapping_cb,
+            &mut allocator,
+        )
+        .map_err(DeviceManagerError::VirtioDevice)?;
+
         let bars = virtio_pci_device
             .allocate_bars(&mut allocator)
             .map_err(DeviceManagerError::AllocateBars)?;