msix: Add gsi_msi_routes to MsixConfig

Because MsixConfig will be responsible for updating KVM GSI routes at
some point, it is necessary that it can access the list of routes
contained by gsi_msi_routes.

Signed-off-by: Sebastien Boeuf <sebastien.boeuf@intel.com>

Cargo.lock

@@ -1110,6 +1110,7 @@ dependencies = [
  "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "devices 0.1.0",
  "epoll 4.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "kvm-bindings 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "kvm-ioctls 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)",

pci/src/msix.rs

@@ -11,7 +11,10 @@ use std::sync::Arc;
 use crate::device::InterruptParameters;
 use crate::{InterruptDelivery, InterruptRoute, PciCapability, PciCapabilityID};
 use byteorder::{ByteOrder, LittleEndian};
+use kvm_bindings::kvm_irq_routing_entry;
 use kvm_ioctls::VmFd;
+use std::collections::HashMap;
+use std::sync::Mutex;
 use vm_allocator::SystemAllocator;
 use vm_memory::ByteValued;
 
@@ -55,13 +58,19 @@ pub struct MsixConfig {
     pub pba_entries: Vec<u64>,
     pub irq_routes: Vec<InterruptRoute>,
     _vm_fd: Arc<VmFd>,
+    _gsi_msi_routes: Arc<Mutex<HashMap<u32, kvm_irq_routing_entry>>>,
     interrupt_cb: Option<Arc<InterruptDelivery>>,
     masked: bool,
     enabled: bool,
 }
 
 impl MsixConfig {
-    pub fn new(msix_vectors: u16, allocator: &mut SystemAllocator, vm_fd: Arc<VmFd>) -> Self {
+    pub fn new(
+        msix_vectors: u16,
+        allocator: &mut SystemAllocator,
+        vm_fd: Arc<VmFd>,
+        gsi_msi_routes: Arc<Mutex<HashMap<u32, kvm_irq_routing_entry>>>,
+    ) -> Self {
         assert!(msix_vectors <= MAX_MSIX_VECTORS_PER_DEVICE);
 
         let mut table_entries: Vec<MsixTableEntry> = Vec::new();
@@ -80,6 +89,7 @@ impl MsixConfig {
             pba_entries,
             irq_routes,
             _vm_fd: vm_fd,
+            _gsi_msi_routes: gsi_msi_routes,
             interrupt_cb: None,
             masked: false,
             enabled: false,

vfio/src/vfio_pci.rs

@@ -412,7 +412,12 @@ impl VfioPciDevice {
             table,
             pba,
         };
-        let msix_config = MsixConfig::new(msix_cap.table_size(), allocator, self.vm_fd.clone());
+        let msix_config = MsixConfig::new(
+            msix_cap.table_size(),
+            allocator,
+            self.vm_fd.clone(),
+            self.gsi_msi_routes.clone(),
+        );
 
         self.interrupt.msix = Some(VfioMsix {
             bar: msix_config,

vm-virtio/Cargo.toml

@@ -14,6 +14,7 @@ arc-swap = ">=0.4.4"
 byteorder = "1.3.2"
 devices = { path = "../devices" }
 epoll = ">=4.0.1"
+kvm-bindings = "0.2.0"
 kvm-ioctls = "0.4.0"
 libc = "0.2.60"
 log = "0.4.8"

vm-virtio/src/transport/pci_device.rs

@@ -13,8 +13,17 @@ extern crate vm_allocator;
 extern crate vm_memory;
 extern crate vmm_sys_util;
 
+use super::VirtioPciCommonConfig;
+use crate::transport::VirtioTransport;
+use crate::{
+    Queue, VirtioDevice, VirtioDeviceType, VirtioInterrupt, VirtioInterruptType,
+    VirtioIommuRemapping, DEVICE_ACKNOWLEDGE, DEVICE_DRIVER, DEVICE_DRIVER_OK, DEVICE_FAILED,
+    DEVICE_FEATURES_OK, DEVICE_INIT, INTERRUPT_STATUS_CONFIG_CHANGED, INTERRUPT_STATUS_USED_RING,
+    VIRTIO_MSI_NO_VECTOR,
+};
 use arc_swap::ArcSwap;
 use devices::BusDevice;
+use kvm_bindings::kvm_irq_routing_entry;
 use kvm_ioctls::VmFd;
 use libc::EFD_NONBLOCK;
 use pci::{
@@ -24,6 +33,7 @@ use pci::{
     PciMassStorageSubclass, PciNetworkControllerSubclass, PciSubclass,
 };
 use std::any::Any;
+use std::collections::HashMap;
 use std::result;
 use std::sync::atomic::{AtomicU16, AtomicUsize, Ordering};
 use std::sync::{Arc, Mutex};
@@ -32,15 +42,6 @@ use vm_device::{Migratable, MigratableError, Pausable, Snapshotable};
 use vm_memory::{Address, ByteValued, GuestAddress, GuestMemoryMmap, GuestUsize, Le32};
 use vmm_sys_util::{errno::Result, eventfd::EventFd};
 
-use super::VirtioPciCommonConfig;
-use crate::transport::VirtioTransport;
-use crate::{
-    Queue, VirtioDevice, VirtioDeviceType, VirtioInterrupt, VirtioInterruptType,
-    VirtioIommuRemapping, DEVICE_ACKNOWLEDGE, DEVICE_DRIVER, DEVICE_DRIVER_OK, DEVICE_FAILED,
-    DEVICE_FEATURES_OK, DEVICE_INIT, INTERRUPT_STATUS_CONFIG_CHANGED, INTERRUPT_STATUS_USED_RING,
-    VIRTIO_MSI_NO_VECTOR,
-};
-
 #[allow(clippy::enum_variant_names)]
 enum PciCapabilityType {
     CommonConfig = 1,
@@ -259,6 +260,7 @@ impl VirtioPciDevice {
         iommu_mapping_cb: Option<Arc<VirtioIommuRemapping>>,
         allocator: &mut SystemAllocator,
         vm_fd: &Arc<VmFd>,
+        gsi_msi_routes: Arc<Mutex<HashMap<u32, kvm_irq_routing_entry>>>,
     ) -> Result<Self> {
         let device_clone = device.clone();
         let locked_device = device_clone.lock().unwrap();
@@ -283,6 +285,7 @@ impl VirtioPciDevice {
                 msix_num,
                 allocator,
                 vm_fd.clone(),
+                gsi_msi_routes,
             )));
             let msix_config_clone = msix_config.clone();
             (Some(msix_config), Some(msix_config_clone))

vmm/src/device_manager.rs

@@ -577,6 +577,7 @@ impl DeviceManager {
                     &mut pci_bus,
                     mapping,
                     migratable_devices,
+                    &gsi_msi_routes,
                 )?;
 
                 if let Some(dev_id) = virtio_iommu_attach_dev {
@@ -590,7 +591,7 @@ impl DeviceManager {
                 &mut pci_bus,
                 memory_manager,
                 &mut iommu_device,
-                gsi_msi_routes,
+                &gsi_msi_routes,
             )?;
 
             iommu_attached_devices.append(&mut vfio_iommu_device_ids);
@@ -614,6 +615,7 @@ impl DeviceManager {
                     &mut pci_bus,
                     &None,
                     migratable_devices,
+                    &gsi_msi_routes,
                 )?;
 
                 *virt_iommu = Some((iommu_id, iommu_attached_devices));
@@ -1350,7 +1352,7 @@ impl DeviceManager {
         pci: &mut PciBus,
         memory_manager: &Arc<Mutex<MemoryManager>>,
         iommu_device: &mut Option<vm_virtio::Iommu>,
-        gsi_msi_routes: Arc<Mutex<HashMap<u32, kvm_irq_routing_entry>>>,
+        gsi_msi_routes: &Arc<Mutex<HashMap<u32, kvm_irq_routing_entry>>>,
     ) -> DeviceManagerResult<Vec<u32>> {
         let mut mem_slot = memory_manager.lock().unwrap().allocate_kvm_memory_slot();
         let mut iommu_attached_device_ids = Vec::new();
@@ -1432,6 +1434,7 @@ impl DeviceManager {
         pci: &mut PciBus,
         iommu_mapping: &Option<Arc<IommuMapping>>,
         migratable_devices: &mut Vec<Arc<Mutex<dyn Migratable>>>,
+        gsi_msi_routes: &Arc<Mutex<HashMap<u32, kvm_irq_routing_entry>>>,
     ) -> DeviceManagerResult<Option<u32>> {
         // Allows support for one MSI-X vector per queue. It also adds 1
         // as we need to take into account the dedicated vector to notify
@@ -1474,6 +1477,7 @@ impl DeviceManager {
             iommu_mapping_cb,
             &mut allocator,
             vm_fd,
+            gsi_msi_routes.clone(),
         )
         .map_err(DeviceManagerError::VirtioDevice)?;