msi: Set KVM routes from MsiConfig instead of VFIO

Now that MsiConfig has access to both KVM VmFd and the list of GSI
routes, the update of the KVM GSI routes can be directly done from
MsiConfig instead of specifically from the vfio-pci implementation.

By moving the KVM GSI routes update at the MsiConfig level, any PCI
device such as vfio-pci, virtio-pci, or any other emulated PCI device
can benefit from it, without having to implement it on their own.

Signed-off-by: Sebastien Boeuf <sebastien.boeuf@intel.com>

pci/src/msi.rs

@@ -6,9 +6,9 @@
 extern crate byteorder;
 extern crate vm_memory;
 
-use crate::InterruptRoute;
+use crate::{set_kvm_routes, InterruptRoute};
 use byteorder::{ByteOrder, LittleEndian};
-use kvm_bindings::kvm_irq_routing_entry;
+use kvm_bindings::{kvm_irq_routing_entry, KVM_IRQ_ROUTING_MSI};
 use kvm_ioctls::VmFd;
 use std::collections::HashMap;
 use std::sync::{Arc, Mutex};
@@ -161,8 +161,8 @@ impl MsiCap {
 pub struct MsiConfig {
     pub cap: MsiCap,
     pub irq_routes: Vec<InterruptRoute>,
-    _vm_fd: Arc<VmFd>,
-    _gsi_msi_routes: Arc<Mutex<HashMap<u32, kvm_irq_routing_entry>>>,
+    vm_fd: Arc<VmFd>,
+    gsi_msi_routes: Arc<Mutex<HashMap<u32, kvm_irq_routing_entry>>>,
 }
 
 impl MsiConfig {
@@ -185,8 +185,8 @@ impl MsiConfig {
         MsiConfig {
             cap,
             irq_routes,
-            _vm_fd: vm_fd,
-            _gsi_msi_routes: gsi_msi_routes,
+            vm_fd,
+            gsi_msi_routes,
         }
     }
 
@@ -194,10 +194,6 @@ impl MsiConfig {
         self.cap.enabled()
     }
 
-    pub fn update(&mut self, offset: u64, data: &[u8]) {
-        self.cap.update(offset, data)
-    }
-
     pub fn size(&self) -> u64 {
         self.cap.size()
     }
@@ -209,4 +205,53 @@ impl MsiConfig {
     pub fn vector_masked(&self, vector: usize) -> bool {
         self.cap.vector_masked(vector)
     }
+
+    pub fn update(&mut self, offset: u64, data: &[u8]) {
+        let old_enabled = self.cap.enabled();
+
+        self.cap.update(offset, data);
+
+        let mut gsi_msi_routes = self.gsi_msi_routes.lock().unwrap();
+
+        if self.cap.enabled() {
+            for (idx, route) in self.irq_routes.iter().enumerate() {
+                if !old_enabled {
+                    if let Err(e) = self.irq_routes[idx].enable(&self.vm_fd) {
+                        error!("Failed enabling irq_fd: {:?}", e);
+                    }
+                }
+
+                // Ignore MSI vector if masked.
+                if self.cap.vector_masked(idx) {
+                    continue;
+                }
+
+                let mut entry = kvm_irq_routing_entry {
+                    gsi: route.gsi,
+                    type_: KVM_IRQ_ROUTING_MSI,
+                    ..Default::default()
+                };
+
+                entry.u.msi.address_lo = self.cap.msg_addr_lo;
+                entry.u.msi.address_hi = self.cap.msg_addr_hi;
+                entry.u.msi.data = u32::from(self.cap.msg_data) | (idx as u32);
+
+                gsi_msi_routes.insert(route.gsi, entry);
+            }
+        } else {
+            for route in self.irq_routes.iter() {
+                if old_enabled {
+                    if let Err(e) = route.disable(&self.vm_fd) {
+                        error!("Failed disabling irq_fd: {:?}", e);
+                    }
+                }
+
+                gsi_msi_routes.remove(&route.gsi);
+            }
+        }
+
+        if let Err(e) = set_kvm_routes(&self.vm_fd, &gsi_msi_routes) {
+            error!("Failed updating KVM routes: {:?}", e);
+        }
+    }
 }

vfio/src/vfio_pci.rs

@@ -209,11 +209,13 @@ impl Interrupt {
     }
 }
 
+#[allow(dead_code)]
 struct InterruptRoute {
     gsi: u32,
     irq_fd: EventFd,
 }
 
+#[allow(dead_code)]
 impl InterruptRoute {
     fn new(allocator: &mut SystemAllocator) -> Result<Self> {
         let irq_fd = EventFd::new(libc::EFD_NONBLOCK).map_err(VfioPciError::EventFd)?;
@@ -300,6 +302,7 @@ pub struct VfioPciDevice {
     gsi_msi_routes: Arc<Mutex<HashMap<u32, kvm_irq_routing_entry>>>,
 }
 
+#[allow(dead_code)]
 impl VfioPciDevice {
     /// Constructs a new Vfio Pci device for the given Vfio device
     pub fn new(
@@ -518,35 +521,27 @@ impl VfioPciDevice {
 
     fn update_msi_capabilities(&mut self, offset: u64, data: &[u8]) -> Result<()> {
         match self.interrupt.update_msi(offset, data) {
-            Some(InterruptUpdateAction::EnableMsi) => match self.enable_irq_fds() {
-                Ok(fds) => {
-                    if let Err(e) = self.device.enable_msi(fds) {
+            Some(InterruptUpdateAction::EnableMsi) => {
+                if let Some(msi) = &self.interrupt.msi {
+                    let mut irq_fds: Vec<&EventFd> = Vec::new();
+                    for r in msi.cfg.irq_routes.iter() {
+                        irq_fds.push(&r.irq_fd);
+                    }
+
+                    if let Err(e) = self.device.enable_msi(irq_fds) {
                         warn!("Could not enable MSI: {}", e);
                     }
                 }
-                Err(e) => warn!("Could not get IRQ fds: {}", e),
-            },
+            }
             Some(InterruptUpdateAction::DisableMsi) => {
                 if let Err(e) = self.device.disable_msi() {
                     warn!("Could not disable MSI: {}", e);
                 }
-                if let Err(e) = self.disable_irq_fds() {
-                    warn!("Could not disable MSI: {}", e);
-                }
             }
             _ => {}
         }
 
-        // Update the gsi_msi_routes table now that the MSI cache has been
-        // updated. The point is to always update the table based on latest
-        // changes to the cache, and based on the state of masking flags, the
-        // KVM GSI routes should be configured.
-        if let Some(msi) = &self.interrupt.msi {
-            return self.update_msi_interrupt_routes(&msi);
-        }
-
-        // If the code reach this point, something went wrong.
-        Err(VfioPciError::MsiNotConfigured)
+        Ok(())
     }
 
     fn update_msix_capabilities(&mut self, offset: u64, data: &[u8]) -> Result<()> {