pci: Implement Snapshottable trait for MsixConfig

In order to restore devices relying on MSI-X, the MsixConfig structure
must be restored with the correct values. Additionally, the KVM routes
must be restored so that interrupts can be delivered through KVM the way
they were configured before the snapshot was taken.

Signed-off-by: Sebastien Boeuf <sebastien.boeuf@intel.com>

Cargo.lock

@@ -667,13 +667,18 @@ dependencies = [
 name = "pci"
 version = "0.1.0"
 dependencies = [
+ "anyhow",
  "byteorder",
  "devices",
  "libc",
  "log 0.4.8",
+ "serde",
+ "serde_derive",
+ "serde_json",
  "vm-allocator",
  "vm-device",
  "vm-memory",
+ "vm-migration",
 ]
 
 [[package]]

pci/Cargo.toml

@@ -5,10 +5,15 @@ authors = ["Samuel Ortiz <sameo@linux.intel.com>"]
 edition = "2018"
 
 [dependencies]
-vm-allocator = { path = "../vm-allocator" }
+anyhow = "1.0"
 byteorder = "1.3.4"
 devices = { path = "../devices" }
 libc = "0.2.69"
 log = "0.4.8"
+serde = {version = ">=1.0.27", features = ["rc"] }
+serde_derive = ">=1.0.27"
+serde_json = ">=1.0.9"
+vm-allocator = { path = "../vm-allocator" }
 vm-device = { path = "../vm-device" }
 vm-memory = "0.2.0"
+vm-migration = { path = "../vm-migration" }

pci/src/lib.rs

@@ -6,6 +6,10 @@
 #[macro_use]
 extern crate log;
 extern crate devices;
+extern crate serde;
+#[macro_use]
+extern crate serde_derive;
+extern crate serde_json;
 extern crate vm_memory;
 
 mod bus;

pci/src/msix.rs

@@ -6,14 +6,17 @@
 extern crate byteorder;
 extern crate vm_memory;
 
-use std::sync::Arc;
-
 use crate::{PciCapability, PciCapabilityID};
+use anyhow::anyhow;
 use byteorder::{ByteOrder, LittleEndian};
+use std::io;
+use std::result;
+use std::sync::Arc;
 use vm_device::interrupt::{
     InterruptIndex, InterruptSourceConfig, InterruptSourceGroup, MsiIrqSourceConfig,
 };
 use vm_memory::ByteValued;
+use vm_migration::{MigratableError, Pausable, Snapshot, SnapshotDataSection, Snapshottable};
 
 const MAX_MSIX_VECTORS_PER_DEVICE: u16 = 2048;
 const MSIX_TABLE_ENTRIES_MODULO: u64 = 16;
@@ -25,7 +28,15 @@ const FUNCTION_MASK_MASK: u16 = (1 << FUNCTION_MASK_BIT) as u16;
 const MSIX_ENABLE_MASK: u16 = (1 << MSIX_ENABLE_BIT) as u16;
 pub const MSIX_TABLE_ENTRY_SIZE: usize = 16;
 
-#[derive(Debug, Clone)]
+#[derive(Debug)]
+enum Error {
+    /// Failed enabling the interrupt route.
+    EnableInterruptRoute(io::Error),
+    /// Failed updating the interrupt route.
+    UpdateInterruptRoute(io::Error),
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct MsixTableEntry {
     pub msg_addr_lo: u32,
     pub msg_addr_hi: u32,
@@ -50,6 +61,14 @@ impl Default for MsixTableEntry {
     }
 }
 
+#[derive(Serialize, Deserialize)]
+struct MsixConfigState {
+    table_entries: Vec<MsixTableEntry>,
+    pba_entries: Vec<u64>,
+    masked: bool,
+    enabled: bool,
+}
+
 pub struct MsixConfig {
     pub table_entries: Vec<MsixTableEntry>,
     pub pba_entries: Vec<u64>,
@@ -80,6 +99,46 @@ impl MsixConfig {
         }
     }
 
+    fn state(&self) -> MsixConfigState {
+        MsixConfigState {
+            table_entries: self.table_entries.clone(),
+            pba_entries: self.pba_entries.clone(),
+            masked: self.masked,
+            enabled: self.enabled,
+        }
+    }
+
+    fn set_state(&mut self, state: &MsixConfigState) -> result::Result<(), Error> {
+        self.table_entries = state.table_entries.clone();
+        self.pba_entries = state.pba_entries.clone();
+        self.masked = state.masked;
+        self.enabled = state.enabled;
+
+        if self.enabled && !self.masked {
+            for (idx, table_entry) in self.table_entries.iter().enumerate() {
+                if table_entry.masked() {
+                    continue;
+                }
+
+                let config = MsiIrqSourceConfig {
+                    high_addr: table_entry.msg_addr_hi,
+                    low_addr: table_entry.msg_addr_lo,
+                    data: table_entry.msg_data,
+                };
+
+                self.interrupt_source_group
+                    .update(idx as InterruptIndex, InterruptSourceConfig::MsiIrq(config))
+                    .map_err(Error::UpdateInterruptRoute)?;
+
+                self.interrupt_source_group
+                    .enable()
+                    .map_err(Error::EnableInterruptRoute)?;
+            }
+        }
+
+        Ok(())
+    }
+
     pub fn masked(&self) -> bool {
         self.masked
     }
@@ -365,6 +424,52 @@ impl MsixConfig {
     }
 }
 
+impl Pausable for MsixConfig {}
+
+impl Snapshottable for MsixConfig {
+    fn id(&self) -> String {
+        String::from("msix_config")
+    }
+
+    fn snapshot(&self) -> std::result::Result<Snapshot, MigratableError> {
+        let snapshot =
+            serde_json::to_vec(&self.state()).map_err(|e| MigratableError::Snapshot(e.into()))?;
+
+        let mut msix_snapshot = Snapshot::new(self.id().as_str());
+        msix_snapshot.add_data_section(SnapshotDataSection {
+            id: format!("{}-section", self.id()),
+            snapshot,
+        });
+
+        Ok(msix_snapshot)
+    }
+
+    fn restore(&mut self, snapshot: Snapshot) -> std::result::Result<(), MigratableError> {
+        if let Some(msix_section) = snapshot
+            .snapshot_data
+            .get(&format!("{}-section", self.id()))
+        {
+            let msix_state = match serde_json::from_slice(&msix_section.snapshot) {
+                Ok(state) => state,
+                Err(error) => {
+                    return Err(MigratableError::Restore(anyhow!(
+                        "Could not deserialize MSI-X {}",
+                        error
+                    )))
+                }
+            };
+
+            return self.set_state(&msix_state).map_err(|e| {
+                MigratableError::Restore(anyhow!("Could not restore MSI-X state {:?}", e))
+            });
+        }
+
+        Err(MigratableError::Restore(anyhow!(
+            "Could not find MSI-X snapshot section"
+        )))
+    }
+}
+
 #[allow(dead_code)]
 #[repr(packed)]
 #[derive(Clone, Copy, Default)]