From 376db311070771f9db1437814df83da63aa0a0d3 Mon Sep 17 00:00:00 2001 From: Sebastien Boeuf Date: Wed, 6 May 2020 17:50:56 +0200 Subject: [PATCH] pci: Implement Snapshottable trait for MsixConfig In order to restore devices relying on MSI-X, the MsixConfig structure must be restored with the correct values. Additionally, the KVM routes must be restored so that interrupts can be delivered through KVM the way they were configured before the snapshot was taken. Signed-off-by: Sebastien Boeuf --- Cargo.lock | 5 +++ pci/Cargo.toml | 7 ++- pci/src/lib.rs | 4 ++ pci/src/msix.rs | 111 ++++++++++++++++++++++++++++++++++++++++++++++-- 4 files changed, 123 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8e89b3ed1..f11742711 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -667,13 +667,18 @@ dependencies = [ name = "pci" version = "0.1.0" dependencies = [ + "anyhow", "byteorder", "devices", "libc", "log 0.4.8", + "serde", + "serde_derive", + "serde_json", "vm-allocator", "vm-device", "vm-memory", + "vm-migration", ] [[package]] diff --git a/pci/Cargo.toml b/pci/Cargo.toml index ed4d6c2ac..dde9c14f1 100644 --- a/pci/Cargo.toml +++ b/pci/Cargo.toml @@ -5,10 +5,15 @@ authors = ["Samuel Ortiz "] edition = "2018" [dependencies] -vm-allocator = { path = "../vm-allocator" } +anyhow = "1.0" byteorder = "1.3.4" devices = { path = "../devices" } libc = "0.2.69" log = "0.4.8" +serde = {version = ">=1.0.27", features = ["rc"] } +serde_derive = ">=1.0.27" +serde_json = ">=1.0.9" +vm-allocator = { path = "../vm-allocator" } vm-device = { path = "../vm-device" } vm-memory = "0.2.0" +vm-migration = { path = "../vm-migration" } diff --git a/pci/src/lib.rs b/pci/src/lib.rs index 5e7ad586f..2658fea18 100644 --- a/pci/src/lib.rs +++ b/pci/src/lib.rs @@ -6,6 +6,10 @@ #[macro_use] extern crate log; extern crate devices; +extern crate serde; +#[macro_use] +extern crate serde_derive; +extern crate serde_json; extern crate vm_memory; mod bus; diff --git a/pci/src/msix.rs b/pci/src/msix.rs index a83218dca..f37a7c138 100644 --- a/pci/src/msix.rs +++ b/pci/src/msix.rs @@ -6,14 +6,17 @@ extern crate byteorder; extern crate vm_memory; -use std::sync::Arc; - use crate::{PciCapability, PciCapabilityID}; +use anyhow::anyhow; use byteorder::{ByteOrder, LittleEndian}; +use std::io; +use std::result; +use std::sync::Arc; use vm_device::interrupt::{ InterruptIndex, InterruptSourceConfig, InterruptSourceGroup, MsiIrqSourceConfig, }; use vm_memory::ByteValued; +use vm_migration::{MigratableError, Pausable, Snapshot, SnapshotDataSection, Snapshottable}; const MAX_MSIX_VECTORS_PER_DEVICE: u16 = 2048; const MSIX_TABLE_ENTRIES_MODULO: u64 = 16; @@ -25,7 +28,15 @@ const FUNCTION_MASK_MASK: u16 = (1 << FUNCTION_MASK_BIT) as u16; const MSIX_ENABLE_MASK: u16 = (1 << MSIX_ENABLE_BIT) as u16; pub const MSIX_TABLE_ENTRY_SIZE: usize = 16; -#[derive(Debug, Clone)] +#[derive(Debug)] +enum Error { + /// Failed enabling the interrupt route. + EnableInterruptRoute(io::Error), + /// Failed updating the interrupt route. + UpdateInterruptRoute(io::Error), +} + +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct MsixTableEntry { pub msg_addr_lo: u32, pub msg_addr_hi: u32, @@ -50,6 +61,14 @@ impl Default for MsixTableEntry { } } +#[derive(Serialize, Deserialize)] +struct MsixConfigState { + table_entries: Vec, + pba_entries: Vec, + masked: bool, + enabled: bool, +} + pub struct MsixConfig { pub table_entries: Vec, pub pba_entries: Vec, @@ -80,6 +99,46 @@ impl MsixConfig { } } + fn state(&self) -> MsixConfigState { + MsixConfigState { + table_entries: self.table_entries.clone(), + pba_entries: self.pba_entries.clone(), + masked: self.masked, + enabled: self.enabled, + } + } + + fn set_state(&mut self, state: &MsixConfigState) -> result::Result<(), Error> { + self.table_entries = state.table_entries.clone(); + self.pba_entries = state.pba_entries.clone(); + self.masked = state.masked; + self.enabled = state.enabled; + + if self.enabled && !self.masked { + for (idx, table_entry) in self.table_entries.iter().enumerate() { + if table_entry.masked() { + continue; + } + + let config = MsiIrqSourceConfig { + high_addr: table_entry.msg_addr_hi, + low_addr: table_entry.msg_addr_lo, + data: table_entry.msg_data, + }; + + self.interrupt_source_group + .update(idx as InterruptIndex, InterruptSourceConfig::MsiIrq(config)) + .map_err(Error::UpdateInterruptRoute)?; + + self.interrupt_source_group + .enable() + .map_err(Error::EnableInterruptRoute)?; + } + } + + Ok(()) + } + pub fn masked(&self) -> bool { self.masked } @@ -365,6 +424,52 @@ impl MsixConfig { } } +impl Pausable for MsixConfig {} + +impl Snapshottable for MsixConfig { + fn id(&self) -> String { + String::from("msix_config") + } + + fn snapshot(&self) -> std::result::Result { + let snapshot = + serde_json::to_vec(&self.state()).map_err(|e| MigratableError::Snapshot(e.into()))?; + + let mut msix_snapshot = Snapshot::new(self.id().as_str()); + msix_snapshot.add_data_section(SnapshotDataSection { + id: format!("{}-section", self.id()), + snapshot, + }); + + Ok(msix_snapshot) + } + + fn restore(&mut self, snapshot: Snapshot) -> std::result::Result<(), MigratableError> { + if let Some(msix_section) = snapshot + .snapshot_data + .get(&format!("{}-section", self.id())) + { + let msix_state = match serde_json::from_slice(&msix_section.snapshot) { + Ok(state) => state, + Err(error) => { + return Err(MigratableError::Restore(anyhow!( + "Could not deserialize MSI-X {}", + error + ))) + } + }; + + return self.set_state(&msix_state).map_err(|e| { + MigratableError::Restore(anyhow!("Could not restore MSI-X state {:?}", e)) + }); + } + + Err(MigratableError::Restore(anyhow!( + "Could not find MSI-X snapshot section" + ))) + } +} + #[allow(dead_code)] #[repr(packed)] #[derive(Clone, Copy, Default)]