From 1a4b5ecc75c85ceb6e0d2ebda1dbd218592f7141 Mon Sep 17 00:00:00 2001 From: Sebastien Boeuf Date: Thu, 16 Jan 2020 09:47:14 +0100 Subject: [PATCH] msi: Set KVM routes from MsiConfig instead of VFIO Now that MsiConfig has access to both KVM VmFd and the list of GSI routes, the update of the KVM GSI routes can be directly done from MsiConfig instead of specifically from the vfio-pci implementation. By moving the KVM GSI routes update at the MsiConfig level, any PCI device such as vfio-pci, virtio-pci, or any other emulated PCI device can benefit from it, without having to implement it on their own. Signed-off-by: Sebastien Boeuf --- pci/src/msi.rs | 65 +++++++++++++++++++++++++++++++++++++------- vfio/src/vfio_pci.rs | 31 +++++++++------------ 2 files changed, 68 insertions(+), 28 deletions(-) diff --git a/pci/src/msi.rs b/pci/src/msi.rs index b1c5ed12b..fe5e74b0e 100644 --- a/pci/src/msi.rs +++ b/pci/src/msi.rs @@ -6,9 +6,9 @@ extern crate byteorder; extern crate vm_memory; -use crate::InterruptRoute; +use crate::{set_kvm_routes, InterruptRoute}; use byteorder::{ByteOrder, LittleEndian}; -use kvm_bindings::kvm_irq_routing_entry; +use kvm_bindings::{kvm_irq_routing_entry, KVM_IRQ_ROUTING_MSI}; use kvm_ioctls::VmFd; use std::collections::HashMap; use std::sync::{Arc, Mutex}; @@ -161,8 +161,8 @@ impl MsiCap { pub struct MsiConfig { pub cap: MsiCap, pub irq_routes: Vec, - _vm_fd: Arc, - _gsi_msi_routes: Arc>>, + vm_fd: Arc, + gsi_msi_routes: Arc>>, } impl MsiConfig { @@ -185,8 +185,8 @@ impl MsiConfig { MsiConfig { cap, irq_routes, - _vm_fd: vm_fd, - _gsi_msi_routes: gsi_msi_routes, + vm_fd, + gsi_msi_routes, } } @@ -194,10 +194,6 @@ impl MsiConfig { self.cap.enabled() } - pub fn update(&mut self, offset: u64, data: &[u8]) { - self.cap.update(offset, data) - } - pub fn size(&self) -> u64 { self.cap.size() } @@ -209,4 +205,53 @@ impl MsiConfig { pub fn vector_masked(&self, vector: usize) -> bool { self.cap.vector_masked(vector) } + + pub fn update(&mut self, offset: u64, data: &[u8]) { + let old_enabled = self.cap.enabled(); + + self.cap.update(offset, data); + + let mut gsi_msi_routes = self.gsi_msi_routes.lock().unwrap(); + + if self.cap.enabled() { + for (idx, route) in self.irq_routes.iter().enumerate() { + if !old_enabled { + if let Err(e) = self.irq_routes[idx].enable(&self.vm_fd) { + error!("Failed enabling irq_fd: {:?}", e); + } + } + + // Ignore MSI vector if masked. + if self.cap.vector_masked(idx) { + continue; + } + + let mut entry = kvm_irq_routing_entry { + gsi: route.gsi, + type_: KVM_IRQ_ROUTING_MSI, + ..Default::default() + }; + + entry.u.msi.address_lo = self.cap.msg_addr_lo; + entry.u.msi.address_hi = self.cap.msg_addr_hi; + entry.u.msi.data = u32::from(self.cap.msg_data) | (idx as u32); + + gsi_msi_routes.insert(route.gsi, entry); + } + } else { + for route in self.irq_routes.iter() { + if old_enabled { + if let Err(e) = route.disable(&self.vm_fd) { + error!("Failed disabling irq_fd: {:?}", e); + } + } + + gsi_msi_routes.remove(&route.gsi); + } + } + + if let Err(e) = set_kvm_routes(&self.vm_fd, &gsi_msi_routes) { + error!("Failed updating KVM routes: {:?}", e); + } + } } diff --git a/vfio/src/vfio_pci.rs b/vfio/src/vfio_pci.rs index 44c636e73..f8dedd297 100644 --- a/vfio/src/vfio_pci.rs +++ b/vfio/src/vfio_pci.rs @@ -209,11 +209,13 @@ impl Interrupt { } } +#[allow(dead_code)] struct InterruptRoute { gsi: u32, irq_fd: EventFd, } +#[allow(dead_code)] impl InterruptRoute { fn new(allocator: &mut SystemAllocator) -> Result { let irq_fd = EventFd::new(libc::EFD_NONBLOCK).map_err(VfioPciError::EventFd)?; @@ -300,6 +302,7 @@ pub struct VfioPciDevice { gsi_msi_routes: Arc>>, } +#[allow(dead_code)] impl VfioPciDevice { /// Constructs a new Vfio Pci device for the given Vfio device pub fn new( @@ -518,35 +521,27 @@ impl VfioPciDevice { fn update_msi_capabilities(&mut self, offset: u64, data: &[u8]) -> Result<()> { match self.interrupt.update_msi(offset, data) { - Some(InterruptUpdateAction::EnableMsi) => match self.enable_irq_fds() { - Ok(fds) => { - if let Err(e) = self.device.enable_msi(fds) { + Some(InterruptUpdateAction::EnableMsi) => { + if let Some(msi) = &self.interrupt.msi { + let mut irq_fds: Vec<&EventFd> = Vec::new(); + for r in msi.cfg.irq_routes.iter() { + irq_fds.push(&r.irq_fd); + } + + if let Err(e) = self.device.enable_msi(irq_fds) { warn!("Could not enable MSI: {}", e); } } - Err(e) => warn!("Could not get IRQ fds: {}", e), - }, + } Some(InterruptUpdateAction::DisableMsi) => { if let Err(e) = self.device.disable_msi() { warn!("Could not disable MSI: {}", e); } - if let Err(e) = self.disable_irq_fds() { - warn!("Could not disable MSI: {}", e); - } } _ => {} } - // Update the gsi_msi_routes table now that the MSI cache has been - // updated. The point is to always update the table based on latest - // changes to the cache, and based on the state of masking flags, the - // KVM GSI routes should be configured. - if let Some(msi) = &self.interrupt.msi { - return self.update_msi_interrupt_routes(&msi); - } - - // If the code reach this point, something went wrong. - Err(VfioPciError::MsiNotConfigured) + Ok(()) } fn update_msix_capabilities(&mut self, offset: u64, data: &[u8]) -> Result<()> {