pci, virtio-devices: Extend barrier returning through PCI code

We need to be able to return the barrier from the code that prepares to
activate the virtio device. This triggered by a write to the
configuration fields stored in the PCI BAR. Since bars can be accessed
by both memory mapping and through PCI config I/O several prototypes
must be changed.

Signed-off-by: Rob Bradford <robert.bradford@intel.com>

pci/src/bus.rs

@@ -71,8 +71,14 @@ impl PciRoot {
 impl BusDevice for PciRoot {}
 
 impl PciDevice for PciRoot {
-    fn write_config_register(&mut self, reg_idx: usize, offset: u64, data: &[u8]) {
+    fn write_config_register(
+        &mut self,
+        reg_idx: usize,
+        offset: u64,
+        data: &[u8],
+    ) -> Option<Arc<Barrier>> {
         self.config.write_config_register(reg_idx, offset, data);
+        None
     }
 
     fn read_config_register(&mut self, reg_idx: usize) -> u32 {
@@ -225,14 +231,14 @@ impl PciConfigIo {
             })
     }
 
-    pub fn config_space_write(&mut self, offset: u64, data: &[u8]) {
+    pub fn config_space_write(&mut self, offset: u64, data: &[u8]) -> Option<Arc<Barrier>> {
         if offset as usize + data.len() > 4 {
-            return;
+            return None;
         }
 
         let enabled = (self.config_address & 0x8000_0000) != 0;
         if !enabled {
-            return;
+            return None;
         }
 
         let (bus, device, _function, register) =
@@ -240,7 +246,7 @@ impl PciConfigIo {
 
         // Only support one bus.
         if bus != 0 {
-            return;
+            return None;
         }
 
         let pci_bus = self.pci_bus.lock().unwrap();
@@ -265,7 +271,9 @@ impl PciConfigIo {
             }
 
             // Update the register value
-            device.write_config_register(register, offset, data);
+            device.write_config_register(register, offset, data)
+        } else {
+            None
         }
     }
 
@@ -315,12 +323,13 @@ impl BusDevice for PciConfigIo {
     fn write(&mut self, _base: u64, offset: u64, data: &[u8]) -> Option<Arc<Barrier>> {
         // `offset` is relative to 0xcf8
         match offset {
-            o @ 0..=3 => self.set_config_address(o, data),
+            o @ 0..=3 => {
+                self.set_config_address(o, data);
+                None
+            }
             o @ 4..=7 => self.config_space_write(o - 4, data),
-            _ => (),
+            _ => None,
-        };
+        }
-
-        None
     }
 }
 

pci/src/device.rs

@@ -5,6 +5,7 @@
 use crate::configuration::{self, PciBarRegionType};
 use std::any::Any;
 use std::fmt::{self, Display};
+use std::sync::{Arc, Barrier};
 use std::{self, io, result};
 use vm_allocator::SystemAllocator;
 use vm_device::BusDevice;
@@ -63,7 +64,12 @@ pub trait PciDevice: BusDevice {
     /// Sets a register in the configuration space.
     /// * `reg_idx` - The index of the config register to modify.
     /// * `offset` - Offset in to the register.
-    fn write_config_register(&mut self, reg_idx: usize, offset: u64, data: &[u8]);
+    fn write_config_register(
+        &mut self,
+        reg_idx: usize,
+        offset: u64,
+        data: &[u8],
+    ) -> Option<Arc<Barrier>>;
     /// Gets a register from the configuration space.
     /// * `reg_idx` - The index of the config register to read.
     fn read_config_register(&mut self, reg_idx: usize) -> u32;
@@ -82,7 +88,9 @@ pub trait PciDevice: BusDevice {
     /// Writes to a BAR region mapped in to the device.
     /// * `addr` - The guest address inside the BAR.
     /// * `data` - The data to write.
-    fn write_bar(&mut self, _base: u64, _offset: u64, _data: &[u8]) {}
+    fn write_bar(&mut self, _base: u64, _offset: u64, _data: &[u8]) -> Option<Arc<Barrier>> {
+        None
+    }
     /// Relocates the BAR to a different address in guest address space.
     fn move_bar(&mut self, _old_base: u64, _new_base: u64) -> result::Result<(), io::Error> {
         Ok(())

pci/src/vfio.rs

@@ -653,9 +653,7 @@ impl BusDevice for VfioPciDevice {
     }
 
     fn write(&mut self, base: u64, offset: u64, data: &[u8]) -> Option<Arc<Barrier>> {
-        self.write_bar(base, offset, data);
+        self.write_bar(base, offset, data)
-
-        None
     }
 }
 
@@ -885,7 +883,12 @@ impl PciDevice for VfioPciDevice {
         Ok(())
     }
 
-    fn write_config_register(&mut self, reg_idx: usize, offset: u64, data: &[u8]) {
+    fn write_config_register(
+        &mut self,
+        reg_idx: usize,
+        offset: u64,
+        data: &[u8],
+    ) -> Option<Arc<Barrier>> {
         // When the guest wants to write to a BAR, we trap it into
         // our local configuration space. We're not reprogramming
         // VFIO device.
@@ -895,9 +898,9 @@ impl PciDevice for VfioPciDevice {
             // We keep our local cache updated with the BARs.
             // We'll read it back from there when the guest is asking
             // for BARs (see read_config_register()).
-            return self
+            self.configuration
-                .configuration
                 .write_config_register(reg_idx, offset, data);
+            return None;
         }
 
         let reg = (reg_idx * PCI_CONFIG_REGISTER_SIZE) as u64;
@@ -933,6 +936,8 @@ impl PciDevice for VfioPciDevice {
         // to the device region to update the MSI Enable bit.
         self.device
             .region_write(VFIO_PCI_CONFIG_REGION_INDEX, data, reg + offset);
+
+        None
     }
 
     fn read_config_register(&mut self, reg_idx: usize) -> u32 {
@@ -990,7 +995,7 @@ impl PciDevice for VfioPciDevice {
         }
     }
 
-    fn write_bar(&mut self, base: u64, offset: u64, data: &[u8]) {
+    fn write_bar(&mut self, base: u64, offset: u64, data: &[u8]) -> Option<Arc<Barrier>> {
         let addr = base + offset;
         if let Some(region) = self.find_region(addr) {
             let offset = addr - region.start.raw_value();
@@ -1002,6 +1007,8 @@ impl PciDevice for VfioPciDevice {
                 self.device.region_write(region.index, data, offset);
             }
         }
+
+        None
     }
 
     fn move_bar(&mut self, old_base: u64, new_base: u64) -> result::Result<(), io::Error> {

virtio-devices/src/transport/pci_device.rs

@@ -617,18 +617,19 @@ impl VirtioPciDevice {
         }
     }
 
-    fn write_cap_pci_cfg(&mut self, offset: usize, data: &[u8]) {
+    fn write_cap_pci_cfg(&mut self, offset: usize, data: &[u8]) -> Option<Arc<Barrier>> {
         let cap_slice = self.cap_pci_cfg_info.cap.as_mut_slice();
         let data_len = data.len();
         let cap_len = cap_slice.len();
         if offset + data_len > cap_len {
             error!("Failed to write cap_pci_cfg to config space");
-            return;
+            return None;
         }
 
         if offset < std::mem::size_of::<VirtioPciCap>() {
             let (_, right) = cap_slice.split_at_mut(offset);
             right[..data_len].copy_from_slice(&data[..]);
+            None
         } else {
             // Safe since we know self.cap_pci_cfg_info.cap.cap.offset is 32bits long.
             let bar_offset: u32 =
@@ -733,7 +734,12 @@ impl VirtioInterrupt for VirtioInterruptMsix {
 }
 
 impl PciDevice for VirtioPciDevice {
-    fn write_config_register(&mut self, reg_idx: usize, offset: u64, data: &[u8]) {
+    fn write_config_register(
+        &mut self,
+        reg_idx: usize,
+        offset: u64,
+        data: &[u8],
+    ) -> Option<Arc<Barrier>> {
         // Handle the special case where the capability VIRTIO_PCI_CAP_PCI_CFG
         // is accessed. This capability has a special meaning as it allows the
         // guest to access other capabilities without mapping the PCI BAR.
@@ -743,10 +749,11 @@ impl PciDevice for VirtioPciDevice {
                 <= self.cap_pci_cfg_info.offset + self.cap_pci_cfg_info.cap.bytes().len()
         {
             let offset = base + offset as usize - self.cap_pci_cfg_info.offset;
-            self.write_cap_pci_cfg(offset, data);
+            self.write_cap_pci_cfg(offset, data)
         } else {
             self.configuration
                 .write_config_register(reg_idx, offset, data);
+            None
         }
     }
 
@@ -925,7 +932,7 @@ impl PciDevice for VirtioPciDevice {
         }
     }
 
-    fn write_bar(&mut self, _base: u64, offset: u64, data: &[u8]) {
+    fn write_bar(&mut self, _base: u64, offset: u64, data: &[u8]) -> Option<Arc<Barrier>> {
         match offset {
             o if o < COMMON_CONFIG_BAR_OFFSET + COMMON_CONFIG_SIZE => self.common_config.write(
                 o - COMMON_CONFIG_BAR_OFFSET,
@@ -1006,6 +1013,8 @@ impl PciDevice for VirtioPciDevice {
                 self.common_config.driver_status = crate::DEVICE_FAILED as u8;
             }
         }
+
+        None
     }
 
     fn as_any(&mut self) -> &mut dyn Any {
@@ -1019,8 +1028,7 @@ impl BusDevice for VirtioPciDevice {
     }
 
     fn write(&mut self, base: u64, offset: u64, data: &[u8]) -> Option<Arc<Barrier>> {
-        self.write_bar(base, offset, data);
+        self.write_bar(base, offset, data)
-        None
     }
 }