External/cloud-hypervisor
1
0
Fork 0
mirror of https://github.com/cloud-hypervisor/cloud-hypervisor.git synced 2025-01-22 04:25:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

vmm: Extract PCI related state from DeviceManager

Browse Source 
Move the PCI related state from the DeviceManager struct to a PciSegment
struct inside the DeviceManager. This is in preparation for multiple
segment support. Currently this state is just the bus itself, the MMIO
and PIO config devices and hotplug related state.

The main change that this required is using the Arc<Mutex<PciBus>> in
the device addition logic in order to ensure that
the bus could be created earlier.

Signed-off-by: Rob Bradford <robert.bradford@intel.com>
This commit is contained in:
Rob Bradford 2021-10-04 15:33:29 +01:00
parent 9aadbbe6d9
commit 0eb78ab177
2 changed files with 205 additions and 214 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
pci/src/bus.rs
View File

@ -188,25 +188,20 @@ impl PciBus {
} }
} }
#[derive(Default)]
pub struct PciConfigIo { pub struct PciConfigIo {
/// Config space register. /// Config space register.
config_address: u32, config_address: u32,
pci_bus: Option<Arc<Mutex<PciBus>>>, pci_bus: Arc<Mutex<PciBus>>,
} }
impl PciConfigIo { impl PciConfigIo {
pub fn new() -> Self { pub fn new(pci_bus: Arc<Mutex<PciBus>>) -> Self {
PciConfigIo { PciConfigIo {
config_address: 0, config_address: 0,
pci_bus: None, pci_bus,
} }
} }
pub fn set_bus(&mut self, pci_bus: Arc<Mutex<PciBus>>) {
self.pci_bus = Some(pci_bus)
}
pub fn config_space_read(&self) -> u32 { pub fn config_space_read(&self) -> u32 {
let enabled = (self.config_address & 0x8000_0000) != 0; let enabled = (self.config_address & 0x8000_0000) != 0;
if !enabled { if !enabled {
@ -228,7 +223,6 @@ impl PciConfigIo {
self.pci_bus self.pci_bus
.as_ref() .as_ref()
.unwrap()
.lock() .lock()
.unwrap() .unwrap()
.devices .devices
@ -256,7 +250,7 @@ impl PciConfigIo {
return None; return None;
} }
let pci_bus = self.pci_bus.as_ref().unwrap().lock().unwrap(); let pci_bus = self.pci_bus.as_ref().lock().unwrap();
if let Some(d) = pci_bus.devices.get(&(device as u32)) { if let Some(d) = pci_bus.devices.get(&(device as u32)) {
let mut device = d.lock().unwrap(); let mut device = d.lock().unwrap();

339
vmm/src/device_manager.rs
View File

@ -826,11 +826,7 @@ pub struct DeviceManager {
// Counter to keep track of the consumed device IDs. // Counter to keep track of the consumed device IDs.
device_id_cnt: Wrapping<usize>, device_id_cnt: Wrapping<usize>,
// Keep a reference to the PCI bus pci_segment: PciSegment,
pci_bus: Option<Arc<Mutex<PciBus>>>,
#[cfg(target_arch = "x86_64")]
pci_config_io: Arc<Mutex<PciConfigIo>>,
#[cfg_attr(target_arch = "aarch64", allow(dead_code))] #[cfg_attr(target_arch = "aarch64", allow(dead_code))]
// MSI Interrupt Manager // MSI Interrupt Manager
@ -857,15 +853,6 @@ pub struct DeviceManager {
// information for filling the ACPI VIOT table. // information for filling the ACPI VIOT table.
iommu_attached_devices: Option<(u32, Vec<u32>)>, iommu_attached_devices: Option<(u32, Vec<u32>)>,
// Bitmap of PCI devices to hotplug.
pci_devices_up: u32,
// Bitmap of PCI devices to hotunplug.
pci_devices_down: u32,
// List of allocated IRQs for each PCI slot.
pci_irq_slots: [u8; 32],
// Tree of devices, representing the dependencies between devices. // Tree of devices, representing the dependencies between devices.
// Useful for introspection, snapshot and restore. // Useful for introspection, snapshot and restore.
device_tree: Arc<Mutex<DeviceTree>>, device_tree: Arc<Mutex<DeviceTree>>,
@ -908,6 +895,120 @@ pub struct DeviceManager {
restoring: bool, restoring: bool,
} }
struct PciSegment {
id: u16,
pci_bus: Arc<Mutex<PciBus>>,
pci_config_mmio: Arc<Mutex<PciConfigMmio>>,
mmio_config_address: u64,
#[cfg(target_arch = "x86_64")]
pci_config_io: Option<Arc<Mutex<PciConfigIo>>>,
// Bitmap of PCI devices to hotplug.
pci_devices_up: u32,
// Bitmap of PCI devices to hotunplug.
pci_devices_down: u32,
// List of allocated IRQs for each PCI slot.
pci_irq_slots: [u8; 32],
}
impl PciSegment {
fn new_default_segment(
address_manager: &Arc<AddressManager>,
) -> DeviceManagerResult<PciSegment> {
let pci_root = PciRoot::new(None);
let pci_bus = Arc::new(Mutex::new(PciBus::new(
pci_root,
Arc::clone(address_manager) as Arc<dyn DeviceRelocation>,
)));
let pci_config_mmio = Arc::new(Mutex::new(PciConfigMmio::new(Arc::clone(&pci_bus))));
address_manager
.mmio_bus
.insert(
Arc::clone(&pci_config_mmio) as Arc<Mutex<dyn BusDevice>>,
arch::layout::PCI_MMCONFIG_START.0,
arch::layout::PCI_MMCONFIG_SIZE,
)
.map_err(DeviceManagerError::BusError)?;
#[cfg(target_arch = "x86_64")]
let pci_config_io = Arc::new(Mutex::new(PciConfigIo::new(Arc::clone(&pci_bus))));
#[cfg(target_arch = "x86_64")]
address_manager
.io_bus
.insert(
pci_config_io.clone(),
PCI_CONFIG_IO_PORT,
PCI_CONFIG_IO_PORT_SIZE,
)
.map_err(DeviceManagerError::BusError)?;
let mut segment = PciSegment {
id: 0,
pci_bus,
pci_config_mmio,
mmio_config_address: arch::layout::PCI_MMCONFIG_START.0,
pci_devices_up: 0,
pci_devices_down: 0,
pci_irq_slots: [0; 32],
#[cfg(target_arch = "x86_64")]
pci_config_io: Some(pci_config_io),
};
// Reserve some IRQs for PCI devices in case they need to support INTx.
segment.reserve_legacy_interrupts_for_pci_devices(address_manager)?;
info!(
"Adding PCI segment: id={}, PCI MMIO config address: 0x{:x}",
segment.id, segment.mmio_config_address
);
Ok(segment)
}
fn next_device_bdf(&self) -> DeviceManagerResult<u32> {
// We need to shift the device id since the 3 first bits
// are dedicated to the PCI function, and we know we don't
// do multifunction. Also, because we only support one PCI
// bus, the bus 0, we don't need to add anything to the
// global device ID.
Ok(self
.pci_bus
.lock()
.unwrap()
.next_device_id()
.map_err(DeviceManagerError::NextPciDeviceId)?
<< 3)
}
fn reserve_legacy_interrupts_for_pci_devices(
&mut self,
address_manager: &Arc<AddressManager>,
) -> DeviceManagerResult<()> {
// Reserve 8 IRQs which will be shared across all PCI devices.
let num_irqs = 8;
let mut irqs: Vec<u8> = Vec::new();
for _ in 0..num_irqs {
irqs.push(
address_manager
.allocator
.lock()
.unwrap()
.allocate_irq()
.ok_or(DeviceManagerError::AllocateIrq)? as u8,
);
}
// There are 32 devices on the PCI bus, let's assign them an IRQ.
for i in 0..32 {
self.pci_irq_slots[i] = irqs[(i % num_irqs) as usize];
}
Ok(())
}
}
impl DeviceManager { impl DeviceManager {
#[allow(clippy::too_many_arguments)] #[allow(clippy::too_many_arguments)]
pub fn new( pub fn new(
@ -964,19 +1065,13 @@ impl DeviceManager {
virtio_devices: Vec::new(), virtio_devices: Vec::new(),
bus_devices: Vec::new(), bus_devices: Vec::new(),
device_id_cnt: Wrapping(0), device_id_cnt: Wrapping(0),
pci_bus: None,
#[cfg(target_arch = "x86_64")]
// Create early so ready for use in `VmmOps`
pci_config_io: Arc::new(Mutex::new(PciConfigIo::new())),
msi_interrupt_manager, msi_interrupt_manager,
legacy_interrupt_manager: None, legacy_interrupt_manager: None,
passthrough_device: None, passthrough_device: None,
vfio_container: None, vfio_container: None,
iommu_device: None, iommu_device: None,
iommu_attached_devices: None, iommu_attached_devices: None,
pci_devices_up: 0, pci_segment: PciSegment::new_default_segment(&address_manager)?,
pci_devices_down: 0,
pci_irq_slots: [0; 32],
device_tree, device_tree,
exit_evt: exit_evt.try_clone().map_err(DeviceManagerError::EventFd)?, exit_evt: exit_evt.try_clone().map_err(DeviceManagerError::EventFd)?,
reset_evt: reset_evt.try_clone().map_err(DeviceManagerError::EventFd)?, reset_evt: reset_evt.try_clone().map_err(DeviceManagerError::EventFd)?,
@ -1095,9 +1190,6 @@ impl DeviceManager {
console_resize_pipe, console_resize_pipe,
)?; )?;
// Reserve some IRQs for PCI devices in case they need to support INTx.
self.reserve_legacy_interrupts_for_pci_devices()?;
self.legacy_interrupt_manager = Some(legacy_interrupt_manager); self.legacy_interrupt_manager = Some(legacy_interrupt_manager);
virtio_devices.append(&mut self.make_virtio_devices()?); virtio_devices.append(&mut self.make_virtio_devices()?);
@ -1109,29 +1201,6 @@ impl DeviceManager {
Ok(()) Ok(())
} }
fn reserve_legacy_interrupts_for_pci_devices(&mut self) -> DeviceManagerResult<()> {
// Reserve 8 IRQs which will be shared across all PCI devices.
let num_irqs = 8;
let mut irqs: Vec<u8> = Vec::new();
for _ in 0..num_irqs {
irqs.push(
self.address_manager
.allocator
.lock()
.unwrap()
.allocate_irq()
.ok_or(DeviceManagerError::AllocateIrq)? as u8,
);
}
// There are 32 devices on the PCI bus, let's assign them an IRQ.
for i in 0..32 {
self.pci_irq_slots[i] = irqs[(i % num_irqs) as usize];
}
Ok(())
}
fn state(&self) -> DeviceManagerState { fn state(&self) -> DeviceManagerState {
DeviceManagerState { DeviceManagerState {
device_tree: self.device_tree.lock().unwrap().clone(), device_tree: self.device_tree.lock().unwrap().clone(),
@ -1169,12 +1238,6 @@ impl DeviceManager {
&mut self, &mut self,
virtio_devices: Vec<(VirtioDeviceArc, bool, String)>, virtio_devices: Vec<(VirtioDeviceArc, bool, String)>,
) -> DeviceManagerResult<()> { ) -> DeviceManagerResult<()> {
let pci_root = PciRoot::new(None);
let mut pci_bus = PciBus::new(
pci_root,
Arc::clone(&self.address_manager) as Arc<dyn DeviceRelocation>,
);
let iommu_id = String::from(IOMMU_DEVICE_NAME); let iommu_id = String::from(IOMMU_DEVICE_NAME);
let (iommu_device, iommu_mapping) = if self.config.lock().unwrap().iommu { let (iommu_device, iommu_mapping) = if self.config.lock().unwrap().iommu {
@ -1204,7 +1267,7 @@ impl DeviceManager {
}; };
let mut iommu_attached_devices = Vec::new(); let mut iommu_attached_devices = Vec::new();
{
for (device, iommu_attached, id) in virtio_devices { for (device, iommu_attached, id) in virtio_devices {
let mapping: &Option<Arc<IommuMapping>> = if iommu_attached { let mapping: &Option<Arc<IommuMapping>> = if iommu_attached {
&iommu_mapping &iommu_mapping
@ -1212,55 +1275,32 @@ impl DeviceManager {
&None &None
}; };
let dev_id = self.add_virtio_pci_device(device, &mut pci_bus, mapping, id)?; let dev_id = self.add_virtio_pci_device(device, mapping, id)?;
if iommu_attached { if iommu_attached {
iommu_attached_devices.push(dev_id); iommu_attached_devices.push(dev_id);
} }
} }
let mut vfio_iommu_device_ids = self.add_vfio_devices(&mut pci_bus)?; let mut vfio_iommu_device_ids = self.add_vfio_devices()?;
iommu_attached_devices.append(&mut vfio_iommu_device_ids); iommu_attached_devices.append(&mut vfio_iommu_device_ids);
let mut vfio_user_iommu_device_ids = self.add_user_devices(&mut pci_bus)?; let mut vfio_user_iommu_device_ids = self.add_user_devices()?;
iommu_attached_devices.append(&mut vfio_user_iommu_device_ids); iommu_attached_devices.append(&mut vfio_user_iommu_device_ids);
if let Some(iommu_device) = iommu_device { if let Some(iommu_device) = iommu_device {
let dev_id = self.add_virtio_pci_device(iommu_device, &mut pci_bus, &None, iommu_id)?; let dev_id = self.add_virtio_pci_device(iommu_device, &None, iommu_id)?;
self.iommu_attached_devices = Some((dev_id, iommu_attached_devices)); self.iommu_attached_devices = Some((dev_id, iommu_attached_devices));
} }
}
let pci_bus = Arc::new(Mutex::new(pci_bus));
#[cfg(target_arch = "x86_64")]
self.pci_config_io
.lock()
.unwrap()
.set_bus(Arc::clone(&pci_bus));
#[cfg(target_arch = "x86_64")] #[cfg(target_arch = "x86_64")]
self.bus_devices self.bus_devices
.push(Arc::clone(&self.pci_config_io) as Arc<Mutex<dyn BusDevice>>); .push(Arc::clone(self.pci_segment.pci_config_io.as_ref().unwrap())
#[cfg(target_arch = "x86_64")] as Arc<Mutex<dyn BusDevice>>);
self.address_manager
.io_bus
.insert(
self.pci_config_io.clone(),
PCI_CONFIG_IO_PORT,
PCI_CONFIG_IO_PORT_SIZE,
)
.map_err(DeviceManagerError::BusError)?;
let pci_config_mmio = Arc::new(Mutex::new(PciConfigMmio::new(Arc::clone(&pci_bus))));
self.bus_devices
.push(Arc::clone(&pci_config_mmio) as Arc<Mutex<dyn BusDevice>>);
self.address_manager
.mmio_bus
.insert(
pci_config_mmio,
arch::layout::PCI_MMCONFIG_START.0,
arch::layout::PCI_MMCONFIG_SIZE,
)
.map_err(DeviceManagerError::BusError)?;
self.pci_bus = Some(pci_bus); self.bus_devices
.push(Arc::clone(&self.pci_segment.pci_config_mmio) as Arc<Mutex<dyn BusDevice>>);
Ok(()) Ok(())
} }
@ -2819,7 +2859,6 @@ impl DeviceManager {
fn add_passthrough_device( fn add_passthrough_device(
&mut self, &mut self,
pci: &mut PciBus,
device_cfg: &mut DeviceConfig, device_cfg: &mut DeviceConfig,
) -> DeviceManagerResult<(u32, String)> { ) -> DeviceManagerResult<(u32, String)> {
// If the passthrough device has not been created yet, it is created // If the passthrough device has not been created yet, it is created
@ -2833,7 +2872,7 @@ impl DeviceManager {
); );
} }
self.add_vfio_device(pci, device_cfg) self.add_vfio_device(device_cfg)
} }
fn create_vfio_container(&self) -> DeviceManagerResult<Arc<VfioContainer>> { fn create_vfio_container(&self) -> DeviceManagerResult<Arc<VfioContainer>> {
@ -2868,18 +2907,9 @@ impl DeviceManager {
fn add_vfio_device( fn add_vfio_device(
&mut self, &mut self,
pci: &mut PciBus,
device_cfg: &mut DeviceConfig, device_cfg: &mut DeviceConfig,
) -> DeviceManagerResult<(u32, String)> { ) -> DeviceManagerResult<(u32, String)> {
// We need to shift the device id since the 3 first bits let pci_device_bdf = self.pci_segment.next_device_bdf()?;
// are dedicated to the PCI function, and we know we don't
// do multifunction. Also, because we only support one PCI
// bus, the bus 0, we don't need to add anything to the
// global device ID.
let pci_device_bdf = pci
.next_device_id()
.map_err(DeviceManagerError::NextPciDeviceId)?
<< 3;
let mut needs_dma_mapping = false; let mut needs_dma_mapping = false;
@ -2955,13 +2985,13 @@ impl DeviceManager {
} }
} }
let legacy_interrupt_group = if let Some(legacy_interrupt_manager) = let legacy_interrupt_group =
&self.legacy_interrupt_manager if let Some(legacy_interrupt_manager) = &self.legacy_interrupt_manager {
{
Some( Some(
legacy_interrupt_manager legacy_interrupt_manager
.create_group(LegacyIrqGroupConfig { .create_group(LegacyIrqGroupConfig {
irq: self.pci_irq_slots[(pci_device_bdf >> 3) as usize] as InterruptIndex, irq: self.pci_segment.pci_irq_slots[(pci_device_bdf >> 3) as usize]
as InterruptIndex,
}) })
.map_err(DeviceManagerError::CreateInterruptGroup)?, .map_err(DeviceManagerError::CreateInterruptGroup)?,
) )
@ -3009,7 +3039,6 @@ impl DeviceManager {
let vfio_pci_device = Arc::new(Mutex::new(vfio_pci_device)); let vfio_pci_device = Arc::new(Mutex::new(vfio_pci_device));
self.add_pci_device( self.add_pci_device(
pci,
vfio_pci_device.clone(), vfio_pci_device.clone(),
vfio_pci_device.clone(), vfio_pci_device.clone(),
pci_device_bdf, pci_device_bdf,
@ -3028,7 +3057,6 @@ impl DeviceManager {
fn add_pci_device( fn add_pci_device(
&mut self, &mut self,
pci_bus: &mut PciBus,
bus_device: Arc<Mutex<dyn BusDevice>>, bus_device: Arc<Mutex<dyn BusDevice>>,
pci_device: Arc<Mutex<dyn PciDevice>>, pci_device: Arc<Mutex<dyn PciDevice>>,
bdf: u32, bdf: u32,
@ -3039,6 +3067,8 @@ impl DeviceManager {
.allocate_bars(&mut self.address_manager.allocator.lock().unwrap()) .allocate_bars(&mut self.address_manager.allocator.lock().unwrap())
.map_err(DeviceManagerError::AllocateBars)?; .map_err(DeviceManagerError::AllocateBars)?;
let mut pci_bus = self.pci_segment.pci_bus.lock().unwrap();
pci_bus pci_bus
.add_device(bdf, pci_device) .add_device(bdf, pci_device)
.map_err(DeviceManagerError::AddPciDevice)?; .map_err(DeviceManagerError::AddPciDevice)?;
@ -3058,13 +3088,13 @@ impl DeviceManager {
Ok(bars) Ok(bars)
} }
fn add_vfio_devices(&mut self, pci: &mut PciBus) -> DeviceManagerResult<Vec<u32>> { fn add_vfio_devices(&mut self) -> DeviceManagerResult<Vec<u32>> {
let mut iommu_attached_device_ids = Vec::new(); let mut iommu_attached_device_ids = Vec::new();
let mut devices = self.config.lock().unwrap().devices.clone(); let mut devices = self.config.lock().unwrap().devices.clone();
if let Some(device_list_cfg) = &mut devices { if let Some(device_list_cfg) = &mut devices {
for device_cfg in device_list_cfg.iter_mut() { for device_cfg in device_list_cfg.iter_mut() {
let (device_id, _) = self.add_passthrough_device(pci, device_cfg)?; let (device_id, _) = self.add_passthrough_device(device_cfg)?;
if device_cfg.iommu && self.iommu_device.is_some() { if device_cfg.iommu && self.iommu_device.is_some() {
iommu_attached_device_ids.push(device_id); iommu_attached_device_ids.push(device_id);
} }
@ -3079,21 +3109,17 @@ impl DeviceManager {
fn add_vfio_user_device( fn add_vfio_user_device(
&mut self, &mut self,
pci: &mut PciBus,
device_cfg: &mut UserDeviceConfig, device_cfg: &mut UserDeviceConfig,
) -> DeviceManagerResult<(u32, String)> { ) -> DeviceManagerResult<(u32, String)> {
let pci_device_bdf = pci let pci_device_bdf = self.pci_segment.next_device_bdf()?;
.next_device_id()
.map_err(DeviceManagerError::NextPciDeviceId)?
<< 3;
let legacy_interrupt_group = if let Some(legacy_interrupt_manager) = let legacy_interrupt_group =
&self.legacy_interrupt_manager if let Some(legacy_interrupt_manager) = &self.legacy_interrupt_manager {
{
Some( Some(
legacy_interrupt_manager legacy_interrupt_manager
.create_group(LegacyIrqGroupConfig { .create_group(LegacyIrqGroupConfig {
irq: self.pci_irq_slots[(pci_device_bdf >> 3) as usize] as InterruptIndex, irq: self.pci_segment.pci_irq_slots[(pci_device_bdf >> 3) as usize]
as InterruptIndex,
}) })
.map_err(DeviceManagerError::CreateInterruptGroup)?, .map_err(DeviceManagerError::CreateInterruptGroup)?,
) )
@ -3156,7 +3182,6 @@ impl DeviceManager {
}; };
self.add_pci_device( self.add_pci_device(
pci,
vfio_user_pci_device.clone(), vfio_user_pci_device.clone(),
vfio_user_pci_device.clone(), vfio_user_pci_device.clone(),
pci_device_bdf, pci_device_bdf,
@ -3175,12 +3200,12 @@ impl DeviceManager {
Ok((pci_device_bdf, vfio_user_name)) Ok((pci_device_bdf, vfio_user_name))
} }
fn add_user_devices(&mut self, pci: &mut PciBus) -> DeviceManagerResult<Vec<u32>> { fn add_user_devices(&mut self) -> DeviceManagerResult<Vec<u32>> {
let mut user_devices = self.config.lock().unwrap().user_devices.clone(); let mut user_devices = self.config.lock().unwrap().user_devices.clone();
if let Some(device_list_cfg) = &mut user_devices { if let Some(device_list_cfg) = &mut user_devices {
for device_cfg in device_list_cfg.iter_mut() { for device_cfg in device_list_cfg.iter_mut() {
let (_device_id, _id) = self.add_vfio_user_device(pci, device_cfg)?; let (_device_id, _id) = self.add_vfio_user_device(device_cfg)?;
} }
} }
@ -3193,7 +3218,6 @@ impl DeviceManager {
fn add_virtio_pci_device( fn add_virtio_pci_device(
&mut self, &mut self,
virtio_device: VirtioDeviceArc, virtio_device: VirtioDeviceArc,
pci: &mut PciBus,
iommu_mapping: &Option<Arc<IommuMapping>>, iommu_mapping: &Option<Arc<IommuMapping>>,
virtio_device_id: String, virtio_device_id: String,
) -> DeviceManagerResult<u32> { ) -> DeviceManagerResult<u32> {
@ -3212,7 +3236,11 @@ impl DeviceManager {
.pci_bdf .pci_bdf
.ok_or(DeviceManagerError::MissingDeviceNodePciBdf)?; .ok_or(DeviceManagerError::MissingDeviceNodePciBdf)?;
pci.get_device_id((pci_device_bdf >> 3) as usize) self.pci_segment
.pci_bus
.lock()
.unwrap()
.get_device_id((pci_device_bdf >> 3) as usize)
.map_err(DeviceManagerError::GetPciDeviceId)?; .map_err(DeviceManagerError::GetPciDeviceId)?;
if node.resources.is_empty() { if node.resources.is_empty() {
@ -3231,14 +3259,7 @@ impl DeviceManager {
(pci_device_bdf, config_bar_addr) (pci_device_bdf, config_bar_addr)
} else { } else {
// We need to shift the device id since the 3 first bits are dedicated let pci_device_bdf = self.pci_segment.next_device_bdf()?;
// to the PCI function, and we know we don't do multifunction.
// Also, because we only support one PCI bus, the bus 0, we don't need
// to add anything to the global device ID.
let pci_device_bdf = pci
.next_device_id()
.map_err(DeviceManagerError::NextPciDeviceId)?
<< 3;
(pci_device_bdf, None) (pci_device_bdf, None)
}; };
@ -3299,7 +3320,6 @@ impl DeviceManager {
let virtio_pci_device = Arc::new(Mutex::new(virtio_pci_device)); let virtio_pci_device = Arc::new(Mutex::new(virtio_pci_device));
let bars = self.add_pci_device( let bars = self.add_pci_device(
pci,
virtio_pci_device.clone(), virtio_pci_device.clone(),
virtio_pci_device.clone(), virtio_pci_device.clone(),
pci_device_bdf, pci_device_bdf,
@ -3351,7 +3371,7 @@ impl DeviceManager {
#[cfg(target_arch = "x86_64")] #[cfg(target_arch = "x86_64")]
// Used to provide a fast path for handling PIO exits // Used to provide a fast path for handling PIO exits
pub fn pci_config_io(&self) -> Arc<Mutex<PciConfigIo>> { pub fn pci_config_io(&self) -> Arc<Mutex<PciConfigIo>> {
self.pci_config_io.clone() Arc::clone(self.pci_segment.pci_config_io.as_ref().unwrap())
} }
pub fn console(&self) -> &Arc<Console> { pub fn console(&self) -> &Arc<Console> {
@ -3444,17 +3464,10 @@ impl DeviceManager {
&mut self, &mut self,
device_cfg: &mut DeviceConfig, device_cfg: &mut DeviceConfig,
) -> DeviceManagerResult<PciDeviceInfo> { ) -> DeviceManagerResult<PciDeviceInfo> {
let pci = if let Some(pci_bus) = &self.pci_bus { let (device_id, device_name) = self.add_passthrough_device(device_cfg)?;
Arc::clone(pci_bus)
} else {
return Err(DeviceManagerError::NoPciBus);
};
let (device_id, device_name) =
self.add_passthrough_device(&mut pci.lock().unwrap(), device_cfg)?;
// Update the PCIU bitmap // Update the PCIU bitmap
self.pci_devices_up |= 1 << (device_id >> 3); self.pci_segment.pci_devices_up |= 1 << (device_id >> 3);
Ok(PciDeviceInfo { Ok(PciDeviceInfo {
id: device_name, id: device_name,
@ -3466,17 +3479,10 @@ impl DeviceManager {
&mut self, &mut self,
device_cfg: &mut UserDeviceConfig, device_cfg: &mut UserDeviceConfig,
) -> DeviceManagerResult<PciDeviceInfo> { ) -> DeviceManagerResult<PciDeviceInfo> {
let pci = if let Some(pci_bus) = &self.pci_bus { let (device_id, device_name) = self.add_vfio_user_device(device_cfg)?;
Arc::clone(pci_bus)
} else {
return Err(DeviceManagerError::NoPciBus);
};
let (device_id, device_name) =
self.add_vfio_user_device(&mut pci.lock().unwrap(), device_cfg)?;
// Update the PCIU bitmap // Update the PCIU bitmap
self.pci_devices_up |= 1 << (device_id >> 3); self.pci_segment.pci_devices_up |= 1 << (device_id >> 3);
Ok(PciDeviceInfo { Ok(PciDeviceInfo {
id: device_name, id: device_name,
@ -3535,24 +3541,19 @@ impl DeviceManager {
} }
// Update the PCID bitmap // Update the PCID bitmap
self.pci_devices_down |= 1 << (pci_device_bdf >> 3); self.pci_segment.pci_devices_down |= 1 << (pci_device_bdf >> 3);
Ok(()) Ok(())
} }
pub fn eject_device(&mut self, device_id: u8) -> DeviceManagerResult<()> { pub fn eject_device(&mut self, device_id: u8) -> DeviceManagerResult<()> {
// Retrieve the PCI bus.
let pci = if let Some(pci_bus) = &self.pci_bus {
Arc::clone(pci_bus)
} else {
return Err(DeviceManagerError::NoPciBus);
};
// Convert the device ID into the corresponding b/d/f. // Convert the device ID into the corresponding b/d/f.
let pci_device_bdf = (device_id as u32) << 3; let pci_device_bdf = (device_id as u32) << 3;
// Give the PCI device ID back to the PCI bus. // Give the PCI device ID back to the PCI bus.
pci.lock() self.pci_segment
.pci_bus
.lock()
.unwrap() .unwrap()
.put_device_id(device_id as usize) .put_device_id(device_id as usize)
.map_err(DeviceManagerError::PutPciDeviceId)?; .map_err(DeviceManagerError::PutPciDeviceId)?;
@ -3625,7 +3626,9 @@ impl DeviceManager {
.map_err(DeviceManagerError::FreePciBars)?; .map_err(DeviceManagerError::FreePciBars)?;
// Remove the device from the PCI bus // Remove the device from the PCI bus
pci.lock() self.pci_segment
.pci_bus
.lock()
.unwrap() .unwrap()
.remove_by_device(&pci_device) .remove_by_device(&pci_device)
.map_err(DeviceManagerError::RemoveDeviceFromPciBus)?; .map_err(DeviceManagerError::RemoveDeviceFromPciBus)?;
@ -3685,23 +3688,16 @@ impl DeviceManager {
warn!("Placing device behind vIOMMU is not available for hotplugged devices"); warn!("Placing device behind vIOMMU is not available for hotplugged devices");
} }
let pci = if let Some(pci_bus) = &self.pci_bus {
Arc::clone(pci_bus)
} else {
return Err(DeviceManagerError::NoPciBus);
};
// Add the virtio device to the device manager list. This is important // Add the virtio device to the device manager list. This is important
// as the list is used to notify virtio devices about memory updates // as the list is used to notify virtio devices about memory updates
// for instance. // for instance.
self.virtio_devices self.virtio_devices
.push((device.clone(), iommu_attached, id.clone())); .push((device.clone(), iommu_attached, id.clone()));
let device_id = let device_id = self.add_virtio_pci_device(device, &None, id.clone())?;
self.add_virtio_pci_device(device, &mut pci.lock().unwrap(), &None, id.clone())?;
// Update the PCIU bitmap // Update the PCIU bitmap
self.pci_devices_up |= 1 << (device_id >> 3); self.pci_segment.pci_devices_up |= 1 << (device_id >> 3);
Ok(PciDeviceInfo { id, bdf: device_id }) Ok(PciDeviceInfo { id, bdf: device_id })
} }
@ -4171,6 +4167,7 @@ impl Aml for DeviceManager {
// Build PCI routing table, listing IRQs assigned to PCI devices. // Build PCI routing table, listing IRQs assigned to PCI devices.
let prt_package_list: Vec<(u32, u32)> = self let prt_package_list: Vec<(u32, u32)> = self
.pci_segment
.pci_irq_slots .pci_irq_slots
.iter() .iter()
.enumerate() .enumerate()
@ -4424,15 +4421,15 @@ impl BusDevice for DeviceManager {
match offset { match offset {
PCIU_FIELD_OFFSET => { PCIU_FIELD_OFFSET => {
assert!(data.len() == PCIU_FIELD_SIZE); assert!(data.len() == PCIU_FIELD_SIZE);
data.copy_from_slice(&self.pci_devices_up.to_le_bytes()); data.copy_from_slice(&self.pci_segment.pci_devices_up.to_le_bytes());
// Clear the PCIU bitmap // Clear the PCIU bitmap
self.pci_devices_up = 0; self.pci_segment.pci_devices_up = 0;
} }
PCID_FIELD_OFFSET => { PCID_FIELD_OFFSET => {
assert!(data.len() == PCID_FIELD_SIZE); assert!(data.len() == PCID_FIELD_SIZE);
data.copy_from_slice(&self.pci_devices_down.to_le_bytes()); data.copy_from_slice(&self.pci_segment.pci_devices_down.to_le_bytes());
// Clear the PCID bitmap // Clear the PCID bitmap
self.pci_devices_down = 0; self.pci_segment.pci_devices_down = 0;
} }
B0EJ_FIELD_OFFSET => { B0EJ_FIELD_OFFSET => {
assert!(data.len() == B0EJ_FIELD_SIZE); assert!(data.len() == B0EJ_FIELD_SIZE);