External/cloud-hypervisor
1
0
Fork 0
mirror of https://github.com/cloud-hypervisor/cloud-hypervisor.git synced 2025-01-18 10:35:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

vmm: Remove hardcoded zero PCI segment id

Browse Source 
Replace the hardcoded zero PCI segment id when adding devices to the bus
and extend the DeviceTree to hold the PCI segment id.

Signed-off-by: Rob Bradford <robert.bradford@intel.com>
This commit is contained in:
Rob Bradford 2021-10-07 18:07:13 +01:00
parent b8b0dab1ae
commit d33d254921
2 changed files with 73 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

100
vmm/src/device_manager.rs
View File

@ -400,6 +400,9 @@ pub enum DeviceManagerError {
/// Missing PCI b/d/f from the DeviceNode. /// Missing PCI b/d/f from the DeviceNode.
MissingDeviceNodePciBdf, MissingDeviceNodePciBdf,
/// Missing PCI segment id from the DeviceNode.
MissingDeviceNodePciSegmentId,
/// No support for device passthrough /// No support for device passthrough
NoDevicePassthroughSupport, NoDevicePassthroughSupport,
@ -2806,7 +2809,9 @@ impl DeviceManager {
&mut self, &mut self,
device_cfg: &mut DeviceConfig, device_cfg: &mut DeviceConfig,
) -> DeviceManagerResult<(u32, String)> { ) -> DeviceManagerResult<(u32, String)> {
let pci_device_bdf = self.pci_segments[0].next_device_bdf()?; // TODO: Fill with PCI segment ID from config when available
let pci_segment_id = 0;
let pci_device_bdf = self.pci_segments[pci_segment_id as usize].next_device_bdf()?;
let mut needs_dma_mapping = false; let mut needs_dma_mapping = false;
@ -2887,7 +2892,8 @@ impl DeviceManager {
Some( Some(
legacy_interrupt_manager legacy_interrupt_manager
.create_group(LegacyIrqGroupConfig { .create_group(LegacyIrqGroupConfig {
irq: self.pci_segments[0].pci_irq_slots[(pci_device_bdf >> 3) as usize] irq: self.pci_segments[pci_segment_id as usize].pci_irq_slots
[(pci_device_bdf >> 3) as usize]
as InterruptIndex, as InterruptIndex,
}) })
.map_err(DeviceManagerError::CreateInterruptGroup)?, .map_err(DeviceManagerError::CreateInterruptGroup)?,
@ -2923,7 +2929,7 @@ impl DeviceManager {
self.add_pci_device( self.add_pci_device(
vfio_pci_device.clone(), vfio_pci_device.clone(),
vfio_pci_device.clone(), vfio_pci_device.clone(),
0, pci_segment_id,
pci_device_bdf, pci_device_bdf,
)?; )?;
@ -2945,6 +2951,7 @@ impl DeviceManager {
} }
node.pci_bdf = Some(pci_device_bdf); node.pci_bdf = Some(pci_device_bdf);
node.pci_segment_id = Some(pci_segment_id);
node.pci_device_handle = Some(PciDeviceHandle::Vfio(vfio_pci_device)); node.pci_device_handle = Some(PciDeviceHandle::Vfio(vfio_pci_device));
self.device_tree self.device_tree
@ -3015,14 +3022,17 @@ impl DeviceManager {
&mut self, &mut self,
device_cfg: &mut UserDeviceConfig, device_cfg: &mut UserDeviceConfig,
) -> DeviceManagerResult<(u32, String)> { ) -> DeviceManagerResult<(u32, String)> {
let pci_device_bdf = self.pci_segments[0].next_device_bdf()?; // TODO: Fill with PCI segment ID from config when available
let pci_segment_id = 0;
let pci_device_bdf = self.pci_segments[pci_segment_id as usize].next_device_bdf()?;
let legacy_interrupt_group = let legacy_interrupt_group =
if let Some(legacy_interrupt_manager) = &self.legacy_interrupt_manager { if let Some(legacy_interrupt_manager) = &self.legacy_interrupt_manager {
Some( Some(
legacy_interrupt_manager legacy_interrupt_manager
.create_group(LegacyIrqGroupConfig { .create_group(LegacyIrqGroupConfig {
irq: self.pci_segments[0].pci_irq_slots[(pci_device_bdf >> 3) as usize] irq: self.pci_segments[pci_segment_id as usize].pci_irq_slots
[(pci_device_bdf >> 3) as usize]
as InterruptIndex, as InterruptIndex,
}) })
.map_err(DeviceManagerError::CreateInterruptGroup)?, .map_err(DeviceManagerError::CreateInterruptGroup)?,
@ -3088,13 +3098,14 @@ impl DeviceManager {
self.add_pci_device( self.add_pci_device(
vfio_user_pci_device.clone(), vfio_user_pci_device.clone(),
vfio_user_pci_device.clone(), vfio_user_pci_device.clone(),
0, pci_segment_id,
pci_device_bdf, pci_device_bdf,
)?; )?;
let mut node = device_node!(vfio_user_name); let mut node = device_node!(vfio_user_name);
node.pci_bdf = Some(pci_device_bdf); node.pci_bdf = Some(pci_device_bdf);
node.pci_segment_id = Some(pci_segment_id);
node.pci_device_handle = Some(PciDeviceHandle::VfioUser(vfio_user_pci_device)); node.pci_device_handle = Some(PciDeviceHandle::VfioUser(vfio_user_pci_device));
self.device_tree self.device_tree
@ -3126,6 +3137,8 @@ impl DeviceManager {
iommu_mapping: &Option<Arc<IommuMapping>>, iommu_mapping: &Option<Arc<IommuMapping>>,
virtio_device_id: String, virtio_device_id: String,
) -> DeviceManagerResult<u32> { ) -> DeviceManagerResult<u32> {
// TODO: Fill with PCI segment ID from config when available
let pci_segment_id: u16 = 0;
let id = format!("{}-{}", VIRTIO_PCI_DEVICE_NAME_PREFIX, virtio_device_id); let id = format!("{}-{}", VIRTIO_PCI_DEVICE_NAME_PREFIX, virtio_device_id);
// Add the new virtio-pci node to the device tree. // Add the new virtio-pci node to the device tree.
@ -3134,41 +3147,45 @@ impl DeviceManager {
// Look for the id in the device tree. If it can be found, that means // Look for the id in the device tree. If it can be found, that means
// the device is being restored, otherwise it's created from scratch. // the device is being restored, otherwise it's created from scratch.
let (pci_device_bdf, config_bar_addr) = let (pci_segment_id, pci_device_bdf, config_bar_addr) = if let Some(node) =
if let Some(node) = self.device_tree.lock().unwrap().get(&id) { self.device_tree.lock().unwrap().get(&id)
info!("Restoring virtio-pci {} resources", id); {
let pci_device_bdf = node info!("Restoring virtio-pci {} resources", id);
.pci_bdf let pci_device_bdf = node
.ok_or(DeviceManagerError::MissingDeviceNodePciBdf)?; .pci_bdf
.ok_or(DeviceManagerError::MissingDeviceNodePciBdf)?;
let pci_segment_id = node
.pci_segment_id
.ok_or(DeviceManagerError::MissingDeviceNodePciSegmentId)?;
self.pci_segments[0] self.pci_segments[pci_segment_id as usize]
.pci_bus .pci_bus
.lock() .lock()
.unwrap() .unwrap()
.get_device_id((pci_device_bdf >> 3) as usize) .get_device_id((pci_device_bdf >> 3) as usize)
.map_err(DeviceManagerError::GetPciDeviceId)?; .map_err(DeviceManagerError::GetPciDeviceId)?;
if node.resources.is_empty() { if node.resources.is_empty() {
return Err(DeviceManagerError::MissingVirtioPciResources);
}
// We know the configuration BAR address is stored on the first
// resource in the list.
let config_bar_addr = match node.resources[0] {
Resource::MmioAddressRange { base, .. } => Some(base),
_ => {
error!("Unexpected resource {:?} for {}", node.resources[0], id);
return Err(DeviceManagerError::MissingVirtioPciResources); return Err(DeviceManagerError::MissingVirtioPciResources);
} }
// We know the configuration BAR address is stored on the first
// resource in the list.
let config_bar_addr = match node.resources[0] {
Resource::MmioAddressRange { base, .. } => Some(base),
_ => {
error!("Unexpected resource {:?} for {}", node.resources[0], id);
return Err(DeviceManagerError::MissingVirtioPciResources);
}
};
(pci_device_bdf, config_bar_addr)
} else {
let pci_device_bdf = self.pci_segments[0].next_device_bdf()?;
(pci_device_bdf, None)
}; };
(pci_segment_id, pci_device_bdf, config_bar_addr)
} else {
let pci_device_bdf = self.pci_segments[pci_segment_id as usize].next_device_bdf()?;
(pci_segment_id, pci_device_bdf, None)
};
// Update the existing virtio node by setting the parent. // Update the existing virtio node by setting the parent.
if let Some(node) = self.device_tree.lock().unwrap().get_mut(&virtio_device_id) { if let Some(node) = self.device_tree.lock().unwrap().get_mut(&virtio_device_id) {
node.parent = Some(id.clone()); node.parent = Some(id.clone());
@ -3219,7 +3236,7 @@ impl DeviceManager {
let bars = self.add_pci_device( let bars = self.add_pci_device(
virtio_pci_device.clone(), virtio_pci_device.clone(),
virtio_pci_device.clone(), virtio_pci_device.clone(),
0, pci_segment_id,
pci_device_bdf, pci_device_bdf,
)?; )?;
@ -3241,6 +3258,7 @@ impl DeviceManager {
} }
node.migratable = Some(Arc::clone(&virtio_pci_device) as Arc<Mutex<dyn Migratable>>); node.migratable = Some(Arc::clone(&virtio_pci_device) as Arc<Mutex<dyn Migratable>>);
node.pci_bdf = Some(pci_device_bdf); node.pci_bdf = Some(pci_device_bdf);
node.pci_segment_id = Some(pci_segment_id);
node.pci_device_handle = Some(PciDeviceHandle::Virtio(virtio_pci_device)); node.pci_device_handle = Some(PciDeviceHandle::Virtio(virtio_pci_device));
self.device_tree.lock().unwrap().insert(id, node); self.device_tree.lock().unwrap().insert(id, node);
@ -3449,12 +3467,12 @@ impl DeviceManager {
Ok(()) Ok(())
} }
pub fn eject_device(&mut self, device_id: u8) -> DeviceManagerResult<()> { pub fn eject_device(&mut self, pci_segment_id: u16, device_id: u8) -> DeviceManagerResult<()> {
// Convert the device ID into the corresponding b/d/f. // Convert the device ID into the corresponding b/d/f.
let pci_device_bdf = (device_id as u32) << 3; let pci_device_bdf = (device_id as u32) << 3;
// Give the PCI device ID back to the PCI bus. // Give the PCI device ID back to the PCI bus.
self.pci_segments[0] self.pci_segments[pci_segment_id as usize]
.pci_bus .pci_bus
.lock() .lock()
.unwrap() .unwrap()
@ -3464,7 +3482,7 @@ impl DeviceManager {
// Remove the device from the device tree along with its children. // Remove the device from the device tree along with its children.
let mut device_tree = self.device_tree.lock().unwrap(); let mut device_tree = self.device_tree.lock().unwrap();
let pci_device_node = device_tree let pci_device_node = device_tree
.remove_node_by_pci_bdf(pci_device_bdf) .remove_node_by_pci_bdf(pci_segment_id, pci_device_bdf)
.ok_or(DeviceManagerError::MissingPciDevice)?; .ok_or(DeviceManagerError::MissingPciDevice)?;
for child in pci_device_node.children.iter() { for child in pci_device_node.children.iter() {
device_tree.remove(child); device_tree.remove(child);
@ -3529,7 +3547,7 @@ impl DeviceManager {
.map_err(DeviceManagerError::FreePciBars)?; .map_err(DeviceManagerError::FreePciBars)?;
// Remove the device from the PCI bus // Remove the device from the PCI bus
self.pci_segments[0] self.pci_segments[pci_segment_id as usize]
.pci_bus .pci_bus
.lock() .lock()
.unwrap() .unwrap()
@ -4112,7 +4130,7 @@ impl BusDevice for DeviceManager {
while slot_bitmap > 0 { while slot_bitmap > 0 {
let slot_id = slot_bitmap.trailing_zeros(); let slot_id = slot_bitmap.trailing_zeros();
if let Err(e) = self.eject_device(slot_id as u8) { if let Err(e) = self.eject_device(0, slot_id as u8) {
error!("Failed ejecting device {}: {:?}", slot_id, e); error!("Failed ejecting device {}: {:?}", slot_id, e);
} }
slot_bitmap &= !(1 << slot_id); slot_bitmap &= !(1 << slot_id);

21
vmm/src/device_tree.rs
View File

@ -16,6 +16,7 @@ pub struct DeviceNode {
pub children: Vec<String>, pub children: Vec<String>,
#[serde(skip)] #[serde(skip)]
pub migratable: Option<Arc<Mutex<dyn Migratable>>>, pub migratable: Option<Arc<Mutex<dyn Migratable>>>,
pub pci_segment_id: Option<u16>,
pub pci_bdf: Option<u32>, pub pci_bdf: Option<u32>,
#[serde(skip)] #[serde(skip)]
pub pci_device_handle: Option<PciDeviceHandle>, pub pci_device_handle: Option<PciDeviceHandle>,
@ -31,6 +32,7 @@ impl DeviceNode {
migratable, migratable,
pci_bdf: None, pci_bdf: None,
pci_device_handle: None, pci_device_handle: None,
pci_segment_id: None,
} }
} }
} }
@ -79,17 +81,22 @@ impl DeviceTree {
pub fn pci_devices(&self) -> Vec<&DeviceNode> { pub fn pci_devices(&self) -> Vec<&DeviceNode> {
self.0 self.0
.values() .values()
.filter(|v| v.pci_bdf.is_some() && v.pci_device_handle.is_some()) .filter(|v| {
v.pci_bdf.is_some() && v.pci_segment_id.is_some() && v.pci_device_handle.is_some()
})
.collect() .collect()
} }
pub fn remove_node_by_pci_bdf(&mut self, pci_bdf: u32) -> Option<DeviceNode> {
pub fn remove_node_by_pci_bdf(
&mut self,
pci_segment_id: u16,
pci_bdf: u32,
) -> Option<DeviceNode> {
let mut id = None; let mut id = None;
for (k, v) in self.0.iter() { for (k, v) in self.0.iter() {
if let Some(bdf) = v.pci_bdf { if v.pci_segment_id == Some(pci_segment_id) && v.pci_bdf == Some(pci_bdf) {
if bdf == pci_bdf { id = Some(k.clone());
id = Some(k.clone()); break;
break;
}
} }
} }