External/cloud-hypervisor
1
0
Fork 0
mirror of https://github.com/cloud-hypervisor/cloud-hypervisor.git synced 2024-12-22 13:45:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

vmm: add pci_segment mmio aperture configs

Browse Source 
When using multiple PCI segments, the 32-bit and 64-bit mmio
aperture is split equally between each segment. Add an option
to configure the 'weight'. For example, a PCI segment with a
`mmio32_aperture_weight` of 2 will be allocated twice as much
32-bit mmio space as a normal PCI segment.

Signed-off-by: Thomas Barrett <tbarrett@crusoeenergy.com>
This commit is contained in:
Thomas Barrett 2024-04-14 19:50:45 +00:00 committed by Bo Chen
parent a84bc06874
commit e7e856d8ac
8 changed files with 334 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
docs/vfio.md
View File

@ -123,3 +123,30 @@ $ ls /sys/kernel/iommu_groups/22/devices/
This means these two devices are under the same IOMMU group 22. In such case,
it is important to bind both devices to VFIO and pass them both through the
VM, otherwise this could cause some functional and security issues.
### Advanced Configuration Options
Some VFIO devices have a 32-bit mmio BAR. When using many such devices, it is
possible to exhaust the 32-bit mmio space available on a PCI segment. The
following example demonstrates an example device with a 16 MiB 32-bit mmio BAR.
```
lspci -s 0000:01:00.0 -v
0000:01:00.0 3D controller: NVIDIA Corporation Device 26b9 (rev a1)
[...]
Memory at f9000000 (32-bit, non-prefetchable) [size=16M]
Memory at 46000000000 (64-bit, prefetchable) [size=64G]
Memory at 48040000000 (64-bit, prefetchable) [size=32M]
[...]
```
When using multiple PCI segments, the 32-bit mmio address space available to
be allocated to VFIO devices is equally split between all PCI segments by
default. This can be tuned with the `--pci-segment` flag. The following example
demonstrates a guest with two PCI segments. 2/3 of the 32-bit mmio address
space is available for use by devices on PCI segment 0 and 1/3 of the 32-bit
mmio address space is available for use by devices on PCI segment 1.
```
--platform num_pci_segments=2
--pci-segment pci_segment=0,mmio32_aperture_weight=2
--pci-segment pci_segment=1,mmio32_aperture_weight=1
```

1
fuzz/fuzz_targets/http_api.rs
View File

@ -187,6 +187,7 @@ impl RequestHandler for StubApiRequestHandler {
watchdog: false,
#[cfg(feature = "guest_debug")]
gdb: false,
pci_segments: None,
platform: None,
tpm: None,
preserved_fds: None,

8
src/main.rs
View File

@ -355,6 +355,13 @@ fn create_app(default_vcpus: String, default_memory: String, default_rng: String
.num_args(1..)
.group("vm-config"),
)
.arg(
Arg::new("pci-segment")
.long("pci-segment")
.help(config::PciSegmentConfig::SYNTAX)
.num_args(1..)
.group("vm-config"),
)
.arg(
Arg::new("watchdog")
.long("watchdog")
@ -934,6 +941,7 @@ mod unit_tests {
watchdog: false,
#[cfg(feature = "guest_debug")]
gdb: false,
pci_segments: None,
platform: None,
tpm: None,
preserved_fds: None,

19
vmm/src/api/openapi/cloud-hypervisor.yaml
View File

@ -618,6 +618,10 @@ components:
pvpanic:
type: boolean
default: false
pci_segments:
type: array
items:
$ref: "#/components/schemas/PciSegmentConfig"
platform:
$ref: "#/components/schemas/PlatformConfig"
tpm:
@ -683,6 +687,21 @@ components:
features:
$ref: "#/components/schemas/CpuFeatures"
PciSegmentConfig:
required:
- pci_segment
type: object
properties:
pci_segment:
type: integer
format: int16
mmio32_aperture_weight:
type: integer
format: int32
mmio64_aperture_weight:
type: integer
format: int32
PlatformConfig:
type: object
properties:

152
vmm/src/config.rs
View File

@ -98,6 +98,8 @@ pub enum Error {
ParseUserDevice(OptionParserError),
/// Missing socket for userspace device
ParseUserDeviceSocketMissing,
/// Error parsing pci segment options
ParsePciSegment(OptionParserError),
/// Failed parsing platform parameters
ParsePlatform(OptionParserError),
/// Failed parsing vDPA device
@ -170,6 +172,8 @@ pub enum ValidationError {
InvalidNumPciSegments(u16),
/// Invalid PCI segment id
InvalidPciSegment(u16),
/// Invalid PCI segment aperture weight
InvalidPciSegmentApertureWeight(u32),
/// Balloon too big
BalloonLargerThanRam(u64, u64),
/// On a IOMMU segment but not behind IOMMU
@ -279,6 +283,9 @@ impl fmt::Display for ValidationError {
InvalidPciSegment(pci_segment) => {
write!(f, "Invalid PCI segment id: {pci_segment}")
}
InvalidPciSegmentApertureWeight(aperture_weight) => {
write!(f, "Invalid PCI segment aperture weight: {aperture_weight}")
}
BalloonLargerThanRam(balloon_size, ram_size) => {
write!(
f,
@ -395,6 +402,7 @@ impl fmt::Display for Error {
ParseTdx(o) => write!(f, "Error parsing --tdx: {o}"),
#[cfg(feature = "tdx")]
FirmwarePathMissing => write!(f, "TDX firmware missing"),
ParsePciSegment(o) => write!(f, "Error parsing --pci-segment: {o}"),
ParsePlatform(o) => write!(f, "Error parsing --platform: {o}"),
ParseVdpa(o) => write!(f, "Error parsing --vdpa: {o}"),
ParseVdpaPathMissing => write!(f, "Error parsing --vdpa: path missing"),
@ -444,6 +452,7 @@ pub struct VmParams<'a> {
pub watchdog: bool,
#[cfg(feature = "guest_debug")]
pub gdb: bool,
pub pci_segments: Option<Vec<&'a str>>,
pub platform: Option<&'a str>,
pub tpm: Option<&'a str>,
#[cfg(feature = "igvm")]
@ -504,6 +513,9 @@ impl<'a> VmParams<'a> {
.get_many::<String>("numa")
.map(|x| x.map(|y| y as &str).collect());
let watchdog = args.get_flag("watchdog");
let pci_segments: Option<Vec<&str>> = args
.get_many::<String>("pci-segment")
.map(|x| x.map(|y| y as &str).collect());
let platform = args.get_one::<String>("platform").map(|x| x as &str);
#[cfg(feature = "guest_debug")]
let gdb = args.contains_id("gdb");
@ -542,6 +554,7 @@ impl<'a> VmParams<'a> {
watchdog,
#[cfg(feature = "guest_debug")]
gdb,
pci_segments,
platform,
tpm,
#[cfg(feature = "igvm")]
@ -678,6 +691,64 @@ impl CpusConfig {
}
}
impl PciSegmentConfig {
pub const SYNTAX: &'static str = "PCI Segment parameters \
\"pci_segment=<segment_id>,mmio32_aperture_weight=<scale>,mmio64_aperture_weight=<scale>\"";
pub fn parse(disk: &str) -> Result<Self> {
let mut parser = OptionParser::new();
parser
.add("mmio32_aperture_weight")
.add("mmio64_aperture_weight")
.add("pci_segment");
parser.parse(disk).map_err(Error::ParsePciSegment)?;
let pci_segment = parser
.convert("pci_segment")
.map_err(Error::ParsePciSegment)?
.unwrap_or_default();
let mmio32_aperture_weight = parser
.convert("mmio32_aperture_weight")
.map_err(Error::ParsePciSegment)?
.unwrap_or(DEFAULT_PCI_SEGMENT_APERTURE_WEIGHT);
let mmio64_aperture_weight = parser
.convert("mmio64_aperture_weight")
.map_err(Error::ParsePciSegment)?
.unwrap_or(DEFAULT_PCI_SEGMENT_APERTURE_WEIGHT);
Ok(PciSegmentConfig {
pci_segment,
mmio32_aperture_weight,
mmio64_aperture_weight,
})
}
pub fn validate(&self, vm_config: &VmConfig) -> ValidationResult<()> {
let num_pci_segments = match &vm_config.platform {
Some(platform_config) => platform_config.num_pci_segments,
None => 1,
};
if self.pci_segment >= num_pci_segments {
return Err(ValidationError::InvalidPciSegment(self.pci_segment));
}
if self.mmio32_aperture_weight == 0 {
return Err(ValidationError::InvalidPciSegmentApertureWeight(
self.mmio32_aperture_weight,
));
}
if self.mmio64_aperture_weight == 0 {
return Err(ValidationError::InvalidPciSegmentApertureWeight(
self.mmio64_aperture_weight,
));
}
Ok(())
}
}
impl PlatformConfig {
pub fn parse(platform: &str) -> Result<Self> {
let mut parser = OptionParser::new();
@ -2449,6 +2520,12 @@ impl VmConfig {
}
}
if let Some(pci_segments) = &self.pci_segments {
for pci_segment in pci_segments {
pci_segment.validate(self)?;
}
}
self.platform.as_ref().map(|p| p.validate()).transpose()?;
self.iommu |= self
.platform
@ -2557,6 +2634,16 @@ impl VmConfig {
vsock = Some(vsock_config);
}
let mut pci_segments: Option<Vec<PciSegmentConfig>> = None;
if let Some(pci_segment_list) = &vm_params.pci_segments {
let mut pci_segment_config_list = Vec::new();
for item in pci_segment_list.iter() {
let pci_segment_config = PciSegmentConfig::parse(item)?;
pci_segment_config_list.push(pci_segment_config);
}
pci_segments = Some(pci_segment_config_list);
}
let platform = vm_params.platform.map(PlatformConfig::parse).transpose()?;
#[cfg(target_arch = "x86_64")]
@ -2643,6 +2730,7 @@ impl VmConfig {
watchdog: vm_params.watchdog,
#[cfg(feature = "guest_debug")]
gdb,
pci_segments,
platform,
tpm,
preserved_fds: None,
@ -2764,6 +2852,7 @@ impl Clone for VmConfig {
#[cfg(target_arch = "x86_64")]
sgx_epc: self.sgx_epc.clone(),
numa: self.numa.clone(),
pci_segments: self.pci_segments.clone(),
platform: self.platform.clone(),
tpm: self.tpm.clone(),
preserved_fds: self
@ -2960,6 +3049,46 @@ mod tests {
Ok(())
}
#[test]
fn test_pci_segment_parsing() -> Result<()> {
assert_eq!(
PciSegmentConfig::parse("pci_segment=0")?,
PciSegmentConfig {
pci_segment: 0,
mmio32_aperture_weight: 1,
mmio64_aperture_weight: 1,
}
);
assert_eq!(
PciSegmentConfig::parse(
"pci_segment=0,mmio32_aperture_weight=1,mmio64_aperture_weight=1"
)?,
PciSegmentConfig {
pci_segment: 0,
mmio32_aperture_weight: 1,
mmio64_aperture_weight: 1,
}
);
assert_eq!(
PciSegmentConfig::parse("pci_segment=0,mmio32_aperture_weight=2")?,
PciSegmentConfig {
pci_segment: 0,
mmio32_aperture_weight: 2,
mmio64_aperture_weight: 1,
}
);
assert_eq!(
PciSegmentConfig::parse("pci_segment=0,mmio64_aperture_weight=2")?,
PciSegmentConfig {
pci_segment: 0,
mmio32_aperture_weight: 1,
mmio64_aperture_weight: 2,
}
);
Ok(())
}
fn disk_fixture() -> DiskConfig {
DiskConfig {
path: Some(PathBuf::from("/path/to_file")),
@ -3536,6 +3665,7 @@ mod tests {
watchdog: false,
#[cfg(feature = "guest_debug")]
gdb: false,
pci_segments: None,
platform: None,
tpm: None,
preserved_fds: None,
@ -3946,6 +4076,28 @@ mod tests {
Err(ValidationError::PciSegmentReused(1, 0, 1))
);
let mut invalid_config = valid_config.clone();
invalid_config.pci_segments = Some(vec![PciSegmentConfig {
pci_segment: 0,
mmio32_aperture_weight: 1,
mmio64_aperture_weight: 0,
}]);
assert_eq!(
invalid_config.validate(),
Err(ValidationError::InvalidPciSegmentApertureWeight(0))
);
let mut invalid_config = valid_config.clone();
invalid_config.pci_segments = Some(vec![PciSegmentConfig {
pci_segment: 0,
mmio32_aperture_weight: 0,
mmio64_aperture_weight: 1,
}]);
assert_eq!(
invalid_config.validate(),
Err(ValidationError::InvalidPciSegmentApertureWeight(0))
);
let mut invalid_config = valid_config.clone();
invalid_config.numa = Some(vec![
NumaConfig {

123
vmm/src/device_manager.rs
View File

@ -22,6 +22,7 @@ use crate::pci_segment::PciSegment;
use crate::seccomp_filters::{get_seccomp_filter, Thread};
use crate::serial_manager::{Error as SerialManagerError, SerialManager};
use crate::sigwinch_listener::start_sigwinch_listener;
use crate::vm_config::DEFAULT_PCI_SEGMENT_APERTURE_WEIGHT;
use crate::GuestRegionMmap;
use crate::PciDeviceInfo;
use crate::{device_node, DEVICE_MANAGER_SNAPSHOT_ID};
@ -969,6 +970,34 @@ pub struct DeviceManager {
mmio_regions: Arc<Mutex<Vec<MmioRegion>>>,
}
fn create_mmio_allocators(
start: u64,
end: u64,
num_pci_segments: u16,
weights: Vec<u32>,
alignment: u64,
) -> Vec<Arc<Mutex<AddressAllocator>>> {
let total_weight: u32 = weights.iter().sum();
// Start each PCI segment mmio range on an aligned boundary
let pci_segment_mmio_size = (end - start + 1) / (alignment * total_weight as u64) * alignment;
let mut mmio_allocators = vec![];
let mut i = 0;
for segment_id in 0..num_pci_segments as u64 {
let weight = weights[segment_id as usize] as u64;
let mmio_start = start + i * pci_segment_mmio_size;
let mmio_size = pci_segment_mmio_size * weight;
let allocator = Arc::new(Mutex::new(
AddressAllocator::new(GuestAddress(mmio_start), mmio_size).unwrap(),
));
mmio_allocators.push(allocator);
i += weight;
}
mmio_allocators
}
impl DeviceManager {
#[allow(clippy::too_many_arguments)]
pub fn new(
@ -1009,22 +1038,16 @@ impl DeviceManager {
1
};
let create_mmio_allocators = |start, end, num_pci_segments, alignment| {
// Start each PCI segment mmio range on an aligned boundary
let pci_segment_mmio_size =
(end - start + 1) / (alignment * num_pci_segments as u64) * alignment;
let mut mmio_allocators = vec![];
for i in 0..num_pci_segments as u64 {
let mmio_start = start + i * pci_segment_mmio_size;
let allocator = Arc::new(Mutex::new(
AddressAllocator::new(GuestAddress(mmio_start), pci_segment_mmio_size).unwrap(),
));
mmio_allocators.push(allocator)
let mut mmio32_aperture_weights: Vec<u32> =
std::iter::repeat(DEFAULT_PCI_SEGMENT_APERTURE_WEIGHT)
.take(num_pci_segments.into())
.collect();
if let Some(pci_segments) = &config.lock().unwrap().pci_segments {
for pci_segment in pci_segments.iter() {
mmio32_aperture_weights[pci_segment.pci_segment as usize] =
pci_segment.mmio32_aperture_weight
}
mmio_allocators
};
}
let start_of_mmio32_area = layout::MEM_32BIT_DEVICES_START.0;
let end_of_mmio32_area = layout::MEM_32BIT_DEVICES_START.0 + layout::MEM_32BIT_DEVICES_SIZE;
@ -1032,15 +1055,28 @@ impl DeviceManager {
start_of_mmio32_area,
end_of_mmio32_area,
num_pci_segments,
mmio32_aperture_weights,
4 << 10,
);
let mut mmio64_aperture_weights: Vec<u32> =
std::iter::repeat(DEFAULT_PCI_SEGMENT_APERTURE_WEIGHT)
.take(num_pci_segments.into())
.collect();
if let Some(pci_segments) = &config.lock().unwrap().pci_segments {
for pci_segment in pci_segments.iter() {
mmio64_aperture_weights[pci_segment.pci_segment as usize] =
pci_segment.mmio64_aperture_weight
}
}
let start_of_mmio64_area = memory_manager.lock().unwrap().start_of_device_area().0;
let end_of_mmio64_area = memory_manager.lock().unwrap().end_of_device_area().0;
let pci_mmio64_allocators = create_mmio_allocators(
start_of_mmio64_area,
end_of_mmio64_area,
num_pci_segments,
mmio64_aperture_weights,
4 << 30,
);
@ -4997,3 +5033,60 @@ impl Drop for DeviceManager {
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_create_mmio_allocators() {
let res = create_mmio_allocators(0x100000, 0x400000, 1, vec![1], 4 << 10);
assert_eq!(res.len(), 1);
assert_eq!(
res[0].lock().unwrap().base(),
vm_memory::GuestAddress(0x100000)
);
assert_eq!(
res[0].lock().unwrap().end(),
vm_memory::GuestAddress(0x3fffff)
);
let res = create_mmio_allocators(0x100000, 0x400000, 2, vec![1, 1], 4 << 10);
assert_eq!(res.len(), 2);
assert_eq!(
res[0].lock().unwrap().base(),
vm_memory::GuestAddress(0x100000)
);
assert_eq!(
res[0].lock().unwrap().end(),
vm_memory::GuestAddress(0x27ffff)
);
assert_eq!(
res[1].lock().unwrap().base(),
vm_memory::GuestAddress(0x280000)
);
assert_eq!(
res[1].lock().unwrap().end(),
vm_memory::GuestAddress(0x3fffff)
);
let res = create_mmio_allocators(0x100000, 0x400000, 2, vec![2, 1], 4 << 10);
assert_eq!(res.len(), 2);
assert_eq!(
res[0].lock().unwrap().base(),
vm_memory::GuestAddress(0x100000)
);
assert_eq!(
res[0].lock().unwrap().end(),
vm_memory::GuestAddress(0x2fffff)
);
assert_eq!(
res[1].lock().unwrap().base(),
vm_memory::GuestAddress(0x300000)
);
assert_eq!(
res[1].lock().unwrap().end(),
vm_memory::GuestAddress(0x3fffff)
);
}
}

1
vmm/src/lib.rs
View File

@ -2151,6 +2151,7 @@ mod unit_tests {
watchdog: false,
#[cfg(feature = "guest_debug")]
gdb: false,
pci_segments: None,
platform: None,
tpm: None,
preserved_fds: None,

17
vmm/src/vm_config.rs
View File

@ -94,6 +94,22 @@ pub struct PlatformConfig {
pub sev_snp: bool,
}
pub const DEFAULT_PCI_SEGMENT_APERTURE_WEIGHT: u32 = 1;
fn default_pci_segment_aperture_weight() -> u32 {
DEFAULT_PCI_SEGMENT_APERTURE_WEIGHT
}
#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)]
pub struct PciSegmentConfig {
#[serde(default)]
pub pci_segment: u16,
#[serde(default = "default_pci_segment_aperture_weight")]
pub mmio32_aperture_weight: u32,
#[serde(default = "default_pci_segment_aperture_weight")]
pub mmio64_aperture_weight: u32,
}
#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)]
pub struct MemoryZoneConfig {
pub id: String,
@ -620,6 +636,7 @@ pub struct VmConfig {
#[cfg(feature = "guest_debug")]
#[serde(default)]
pub gdb: bool,
pub pci_segments: Option<Vec<PciSegmentConfig>>,
pub platform: Option<PlatformConfig>,
pub tpm: Option<TpmConfig>,
// Preserved FDs are the ones that share the same life-time as its holding