From 6b710209b12b35e8c0eb26dcfcf074fca1f2f64f Mon Sep 17 00:00:00 2001 From: Sebastien Boeuf Date: Fri, 9 Jul 2021 11:22:35 +0200 Subject: [PATCH] numa: Add optional `sgx_epc_sections` field to NumaConfig This new option allows the user to define a list of SGX EPC sections attached to a specific NUMA node. Signed-off-by: Sebastien Boeuf --- docs/memory.md | 20 ++++++++++++++- vmm/src/acpi.rs | 30 ++++++++++++++++++++--- vmm/src/api/openapi/cloud-hypervisor.yaml | 4 +++ vmm/src/config.rs | 16 ++++++++++-- vmm/src/vm.rs | 27 ++++++++++++++++++++ 5 files changed, 90 insertions(+), 7 deletions(-) diff --git a/docs/memory.md b/docs/memory.md index 695cf77d5..268ab21b2 100644 --- a/docs/memory.md +++ b/docs/memory.md @@ -347,11 +347,12 @@ struct NumaConfig { cpus: Option>, distances: Option>, memory_zones: Option>, + sgx_epc_sections: Option>, } ``` ``` ---numa Settings related to a given NUMA node "guest_numa_id=,cpus=,distances=,memory_zones=" +--numa Settings related to a given NUMA node "guest_numa_id=,cpus=,distances=,memory_zones=,sgx_epc_sections=" ``` ### `guest_numa_id` @@ -446,6 +447,23 @@ _Example_ --numa guest_numa_id=0,memory_zones=mem0:mem2 guest_numa_id=1,memory_zones=mem1 ``` +### `sgx_epc_sections` + +List of SGX EPC sections attached to the guest NUMA node identified by the +`guest_numa_id` option. This allows for describing a list of SGX EPC sections +which must be seen by the guest as belonging to the NUMA node `guest_numa_id`. + +Multiple values can be provided to define the list. Each value is a string +referring to an existing SGX EPC section identifier. Values are separated from +each other with the `:` separator. + +_Example_ + +``` +--sgx-epc id=epc0,size=32M id=epc1,size=64M id=epc2,size=32M +--numa guest_numa_id=0,sgx_epc_sections=epc1 guest_numa_id=1,sgx_epc_sections=epc0:epc2 +``` + ### PCI bus Cloud Hypervisor supports only one PCI bus, which is why it has been tied to diff --git a/vmm/src/acpi.rs b/vmm/src/acpi.rs index 9760d142f..bd8605c9a 100644 --- a/vmm/src/acpi.rs +++ b/vmm/src/acpi.rs @@ -103,12 +103,24 @@ impl MemoryAffinity { proximity_domain: u32, flags: MemAffinityFlags, ) -> Self { - let base_addr = region.start_addr().raw_value(); + Self::from_range( + region.start_addr().raw_value(), + region.len(), + proximity_domain, + flags, + ) + } + + fn from_range( + base_addr: u64, + size: u64, + proximity_domain: u32, + flags: MemAffinityFlags, + ) -> Self { let base_addr_lo = (base_addr & 0xffff_ffff) as u32; let base_addr_hi = (base_addr >> 32) as u32; - let length = region.len() as u64; - let length_lo = (length & 0xffff_ffff) as u32; - let length_hi = (length >> 32) as u32; + let length_lo = (size & 0xffff_ffff) as u32; + let length_hi = (size >> 32) as u32; MemoryAffinity { type_: 1, @@ -254,6 +266,16 @@ fn create_srat_table(numa_nodes: &NumaNodes) -> Sdt { )) } + #[cfg(target_arch = "x86_64")] + for section in node.sgx_epc_sections() { + srat.append(MemoryAffinity::from_range( + section.start().raw_value(), + section.size(), + proximity_domain, + MemAffinityFlags::ENABLE, + )) + } + for cpu in node.cpus() { let x2apic_id = *cpu as u32; diff --git a/vmm/src/api/openapi/cloud-hypervisor.yaml b/vmm/src/api/openapi/cloud-hypervisor.yaml index 7349a3292..ffb02a32e 100644 --- a/vmm/src/api/openapi/cloud-hypervisor.yaml +++ b/vmm/src/api/openapi/cloud-hypervisor.yaml @@ -898,6 +898,10 @@ components: type: array items: type: string + sgx_epc_sections: + type: array + items: + type: string VmResize: type: object diff --git a/vmm/src/config.rs b/vmm/src/config.rs index d5a76df34..67f24623d 100644 --- a/vmm/src/config.rs +++ b/vmm/src/config.rs @@ -218,6 +218,7 @@ impl fmt::Display for Error { ParseRestore(o) => write!(f, "Error parsing --restore: {}", o), #[cfg(target_arch = "x86_64")] ParseSgxEpc(o) => write!(f, "Error parsing --sgx-epc: {}", o), + #[cfg(target_arch = "x86_64")] ParseSgxEpcIdMissing => write!(f, "Error parsing --sgx-epc: id missing"), ParseNuma(o) => write!(f, "Error parsing --numa: {}", o), ParseRestoreSourceUrlMissing => { @@ -1647,19 +1648,23 @@ pub struct NumaConfig { pub distances: Option>, #[serde(default)] pub memory_zones: Option>, + #[cfg(target_arch = "x86_64")] + #[serde(default)] + pub sgx_epc_sections: Option>, } impl NumaConfig { pub const SYNTAX: &'static str = "Settings related to a given NUMA node \ \"guest_numa_id=,cpus=,distances=,\ - memory_zones=\""; + memory_zones=,sgx_epc_sections=\""; pub fn parse(numa: &str) -> Result { let mut parser = OptionParser::new(); parser .add("guest_numa_id") .add("cpus") .add("distances") - .add("memory_zones"); + .add("memory_zones") + .add("sgx_epc_sections"); parser.parse(numa).map_err(Error::ParseNuma)?; let guest_numa_id = parser @@ -1685,12 +1690,19 @@ impl NumaConfig { .convert::("memory_zones") .map_err(Error::ParseNuma)? .map(|v| v.0); + #[cfg(target_arch = "x86_64")] + let sgx_epc_sections = parser + .convert::("sgx_epc_sections") + .map_err(Error::ParseNuma)? + .map(|v| v.0); Ok(NumaConfig { guest_numa_id, cpus, distances, memory_zones, + #[cfg(target_arch = "x86_64")] + sgx_epc_sections, }) } } diff --git a/vmm/src/vm.rs b/vmm/src/vm.rs index a52296543..88712d575 100644 --- a/vmm/src/vm.rs +++ b/vmm/src/vm.rs @@ -33,6 +33,8 @@ use anyhow::anyhow; use arch::get_host_cpu_phys_bits; #[cfg(feature = "tdx")] use arch::x86_64::tdx::TdvfSection; +#[cfg(target_arch = "x86_64")] +use arch::x86_64::SgxEpcSection; use arch::EntryPoint; use devices::AcpiNotificationFlags; use hypervisor::vm::{HypervisorVmError, VmmOps}; @@ -269,6 +271,8 @@ pub struct NumaNode { cpus: Vec, distances: BTreeMap, memory_zones: Vec, + #[cfg(target_arch = "x86_64")] + sgx_epc_sections: Vec, } impl NumaNode { @@ -291,6 +295,11 @@ impl NumaNode { pub fn memory_zones(&self) -> &Vec { &self.memory_zones } + + #[cfg(target_arch = "x86_64")] + pub fn sgx_epc_sections(&self) -> &Vec { + &self.sgx_epc_sections + } } pub type NumaNodes = BTreeMap; @@ -673,6 +682,24 @@ impl Vm { } } + #[cfg(target_arch = "x86_64")] + if let Some(sgx_epc_sections) = &config.sgx_epc_sections { + if let Some(sgx_epc_region) = mm.sgx_epc_region() { + let mm_sections = sgx_epc_region.epc_sections(); + for sgx_epc_section in sgx_epc_sections.iter() { + if let Some(mm_section) = mm_sections.get(sgx_epc_section) { + node.sgx_epc_sections.push(mm_section.clone()); + } else { + error!("Unknown SGX EPC section '{}'", sgx_epc_section); + return Err(Error::InvalidNumaConfig); + } + } + } else { + error!("Missing SGX EPC region"); + return Err(Error::InvalidNumaConfig); + } + } + numa_nodes.insert(config.guest_numa_id, node); } }