numa: Add optional sgx_epc_sections field to NumaConfig

This new option allows the user to define a list of SGX EPC sections
attached to a specific NUMA node.

Signed-off-by: Sebastien Boeuf <sebastien.boeuf@intel.com>

docs/memory.md

@@ -347,11 +347,12 @@ struct NumaConfig {
     cpus: Option<Vec<u8>>,
     distances: Option<Vec<NumaDistance>>,
     memory_zones: Option<Vec<String>>,
+    sgx_epc_sections: Option<Vec<String>>,
 }
 ```
 
 ```
---numa <numa>	Settings related to a given NUMA node "guest_numa_id=<node_id>,cpus=<cpus_id>,distances=<list_of_distances_to_destination_nodes>,memory_zones=<list_of_memory_zones>"
+--numa <numa>	Settings related to a given NUMA node "guest_numa_id=<node_id>,cpus=<cpus_id>,distances=<list_of_distances_to_destination_nodes>,memory_zones=<list_of_memory_zones>,sgx_epc_sections=<list_of_sgx_epc_sections>"
 ```
 
 ### `guest_numa_id`
@@ -446,6 +447,23 @@ _Example_
 --numa guest_numa_id=0,memory_zones=mem0:mem2 guest_numa_id=1,memory_zones=mem1
 ```
 
+### `sgx_epc_sections`
+
+List of SGX EPC sections attached to the guest NUMA node identified by the
+`guest_numa_id` option. This allows for describing a list of SGX EPC sections
+which must be seen by the guest as belonging to the NUMA node `guest_numa_id`.
+
+Multiple values can be provided to define the list. Each value is a string
+referring to an existing SGX EPC section identifier. Values are separated from
+each other with the `:` separator.
+
+_Example_
+
+```
+--sgx-epc id=epc0,size=32M id=epc1,size=64M id=epc2,size=32M
+--numa guest_numa_id=0,sgx_epc_sections=epc1 guest_numa_id=1,sgx_epc_sections=epc0:epc2
+```
+
 ### PCI bus
 
 Cloud Hypervisor supports only one PCI bus, which is why it has been tied to

vmm/src/acpi.rs

@@ -103,12 +103,24 @@ impl MemoryAffinity {
         proximity_domain: u32,
         flags: MemAffinityFlags,
     ) -> Self {
-        let base_addr = region.start_addr().raw_value();
+        Self::from_range(
+            region.start_addr().raw_value(),
+            region.len(),
+            proximity_domain,
+            flags,
+        )
+    }
+
+    fn from_range(
+        base_addr: u64,
+        size: u64,
+        proximity_domain: u32,
+        flags: MemAffinityFlags,
+    ) -> Self {
         let base_addr_lo = (base_addr & 0xffff_ffff) as u32;
         let base_addr_hi = (base_addr >> 32) as u32;
-        let length = region.len() as u64;
-        let length_lo = (length & 0xffff_ffff) as u32;
-        let length_hi = (length >> 32) as u32;
+        let length_lo = (size & 0xffff_ffff) as u32;
+        let length_hi = (size >> 32) as u32;
 
         MemoryAffinity {
             type_: 1,
@@ -254,6 +266,16 @@ fn create_srat_table(numa_nodes: &NumaNodes) -> Sdt {
             ))
         }
 
+        #[cfg(target_arch = "x86_64")]
+        for section in node.sgx_epc_sections() {
+            srat.append(MemoryAffinity::from_range(
+                section.start().raw_value(),
+                section.size(),
+                proximity_domain,
+                MemAffinityFlags::ENABLE,
+            ))
+        }
+
         for cpu in node.cpus() {
             let x2apic_id = *cpu as u32;
 

vmm/src/api/openapi/cloud-hypervisor.yaml

@@ -898,6 +898,10 @@ components:
           type: array
           items:
             type: string
+        sgx_epc_sections:
+          type: array
+          items:
+            type: string
 
     VmResize:
       type: object

vmm/src/config.rs

@@ -218,6 +218,7 @@ impl fmt::Display for Error {
             ParseRestore(o) => write!(f, "Error parsing --restore: {}", o),
             #[cfg(target_arch = "x86_64")]
             ParseSgxEpc(o) => write!(f, "Error parsing --sgx-epc: {}", o),
+            #[cfg(target_arch = "x86_64")]
             ParseSgxEpcIdMissing => write!(f, "Error parsing --sgx-epc: id missing"),
             ParseNuma(o) => write!(f, "Error parsing --numa: {}", o),
             ParseRestoreSourceUrlMissing => {
@@ -1647,19 +1648,23 @@ pub struct NumaConfig {
     pub distances: Option<Vec<NumaDistance>>,
     #[serde(default)]
     pub memory_zones: Option<Vec<String>>,
+    #[cfg(target_arch = "x86_64")]
+    #[serde(default)]
+    pub sgx_epc_sections: Option<Vec<String>>,
 }
 
 impl NumaConfig {
     pub const SYNTAX: &'static str = "Settings related to a given NUMA node \
         \"guest_numa_id=<node_id>,cpus=<cpus_id>,distances=<list_of_distances_to_destination_nodes>,\
-        memory_zones=<list_of_memory_zones>\"";
+        memory_zones=<list_of_memory_zones>,sgx_epc_sections=<list_of_sgx_epc_sections>\"";
     pub fn parse(numa: &str) -> Result<Self> {
         let mut parser = OptionParser::new();
         parser
             .add("guest_numa_id")
             .add("cpus")
             .add("distances")
-            .add("memory_zones");
+            .add("memory_zones")
+            .add("sgx_epc_sections");
         parser.parse(numa).map_err(Error::ParseNuma)?;
 
         let guest_numa_id = parser
@@ -1685,12 +1690,19 @@ impl NumaConfig {
             .convert::<StringList>("memory_zones")
             .map_err(Error::ParseNuma)?
             .map(|v| v.0);
+        #[cfg(target_arch = "x86_64")]
+        let sgx_epc_sections = parser
+            .convert::<StringList>("sgx_epc_sections")
+            .map_err(Error::ParseNuma)?
+            .map(|v| v.0);
 
         Ok(NumaConfig {
             guest_numa_id,
             cpus,
             distances,
             memory_zones,
+            #[cfg(target_arch = "x86_64")]
+            sgx_epc_sections,
         })
     }
 }

vmm/src/vm.rs

@@ -33,6 +33,8 @@ use anyhow::anyhow;
 use arch::get_host_cpu_phys_bits;
 #[cfg(feature = "tdx")]
 use arch::x86_64::tdx::TdvfSection;
+#[cfg(target_arch = "x86_64")]
+use arch::x86_64::SgxEpcSection;
 use arch::EntryPoint;
 use devices::AcpiNotificationFlags;
 use hypervisor::vm::{HypervisorVmError, VmmOps};
@@ -269,6 +271,8 @@ pub struct NumaNode {
     cpus: Vec<u8>,
     distances: BTreeMap<u32, u8>,
     memory_zones: Vec<String>,
+    #[cfg(target_arch = "x86_64")]
+    sgx_epc_sections: Vec<SgxEpcSection>,
 }
 
 impl NumaNode {
@@ -291,6 +295,11 @@ impl NumaNode {
     pub fn memory_zones(&self) -> &Vec<String> {
         &self.memory_zones
     }
+
+    #[cfg(target_arch = "x86_64")]
+    pub fn sgx_epc_sections(&self) -> &Vec<SgxEpcSection> {
+        &self.sgx_epc_sections
+    }
 }
 
 pub type NumaNodes = BTreeMap<u32, NumaNode>;
@@ -673,6 +682,24 @@ impl Vm {
                     }
                 }
 
+                #[cfg(target_arch = "x86_64")]
+                if let Some(sgx_epc_sections) = &config.sgx_epc_sections {
+                    if let Some(sgx_epc_region) = mm.sgx_epc_region() {
+                        let mm_sections = sgx_epc_region.epc_sections();
+                        for sgx_epc_section in sgx_epc_sections.iter() {
+                            if let Some(mm_section) = mm_sections.get(sgx_epc_section) {
+                                node.sgx_epc_sections.push(mm_section.clone());
+                            } else {
+                                error!("Unknown SGX EPC section '{}'", sgx_epc_section);
+                                return Err(Error::InvalidNumaConfig);
+                            }
+                        }
+                    } else {
+                        error!("Missing SGX EPC region");
+                        return Err(Error::InvalidNumaConfig);
+                    }
+                }
+
                 numa_nodes.insert(config.guest_numa_id, node);
             }
         }