arch, hypervisor, vmm: Patch CPUID subleaves to expose EPC sections

The support for SGX is exposed to the guest through CPUID 0x12. KVM
passes static subleaves 0 and 1 from the host to the guest, without
needing any modification from the VMM itself.

But SGX also relies on dynamic subleaves 2 through N, used for
describing each EPC section. This is not handled by KVM, which means
the VMM is in charge of setting each subleaf starting from index 2
up to index N, depending on the number of EPC sections.

These subleaves 2 through N are not listed as part of the supported
CPUID entries from KVM. But it's important to set them as long as index
0 and 1 are present and indicate that SGX is supported.

Signed-off-by: Sebastien Boeuf <sebastien.boeuf@intel.com>

Cargo.lock

@@ -65,6 +65,7 @@ dependencies = [
  "kvm-ioctls",
  "libc",
  "linux-loader",
+ "log 0.4.8",
  "rand 0.7.3",
  "vm-memory",
 ]

arch/Cargo.toml

@@ -14,6 +14,7 @@ hypervisor = { path = "../hypervisor" }
 kvm-bindings = { git = "https://github.com/cloud-hypervisor/kvm-bindings", branch = "ch" }
 kvm-ioctls = { git = "https://github.com/cloud-hypervisor/kvm-ioctls", branch = "ch" }
 libc = "0.2.72"
+log = "0.4.8"
 vm-memory = { version = "0.2.1", features = ["backend-mmap"] }
 acpi_tables = { path = "../acpi_tables", optional = true }
 arch_gen = { path = "../arch_gen" }

arch/src/lib.rs

@@ -19,13 +19,13 @@ extern crate hypervisor;
 extern crate kvm_bindings;
 extern crate kvm_ioctls;
 extern crate libc;
-
-extern crate vm_memory;
-
+#[macro_use]
+extern crate log;
 #[cfg(feature = "acpi")]
 extern crate acpi_tables;
 extern crate arch_gen;
 extern crate linux_loader;
+extern crate vm_memory;
 
 use std::fmt;
 use std::result;

arch/src/x86_64/mod.rs

@@ -15,7 +15,7 @@ mod mptable;
 pub mod regs;
 use crate::InitramfsConfig;
 use crate::RegionType;
-use hypervisor::CpuId;
+use hypervisor::{CpuId, CpuIdEntry, CPUID_FLAG_VALID_INDEX};
 use linux_loader::loader::bootparam::{boot_params, setup_header};
 use linux_loader::loader::elf::start_info::{
     hvm_memmap_table_entry, hvm_modlist_entry, hvm_start_info,
@@ -156,7 +156,7 @@ pub enum Error {
     /// Error configuring the MSR registers
     MSRSConfiguration(regs::Error),
 
-    /// The call to KVM_SET_CPUID2 failed.
+    /// Failed to set supported CPUs.
     SetSupportedCpusFailed(anyhow::Error),
 
     /// Cannot set the local interruption due to bad configuration.
@@ -164,6 +164,15 @@ pub enum Error {
 
     /// Error setting up SMBIOS table
     SmbiosSetup(smbios::Error),
+
+    /// Could not find any SGX EPC section
+    NoSgxEpcSection,
+
+    /// Missing SGX CPU feature
+    MissingSgxFeature,
+
+    /// Missing SGX_LC CPU feature
+    MissingSgxLaunchControlFeature,
 }
 
 impl From<Error> for super::Error {
@@ -201,8 +210,10 @@ impl CpuidPatch {
     ) {
         let entries = cpuid.as_mut_slice();
 
+        let mut entry_found = false;
         for entry in entries.iter_mut() {
             if entry.function == function && (index == None || index.unwrap() == entry.index) {
+                entry_found = true;
                 match reg {
                     CpuidReg::EAX => {
                         entry.eax = value;
@@ -219,6 +230,38 @@ impl CpuidPatch {
                 }
             }
         }
+
+        if entry_found {
+            return;
+        }
+
+        // Entry not found, so let's add it.
+        if let Some(index) = index {
+            let mut entry = CpuIdEntry {
+                function,
+                index,
+                flags: CPUID_FLAG_VALID_INDEX,
+                ..Default::default()
+            };
+            match reg {
+                CpuidReg::EAX => {
+                    entry.eax = value;
+                }
+                CpuidReg::EBX => {
+                    entry.ebx = value;
+                }
+                CpuidReg::ECX => {
+                    entry.ecx = value;
+                }
+                CpuidReg::EDX => {
+                    entry.edx = value;
+                }
+            }
+
+            if let Err(e) = cpuid.push(entry) {
+                error!("Failed adding new CPUID entry: {:?}", e);
+            }
+        }
     }
 
     pub fn patch_cpuid(cpuid: &mut CpuId, patches: Vec<CpuidPatch>) {
@@ -246,6 +289,41 @@ impl CpuidPatch {
             }
         }
     }
+
+    pub fn is_feature_enabled(
+        cpuid: &CpuId,
+        function: u32,
+        index: u32,
+        reg: CpuidReg,
+        feature_bit: usize,
+    ) -> bool {
+        let entries = cpuid.as_slice();
+        let mask = 1 << feature_bit;
+
+        for entry in entries.iter() {
+            if entry.function == function && entry.index == index {
+                let reg_val: u32;
+                match reg {
+                    CpuidReg::EAX => {
+                        reg_val = entry.eax;
+                    }
+                    CpuidReg::EBX => {
+                        reg_val = entry.ebx;
+                    }
+                    CpuidReg::ECX => {
+                        reg_val = entry.ecx;
+                    }
+                    CpuidReg::EDX => {
+                        reg_val = entry.edx;
+                    }
+                }
+
+                return (reg_val & mask) == mask;
+            }
+        }
+
+        false
+    }
 }
 
 pub fn configure_vcpu(
@@ -730,6 +808,52 @@ pub fn update_cpuid_topology(
     CpuidPatch::set_cpuid_reg(cpuid, 0x1f, Some(2), CpuidReg::ECX, 5 << 8);
 }
 
+// The goal is to update the CPUID sub-leaves to reflect the number of EPC
+// sections exposed to the guest.
+pub fn update_cpuid_sgx(cpuid: &mut CpuId, epc_sections: Vec<SgxEpcSection>) -> Result<(), Error> {
+    // Something's wrong if there's no EPC section.
+    if epc_sections.is_empty() {
+        return Err(Error::NoSgxEpcSection);
+    }
+    // We can't go further if the hypervisor does not support SGX feature.
+    if !CpuidPatch::is_feature_enabled(cpuid, 0x7, 0, CpuidReg::EBX, 2) {
+        return Err(Error::MissingSgxFeature);
+    }
+    // We can't go further if the hypervisor does not support SGX_LC feature.
+    if !CpuidPatch::is_feature_enabled(cpuid, 0x7, 0, CpuidReg::ECX, 30) {
+        return Err(Error::MissingSgxLaunchControlFeature);
+    }
+
+    // Get host CPUID for leaf 0x12, subleaf 0x2. This is to retrieve EPC
+    // properties such as confidentiality and integrity.
+    let leaf = unsafe { std::arch::x86_64::__cpuid_count(0x12, 0x2) };
+
+    for (i, epc_section) in epc_sections.iter().enumerate() {
+        let subleaf_idx = i + 2;
+        let start = epc_section.start().raw_value();
+        let size = epc_section.size() as u64;
+        let eax = (start & 0xffff_f000) as u32 | 0x1;
+        let ebx = (start >> 32) as u32;
+        let ecx = (size & 0xffff_f000) as u32 | (leaf.ecx & 0xf);
+        let edx = (size >> 32) as u32;
+        // CPU Topology leaf 0x12
+        CpuidPatch::set_cpuid_reg(cpuid, 0x12, Some(subleaf_idx as u32), CpuidReg::EAX, eax);
+        CpuidPatch::set_cpuid_reg(cpuid, 0x12, Some(subleaf_idx as u32), CpuidReg::EBX, ebx);
+        CpuidPatch::set_cpuid_reg(cpuid, 0x12, Some(subleaf_idx as u32), CpuidReg::ECX, ecx);
+        CpuidPatch::set_cpuid_reg(cpuid, 0x12, Some(subleaf_idx as u32), CpuidReg::EDX, edx);
+    }
+
+    // Add one NULL entry to terminate the dynamic list
+    let subleaf_idx = epc_sections.len() + 2;
+    // CPU Topology leaf 0x12
+    CpuidPatch::set_cpuid_reg(cpuid, 0x12, Some(subleaf_idx as u32), CpuidReg::EAX, 0);
+    CpuidPatch::set_cpuid_reg(cpuid, 0x12, Some(subleaf_idx as u32), CpuidReg::EBX, 0);
+    CpuidPatch::set_cpuid_reg(cpuid, 0x12, Some(subleaf_idx as u32), CpuidReg::ECX, 0);
+    CpuidPatch::set_cpuid_reg(cpuid, 0x12, Some(subleaf_idx as u32), CpuidReg::EDX, 0);
+
+    Ok(())
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

hypervisor/src/kvm/mod.rs

@@ -31,7 +31,8 @@ use x86_64::{
 
 #[cfg(target_arch = "x86_64")]
 pub use x86_64::{
-    CpuId, ExtendedControlRegisters, LapicState, MsrEntries, VcpuKvmState as CpuState, Xsave,
+    CpuId, CpuIdEntry, ExtendedControlRegisters, LapicState, MsrEntries, VcpuKvmState as CpuState,
+    Xsave, CPUID_FLAG_VALID_INDEX,
 };
 
 #[cfg(target_arch = "x86_64")]

hypervisor/src/kvm/x86_64/mod.rs

@@ -18,13 +18,14 @@ use serde_derive::{Deserialize, Serialize};
 /// Export generically-named wrappers of kvm-bindings for Unix-based platforms
 ///
 pub use {
-    kvm_bindings::kvm_dtable as DescriptorTable, kvm_bindings::kvm_fpu as FpuState,
-    kvm_bindings::kvm_lapic_state as LapicState, kvm_bindings::kvm_mp_state as MpState,
-    kvm_bindings::kvm_msr_entry as MsrEntry, kvm_bindings::kvm_regs as StandardRegisters,
-    kvm_bindings::kvm_segment as SegmentRegister, kvm_bindings::kvm_sregs as SpecialRegisters,
-    kvm_bindings::kvm_vcpu_events as VcpuEvents,
+    kvm_bindings::kvm_cpuid_entry2 as CpuIdEntry, kvm_bindings::kvm_dtable as DescriptorTable,
+    kvm_bindings::kvm_fpu as FpuState, kvm_bindings::kvm_lapic_state as LapicState,
+    kvm_bindings::kvm_mp_state as MpState, kvm_bindings::kvm_msr_entry as MsrEntry,
+    kvm_bindings::kvm_regs as StandardRegisters, kvm_bindings::kvm_segment as SegmentRegister,
+    kvm_bindings::kvm_sregs as SpecialRegisters, kvm_bindings::kvm_vcpu_events as VcpuEvents,
     kvm_bindings::kvm_xcrs as ExtendedControlRegisters, kvm_bindings::kvm_xsave as Xsave,
     kvm_bindings::CpuId, kvm_bindings::MsrList, kvm_bindings::Msrs as MsrEntries,
+    kvm_bindings::KVM_CPUID_FLAG_SIGNIFCANT_INDEX as CPUID_FLAG_VALID_INDEX,
 };
 
 pub const KVM_TSS_ADDRESS: GuestAddress = GuestAddress(0xfffb_d000);

vmm/src/cpu.rs

@@ -22,6 +22,8 @@ use acpi_tables::{aml, aml::Aml, sdt::SDT};
 use anyhow::anyhow;
 #[cfg(feature = "acpi")]
 use arch::layout;
+#[cfg(target_arch = "x86_64")]
+use arch::x86_64::SgxEpcSection;
 use arch::EntryPoint;
 #[cfg(target_arch = "x86_64")]
 use arch::{CpuidPatch, CpuidReg};
@@ -589,9 +591,17 @@ impl CpuManager {
         let mut vcpu_states = Vec::with_capacity(usize::from(config.max_vcpus));
         vcpu_states.resize_with(usize::from(config.max_vcpus), VcpuState::default);
 
-        let device_manager = device_manager.lock().unwrap();
         #[cfg(target_arch = "x86_64")]
-        let cpuid = CpuManager::patch_cpuid(hypervisor, &config.topology)?;
+        let sgx_epc_sections =
+            if let Some(sgx_epc_region) = memory_manager.lock().unwrap().sgx_epc_region() {
+                Some(sgx_epc_region.epc_sections().clone())
+            } else {
+                None
+            };
+        #[cfg(target_arch = "x86_64")]
+        let cpuid = CpuManager::patch_cpuid(hypervisor, &config.topology, sgx_epc_sections)?;
+
+        let device_manager = device_manager.lock().unwrap();
         let cpu_manager = Arc::new(Mutex::new(CpuManager {
             config: config.clone(),
             #[cfg(target_arch = "x86_64")]
@@ -633,6 +643,7 @@ impl CpuManager {
     fn patch_cpuid(
         hypervisor: Arc<dyn hypervisor::Hypervisor>,
         topology: &Option<CpuTopology>,
+        sgx_epc_sections: Option<Vec<SgxEpcSection>>,
     ) -> Result<CpuId> {
         let mut cpuid_patches = Vec::new();
 
@@ -674,6 +685,10 @@ impl CpuManager {
             );
         }
 
+        if let Some(sgx_epc_sections) = sgx_epc_sections {
+            arch::x86_64::update_cpuid_sgx(&mut cpuid, sgx_epc_sections).unwrap();
+        }
+
         Ok(cpuid)
     }