From 84cf12d86a83824c3e7258b8741a27b2fa693cf3 Mon Sep 17 00:00:00 2001 From: Sebastien Boeuf Date: Wed, 8 Jul 2020 11:53:28 +0200 Subject: [PATCH] arch, vmm: Create SGX virtual EPC sections from MemoryManager Based on the presence of one or multiple SGX EPC sections from the VM configuration, the MemoryManager will allocate a contiguous block of guest address space to hold the entire EPC region. Within this EPC region, each EPC section is memory mapped. Signed-off-by: Sebastien Boeuf --- arch/src/x86_64/mod.rs | 46 +++++++++++++++ vmm/src/memory_manager.rs | 119 ++++++++++++++++++++++++++++++++++++++ vmm/src/vm.rs | 11 ++++ 3 files changed, 176 insertions(+) diff --git a/arch/src/x86_64/mod.rs b/arch/src/x86_64/mod.rs index c9aece6aa..cfa47d76d 100644 --- a/arch/src/x86_64/mod.rs +++ b/arch/src/x86_64/mod.rs @@ -58,6 +58,52 @@ pub struct EntryPoint { const E820_RAM: u32 = 1; const E820_RESERVED: u32 = 2; +#[derive(Clone)] +pub struct SgxEpcSection { + start: GuestAddress, + size: GuestUsize, +} + +impl SgxEpcSection { + pub fn new(start: GuestAddress, size: GuestUsize) -> Self { + SgxEpcSection { start, size } + } + pub fn start(&self) -> GuestAddress { + self.start + } + pub fn size(&self) -> GuestUsize { + self.size + } +} + +pub struct SgxEpcRegion { + start: GuestAddress, + size: GuestUsize, + epc_sections: Vec, +} + +impl SgxEpcRegion { + pub fn new(start: GuestAddress, size: GuestUsize) -> Self { + SgxEpcRegion { + start, + size, + epc_sections: Vec::new(), + } + } + pub fn start(&self) -> GuestAddress { + self.start + } + pub fn size(&self) -> GuestUsize { + self.size + } + pub fn epc_sections(&self) -> &Vec { + &self.epc_sections + } + pub fn push(&mut self, epc_section: SgxEpcSection) { + self.epc_sections.push(epc_section); + } +} + // This is a workaround to the Rust enforcement specifying that any implementation of a foreign // trait (in this case `DataInit`) where: // * the type that is implementing the trait is foreign or diff --git a/vmm/src/memory_manager.rs b/vmm/src/memory_manager.rs index e9eb8fc12..ac57dea55 100644 --- a/vmm/src/memory_manager.rs +++ b/vmm/src/memory_manager.rs @@ -3,20 +3,28 @@ // SPDX-License-Identifier: Apache-2.0 // extern crate hypervisor; +#[cfg(target_arch = "x86_64")] +use crate::config::SgxEpcConfig; use crate::config::{HotplugMethod, MemoryConfig}; use crate::MEMORY_MANAGER_SNAPSHOT_ID; #[cfg(feature = "acpi")] use acpi_tables::{aml, aml::Aml}; use anyhow::anyhow; +#[cfg(target_arch = "x86_64")] +use arch::x86_64::{SgxEpcRegion, SgxEpcSection}; use arch::{get_host_cpu_phys_bits, layout, RegionType}; #[cfg(target_arch = "x86_64")] use devices::ioapic; use devices::BusDevice; +#[cfg(target_arch = "x86_64")] +use libc::{MAP_NORESERVE, MAP_POPULATE, MAP_SHARED, PROT_READ, PROT_WRITE}; use std::convert::TryInto; use std::ffi; use std::fs::{File, OpenOptions}; use std::io; +#[cfg(target_arch = "x86_64")] +use std::os::unix::io::AsRawFd; use std::os::unix::io::{FromRawFd, RawFd}; use std::path::PathBuf; use std::result; @@ -71,6 +79,8 @@ pub struct MemoryManager { shared: bool, hugepages: bool, balloon: Option>>, + #[cfg(target_arch = "x86_64")] + sgx_epc_region: Option, } #[derive(Debug)] @@ -126,6 +136,26 @@ pub enum Error { /// Failed to virtio-balloon resize VirtioBalloonResizeFail(virtio_devices::balloon::Error), + + /// Invalid SGX EPC section size + #[cfg(target_arch = "x86_64")] + EpcSectionSizeInvalid, + + /// Failed allocating SGX EPC region + #[cfg(target_arch = "x86_64")] + SgxEpcRangeAllocation, + + /// Failed opening SGX virtual EPC device + #[cfg(target_arch = "x86_64")] + SgxVirtEpcOpen(io::Error), + + /// Failed setting the SGX virtual EPC section size + #[cfg(target_arch = "x86_64")] + SgxVirtEpcFileSetLen(io::Error), + + /// Failed creating a new MmapRegion instance. + #[cfg(target_arch = "x86_64")] + NewMmapRegion(vm_memory::mmap::MmapRegionError), } const ENABLE_FLAG: usize = 0; @@ -343,6 +373,8 @@ impl MemoryManager { shared: config.shared, hugepages: config.hugepages, balloon: None, + #[cfg(target_arch = "x86_64")] + sgx_epc_region: None, })); guest_memory.memory().with_regions(|_, region| { @@ -838,6 +870,93 @@ impl MemoryManager { } Ok(region) } + + #[cfg(target_arch = "x86_64")] + pub fn setup_sgx(&mut self, sgx_epc_config: Vec) -> Result<(), Error> { + // Go over each EPC section and verify its size is a 4k multiple. At + // the same time, calculate the total size needed for the contiguous + // EPC region. + let mut epc_region_size = 0; + for epc_section in sgx_epc_config.iter() { + if epc_section.size == 0 { + return Err(Error::EpcSectionSizeInvalid); + } + if epc_section.size & 0x0fff != 0 { + return Err(Error::EpcSectionSizeInvalid); + } + + epc_region_size += epc_section.size; + } + + // Now that we know about the total size for the EPC region, we can + // proceed with the allocation of the entire range. The EPC region + // must be 4kiB aligned. + let epc_region_start = self + .allocator + .lock() + .unwrap() + .allocate_mmio_addresses(None, epc_region_size as GuestUsize, Some(0x1000)) + .ok_or(Error::SgxEpcRangeAllocation)?; + + let mut sgx_epc_region = SgxEpcRegion::new(epc_region_start, epc_region_size as GuestUsize); + + // Each section can be memory mapped into the allocated region. + let mut epc_section_start = epc_region_start.raw_value(); + for epc_section in sgx_epc_config.iter() { + let file = OpenOptions::new() + .read(true) + .write(true) + .open("/dev/sgx/virt_epc") + .map_err(Error::SgxVirtEpcOpen)?; + + let prot = PROT_READ | PROT_WRITE; + let mut flags = MAP_NORESERVE | MAP_SHARED; + if epc_section.prefault { + flags |= MAP_POPULATE; + } + + // We can't use the vm-memory crate to perform the memory mapping + // here as it would try to ensure the size of the backing file is + // matching the size of the expected mapping. The /dev/sgx/virt_epc + // device does not work that way, it provides a file descriptor + // which is not matching the mapping size, as it's a just a way to + // let KVM know that an EPC section is being created for the guest. + let host_addr = unsafe { + libc::mmap( + std::ptr::null_mut(), + epc_section.size as usize, + prot, + flags, + file.as_raw_fd(), + 0 as libc::off_t, + ) + } as u64; + + let _mem_slot = self.create_userspace_mapping( + epc_section_start, + epc_section.size, + host_addr, + false, + false, + )?; + + sgx_epc_region.push(SgxEpcSection::new( + GuestAddress(epc_section_start), + epc_section.size as GuestUsize, + )); + + epc_section_start += epc_section.size; + } + + self.sgx_epc_region = Some(sgx_epc_region); + + Ok(()) + } + + #[cfg(target_arch = "x86_64")] + pub fn sgx_epc_region(&self) -> &Option { + &self.sgx_epc_region + } } #[cfg(feature = "acpi")] diff --git a/vmm/src/vm.rs b/vmm/src/vm.rs index 8cecd78a7..3e0ab107e 100644 --- a/vmm/src/vm.rs +++ b/vmm/src/vm.rs @@ -351,6 +351,17 @@ impl Vm { ) .map_err(Error::MemoryManager)?; + #[cfg(target_arch = "x86_64")] + { + if let Some(sgx_epc_config) = config.lock().unwrap().sgx_epc.clone() { + memory_manager + .lock() + .unwrap() + .setup_sgx(sgx_epc_config) + .map_err(Error::MemoryManager)?; + } + } + let new_vm = Vm::new_from_memory_manager( config, memory_manager,