From 3f42f86d81951112da9a86c54b1ed7eeaf5b9cda Mon Sep 17 00:00:00 2001 From: Bo Chen Date: Wed, 22 Apr 2020 14:20:17 -0700 Subject: [PATCH] vmm: Add the 'shared' and 'hugepages' controls to MemoryConfig The new 'shared' and 'hugepages' controls aim to replace the 'file' option in MemoryConfig. This patch also updated all related integration tests to use the new controls (instead of providing explicit paths to "/dev/shm" or "/dev/hugepages"). Fixes: #1011 Signed-off-by: Rob Bradford Signed-off-by: Bo Chen --- src/main.rs | 4 +- tests/integration.rs | 12 ++-- vmm/src/api/openapi/cloud-hypervisor.yaml | 6 ++ vmm/src/config.rs | 28 ++++++++- vmm/src/memory_manager.rs | 72 ++++++++++++++++++++--- vmm/src/seccomp_filters.rs | 1 + 6 files changed, 107 insertions(+), 16 deletions(-) diff --git a/src/main.rs b/src/main.rs index a3cc734fe..4e8249a0e 100755 --- a/src/main.rs +++ b/src/main.rs @@ -99,7 +99,7 @@ fn create_app<'a, 'b>( .long("memory") .help( "Memory parameters \ - \"size=,file=,mergeable=on|off,\ + \"size=,mergeable=on|off,shared=on|off,hugepages=on|off,\ hotplug_method=acpi|virtio-mem,\ hotplug_size=\"", ) @@ -501,6 +501,8 @@ mod unit_tests { mergeable: false, hotplug_method: HotplugMethod::Acpi, hotplug_size: None, + shared: false, + hugepages: false, }, kernel: Some(KernelConfig { path: PathBuf::from("/path/to/kernel"), diff --git a/tests/integration.rs b/tests/integration.rs index be89ccffe..a394f1540 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -1175,7 +1175,7 @@ mod tests { let mut cloud_child = GuestCommand::new(&guest) .args(&["--cpus", "boot=4"]) - .args(&["--memory", "size=512M,file=/dev/shm"]) + .args(&["--memory", "size=512M,shared=on"]) .args(&["--kernel", guest.fw_path.as_str()]) .args(&[ "--disk", @@ -1288,7 +1288,7 @@ mod tests { let mut cloud_child = GuestCommand::new(&guest) .args(&["--cpus", format!("boot={}", num_queues / 2).as_str()]) - .args(&["--memory", "size=512M,hotplug_size=2048M,file=/dev/shm"]) + .args(&["--memory", "size=512M,hotplug_size=2048M,shared=on"]) .args(&["--kernel", kernel_path.to_str().unwrap()]) .args(&["--cmdline", CLEAR_KERNEL_CMDLINE]) .default_disks() @@ -1457,7 +1457,7 @@ mod tests { let mut cloud_child = GuestCommand::new(&guest) .args(&["--cpus", format!("boot={}", num_queues).as_str()]) - .args(&["--memory", "size=512M,hotplug_size=2048M,file=/dev/shm"]) + .args(&["--memory", "size=512M,hotplug_size=2048M,shared=on"]) .args(&["--kernel", kernel_path.to_str().unwrap()]) .args(&["--cmdline", CLEAR_KERNEL_CMDLINE]) .args(&[ @@ -1659,7 +1659,7 @@ mod tests { let mut cloud_child = GuestCommand::new(&guest) .args(&["--cpus", format!("boot={}", num_queues).as_str()]) - .args(&["--memory", "size=512M,file=/dev/shm"]) + .args(&["--memory", "size=512M,shared=on"]) .args(&["--kernel", kernel_path.to_str().unwrap()]) .args(&["--cmdline", CLEAR_KERNEL_CMDLINE]) .args(&[ @@ -1814,7 +1814,7 @@ mod tests { let mut guest_command = GuestCommand::new(&guest); guest_command .args(&["--cpus", "boot=1"]) - .args(&["--memory", "size=512M,hotplug_size=2048M,file=/dev/shm"]) + .args(&["--memory", "size=512M,hotplug_size=2048M,shared=on"]) .args(&["--kernel", kernel_path.to_str().unwrap()]) .default_disks() .default_net() @@ -2491,7 +2491,7 @@ mod tests { let mut child = GuestCommand::new(&guest) .args(&["--cpus", "boot=4"]) - .args(&["--memory", "size=2G,file=/dev/hugepages"]) + .args(&["--memory", "size=2G,hugepages=on,shared=on"]) .args(&["--kernel", kernel_path.to_str().unwrap()]) .default_disks() .args(&[ diff --git a/vmm/src/api/openapi/cloud-hypervisor.yaml b/vmm/src/api/openapi/cloud-hypervisor.yaml index 9bb559b20..79d640fe1 100644 --- a/vmm/src/api/openapi/cloud-hypervisor.yaml +++ b/vmm/src/api/openapi/cloud-hypervisor.yaml @@ -380,6 +380,12 @@ components: hotplug_method: type: string default: "acpi" + shared: + type: boolean + default: false + hugepages: + type: boolean + default: false KernelConfig: required: diff --git a/vmm/src/config.rs b/vmm/src/config.rs index 99a898358..a24b0cfe8 100644 --- a/vmm/src/config.rs +++ b/vmm/src/config.rs @@ -438,6 +438,10 @@ pub struct MemoryConfig { pub hotplug_method: HotplugMethod, #[serde(default)] pub hotplug_size: Option, + #[serde(default)] + pub shared: bool, + #[serde(default)] + pub hugepages: bool, } impl MemoryConfig { @@ -448,7 +452,9 @@ impl MemoryConfig { .add("file") .add("mergeable") .add("hotplug_method") - .add("hotplug_size"); + .add("hotplug_size") + .add("shared") + .add("hugepages"); parser.parse(memory).map_err(Error::ParseMemory)?; let size = parser @@ -470,6 +476,16 @@ impl MemoryConfig { .convert::("hotplug_size") .map_err(Error::ParseMemory)? .map(|v| v.0); + let shared = parser + .convert::("shared") + .map_err(Error::ParseMemory)? + .unwrap_or(Toggle(false)) + .0; + let hugepages = parser + .convert::("hugepages") + .map_err(Error::ParseMemory)? + .unwrap_or(Toggle(false)) + .0; Ok(MemoryConfig { size, @@ -477,6 +493,8 @@ impl MemoryConfig { mergeable, hotplug_method, hotplug_size, + shared, + hugepages, }) } } @@ -489,6 +507,8 @@ impl Default for MemoryConfig { mergeable: false, hotplug_method: HotplugMethod::Acpi, hotplug_size: None, + shared: false, + hugepages: false, } } } @@ -1223,6 +1243,10 @@ impl VmConfig { return Err(ValidationError::CpusMaxLowerThanBoot); } + if self.memory.file.is_some() { + error!("Use of backing file ('--memory file=') is deprecated. Use the 'shared' and 'hugepages' controls."); + } + if let Some(disks) = &self.disks { for disk in disks { if disk.vhost_socket.as_ref().and(disk.path.as_ref()).is_some() { @@ -1869,6 +1893,8 @@ mod tests { mergeable: false, hotplug_method: HotplugMethod::Acpi, hotplug_size: None, + shared: false, + hugepages: false, }, kernel: Some(KernelConfig { path: PathBuf::from("/path/to/kernel"), diff --git a/vmm/src/memory_manager.rs b/vmm/src/memory_manager.rs index 8e46fbcfb..291436701 100644 --- a/vmm/src/memory_manager.rs +++ b/vmm/src/memory_manager.rs @@ -15,7 +15,7 @@ use kvm_ioctls::*; use std::convert::TryInto; use std::fs::{File, OpenOptions}; use std::io; -use std::os::unix::io::FromRawFd; +use std::os::unix::io::{FromRawFd, RawFd}; use std::path::PathBuf; use std::sync::{Arc, Mutex}; use url::Url; @@ -62,6 +62,8 @@ pub struct MemoryManager { pub virtiomem_region: Option>, pub virtiomem_resize: Option, snapshot: Mutex>>, + shared: bool, + hugepages: bool, } #[derive(Debug)] @@ -253,6 +255,8 @@ impl MemoryManager { region.size as usize, true, prefault, + false, + false, )?); } } else { @@ -263,6 +267,8 @@ impl MemoryManager { region.1, false, prefault, + config.shared, + config.hugepages, )?); } } @@ -294,6 +300,8 @@ impl MemoryManager { size as usize, false, false, + config.shared, + config.hugepages, )?); virtiomem_resize = Some(vm_virtio::Resize::new().map_err(Error::EventFdFail)?); @@ -344,6 +352,8 @@ impl MemoryManager { virtiomem_region: virtiomem_region.clone(), virtiomem_resize, snapshot: Mutex::new(None), + shared: config.shared, + hugepages: config.hugepages, })); guest_memory.memory().with_regions(|_, region| { @@ -460,12 +470,24 @@ impl MemoryManager { } } + fn memfd_create(name: &std::ffi::CStr, flags: u32) -> Result { + let res = unsafe { libc::syscall(libc::SYS_memfd_create, name.as_ptr(), flags) }; + + if res < 0 { + Err(std::io::Error::last_os_error()) + } else { + Ok(res as RawFd) + } + } + fn create_ram_region( backing_file: &Option, start_addr: GuestAddress, size: usize, copy_on_write: bool, prefault: bool, + shared: bool, + hugepages: bool, ) -> Result, Error> { Ok(Arc::new(match backing_file { Some(ref file) => { @@ -507,11 +529,38 @@ impl MemoryManager { ) .map_err(Error::GuestMemory)? } - None => GuestRegionMmap::new( - MmapRegion::new(size).map_err(Error::GuestMemoryRegion)?, - start_addr, - ) - .map_err(Error::GuestMemory)?, + None => { + let fd = Self::memfd_create( + &std::ffi::CString::new("ch_ram").unwrap(), + if hugepages { + libc::MFD_HUGETLB | libc::MAP_HUGE_2MB as u32 + } else { + 0 + }, + ) + .map_err(Error::SharedFileCreate)?; + + let f = unsafe { File::from_raw_fd(fd) }; + f.set_len(size as u64).map_err(Error::SharedFileSetLen)?; + + let mmap_flags = libc::MAP_NORESERVE + | if shared { + libc::MAP_SHARED + } else { + libc::MAP_PRIVATE + }; + GuestRegionMmap::new( + MmapRegion::build( + Some(FileOffset::new(f, 0)), + size, + libc::PROT_READ | libc::PROT_WRITE, + mmap_flags, + ) + .map_err(Error::GuestMemoryRegion)?, + start_addr, + ) + .map_err(Error::GuestMemory)? + } })) } @@ -555,8 +604,15 @@ impl MemoryManager { } // Allocate memory for the region - let region = - MemoryManager::create_ram_region(&self.backing_file, start_addr, size, false, false)?; + let region = MemoryManager::create_ram_region( + &self.backing_file, + start_addr, + size, + false, + false, + self.shared, + self.hugepages, + )?; // Map it into the guest self.create_userspace_mapping( diff --git a/vmm/src/seccomp_filters.rs b/vmm/src/seccomp_filters.rs index 6c40796b1..942518d1c 100644 --- a/vmm/src/seccomp_filters.rs +++ b/vmm/src/seccomp_filters.rs @@ -222,6 +222,7 @@ pub fn vmm_thread_filter() -> Result { allow_syscall(libc::SYS_listen), allow_syscall(libc::SYS_lseek), allow_syscall(libc::SYS_madvise), + allow_syscall(libc::SYS_memfd_create), allow_syscall(libc::SYS_mmap), allow_syscall(libc::SYS_mprotect), allow_syscall(libc::SYS_mremap),