vmm: Add the 'shared' and 'hugepages' controls to MemoryConfig

The new 'shared' and 'hugepages' controls aim to replace the 'file'
option in MemoryConfig. This patch also updated all related integration
tests to use the new controls (instead of providing explicit paths to
"/dev/shm" or "/dev/hugepages").

Fixes: #1011

Signed-off-by: Rob Bradford <robert.bradford@intel.com>
Signed-off-by: Bo Chen <chen.bo@intel.com>

src/main.rs

@@ -99,7 +99,7 @@ fn create_app<'a, 'b>(
                 .long("memory")
                 .help(
                     "Memory parameters \
-                     \"size=<guest_memory_size>,file=<backing_file_path>,mergeable=on|off,\
+                     \"size=<guest_memory_size>,mergeable=on|off,shared=on|off,hugepages=on|off,\
                      hotplug_method=acpi|virtio-mem,\
                      hotplug_size=<hotpluggable_memory_size>\"",
                 )
@@ -501,6 +501,8 @@ mod unit_tests {
                     mergeable: false,
                     hotplug_method: HotplugMethod::Acpi,
                     hotplug_size: None,
+                    shared: false,
+                    hugepages: false,
                 },
                 kernel: Some(KernelConfig {
                     path: PathBuf::from("/path/to/kernel"),

tests/integration.rs

@@ -1175,7 +1175,7 @@ mod tests {
 
             let mut cloud_child = GuestCommand::new(&guest)
                 .args(&["--cpus", "boot=4"])
-                .args(&["--memory", "size=512M,file=/dev/shm"])
+                .args(&["--memory", "size=512M,shared=on"])
                 .args(&["--kernel", guest.fw_path.as_str()])
                 .args(&[
                     "--disk",
@@ -1288,7 +1288,7 @@ mod tests {
 
             let mut cloud_child = GuestCommand::new(&guest)
                 .args(&["--cpus", format!("boot={}", num_queues / 2).as_str()])
-                .args(&["--memory", "size=512M,hotplug_size=2048M,file=/dev/shm"])
+                .args(&["--memory", "size=512M,hotplug_size=2048M,shared=on"])
                 .args(&["--kernel", kernel_path.to_str().unwrap()])
                 .args(&["--cmdline", CLEAR_KERNEL_CMDLINE])
                 .default_disks()
@@ -1457,7 +1457,7 @@ mod tests {
 
             let mut cloud_child = GuestCommand::new(&guest)
                 .args(&["--cpus", format!("boot={}", num_queues).as_str()])
-                .args(&["--memory", "size=512M,hotplug_size=2048M,file=/dev/shm"])
+                .args(&["--memory", "size=512M,hotplug_size=2048M,shared=on"])
                 .args(&["--kernel", kernel_path.to_str().unwrap()])
                 .args(&["--cmdline", CLEAR_KERNEL_CMDLINE])
                 .args(&[
@@ -1659,7 +1659,7 @@ mod tests {
 
             let mut cloud_child = GuestCommand::new(&guest)
                 .args(&["--cpus", format!("boot={}", num_queues).as_str()])
-                .args(&["--memory", "size=512M,file=/dev/shm"])
+                .args(&["--memory", "size=512M,shared=on"])
                 .args(&["--kernel", kernel_path.to_str().unwrap()])
                 .args(&["--cmdline", CLEAR_KERNEL_CMDLINE])
                 .args(&[
@@ -1814,7 +1814,7 @@ mod tests {
             let mut guest_command = GuestCommand::new(&guest);
             guest_command
                 .args(&["--cpus", "boot=1"])
-                .args(&["--memory", "size=512M,hotplug_size=2048M,file=/dev/shm"])
+                .args(&["--memory", "size=512M,hotplug_size=2048M,shared=on"])
                 .args(&["--kernel", kernel_path.to_str().unwrap()])
                 .default_disks()
                 .default_net()
@@ -2491,7 +2491,7 @@ mod tests {
 
             let mut child = GuestCommand::new(&guest)
                 .args(&["--cpus", "boot=4"])
-                .args(&["--memory", "size=2G,file=/dev/hugepages"])
+                .args(&["--memory", "size=2G,hugepages=on,shared=on"])
                 .args(&["--kernel", kernel_path.to_str().unwrap()])
                 .default_disks()
                 .args(&[

vmm/src/api/openapi/cloud-hypervisor.yaml

@@ -380,6 +380,12 @@ components:
         hotplug_method:
           type: string
           default: "acpi"
+        shared:
+          type: boolean
+          default: false
+        hugepages:
+          type: boolean
+          default: false
 
     KernelConfig:
       required:

vmm/src/config.rs

@@ -438,6 +438,10 @@ pub struct MemoryConfig {
     pub hotplug_method: HotplugMethod,
     #[serde(default)]
     pub hotplug_size: Option<u64>,
+    #[serde(default)]
+    pub shared: bool,
+    #[serde(default)]
+    pub hugepages: bool,
 }
 
 impl MemoryConfig {
@@ -448,7 +452,9 @@ impl MemoryConfig {
             .add("file")
             .add("mergeable")
             .add("hotplug_method")
-            .add("hotplug_size");
+            .add("hotplug_size")
+            .add("shared")
+            .add("hugepages");
         parser.parse(memory).map_err(Error::ParseMemory)?;
 
         let size = parser
@@ -470,6 +476,16 @@ impl MemoryConfig {
             .convert::<ByteSized>("hotplug_size")
             .map_err(Error::ParseMemory)?
             .map(|v| v.0);
+        let shared = parser
+            .convert::<Toggle>("shared")
+            .map_err(Error::ParseMemory)?
+            .unwrap_or(Toggle(false))
+            .0;
+        let hugepages = parser
+            .convert::<Toggle>("hugepages")
+            .map_err(Error::ParseMemory)?
+            .unwrap_or(Toggle(false))
+            .0;
 
         Ok(MemoryConfig {
             size,
@@ -477,6 +493,8 @@ impl MemoryConfig {
             mergeable,
             hotplug_method,
             hotplug_size,
+            shared,
+            hugepages,
         })
     }
 }
@@ -489,6 +507,8 @@ impl Default for MemoryConfig {
             mergeable: false,
             hotplug_method: HotplugMethod::Acpi,
             hotplug_size: None,
+            shared: false,
+            hugepages: false,
         }
     }
 }
@@ -1223,6 +1243,10 @@ impl VmConfig {
             return Err(ValidationError::CpusMaxLowerThanBoot);
         }
 
+        if self.memory.file.is_some() {
+            error!("Use of backing file ('--memory file=') is deprecated. Use the 'shared' and 'hugepages' controls.");
+        }
+
         if let Some(disks) = &self.disks {
             for disk in disks {
                 if disk.vhost_socket.as_ref().and(disk.path.as_ref()).is_some() {
@@ -1869,6 +1893,8 @@ mod tests {
                 mergeable: false,
                 hotplug_method: HotplugMethod::Acpi,
                 hotplug_size: None,
+                shared: false,
+                hugepages: false,
             },
             kernel: Some(KernelConfig {
                 path: PathBuf::from("/path/to/kernel"),

vmm/src/memory_manager.rs

@@ -15,7 +15,7 @@ use kvm_ioctls::*;
 use std::convert::TryInto;
 use std::fs::{File, OpenOptions};
 use std::io;
-use std::os::unix::io::FromRawFd;
+use std::os::unix::io::{FromRawFd, RawFd};
 use std::path::PathBuf;
 use std::sync::{Arc, Mutex};
 use url::Url;
@@ -62,6 +62,8 @@ pub struct MemoryManager {
     pub virtiomem_region: Option<Arc<GuestRegionMmap>>,
     pub virtiomem_resize: Option<vm_virtio::Resize>,
     snapshot: Mutex<Option<GuestMemoryLoadGuard<GuestMemoryMmap>>>,
+    shared: bool,
+    hugepages: bool,
 }
 
 #[derive(Debug)]
@@ -253,6 +255,8 @@ impl MemoryManager {
                     region.size as usize,
                     true,
                     prefault,
+                    false,
+                    false,
                 )?);
             }
         } else {
@@ -263,6 +267,8 @@ impl MemoryManager {
                     region.1,
                     false,
                     prefault,
+                    config.shared,
+                    config.hugepages,
                 )?);
             }
         }
@@ -294,6 +300,8 @@ impl MemoryManager {
                     size as usize,
                     false,
                     false,
+                    config.shared,
+                    config.hugepages,
                 )?);
 
                 virtiomem_resize = Some(vm_virtio::Resize::new().map_err(Error::EventFdFail)?);
@@ -344,6 +352,8 @@ impl MemoryManager {
             virtiomem_region: virtiomem_region.clone(),
             virtiomem_resize,
             snapshot: Mutex::new(None),
+            shared: config.shared,
+            hugepages: config.hugepages,
         }));
 
         guest_memory.memory().with_regions(|_, region| {
@@ -460,12 +470,24 @@ impl MemoryManager {
         }
     }
 
+    fn memfd_create(name: &std::ffi::CStr, flags: u32) -> Result<RawFd, io::Error> {
+        let res = unsafe { libc::syscall(libc::SYS_memfd_create, name.as_ptr(), flags) };
+
+        if res < 0 {
+            Err(std::io::Error::last_os_error())
+        } else {
+            Ok(res as RawFd)
+        }
+    }
+
     fn create_ram_region(
         backing_file: &Option<PathBuf>,
         start_addr: GuestAddress,
         size: usize,
         copy_on_write: bool,
         prefault: bool,
+        shared: bool,
+        hugepages: bool,
     ) -> Result<Arc<GuestRegionMmap>, Error> {
         Ok(Arc::new(match backing_file {
             Some(ref file) => {
@@ -507,11 +529,38 @@ impl MemoryManager {
                 )
                 .map_err(Error::GuestMemory)?
             }
-            None => GuestRegionMmap::new(
+            None => {
-                MmapRegion::new(size).map_err(Error::GuestMemoryRegion)?,
+                let fd = Self::memfd_create(
+                    &std::ffi::CString::new("ch_ram").unwrap(),
+                    if hugepages {
+                        libc::MFD_HUGETLB | libc::MAP_HUGE_2MB as u32
+                    } else {
+                        0
+                    },
+                )
+                .map_err(Error::SharedFileCreate)?;
+
+                let f = unsafe { File::from_raw_fd(fd) };
+                f.set_len(size as u64).map_err(Error::SharedFileSetLen)?;
+
+                let mmap_flags = libc::MAP_NORESERVE
+                    | if shared {
+                        libc::MAP_SHARED
+                    } else {
+                        libc::MAP_PRIVATE
+                    };
+                GuestRegionMmap::new(
+                    MmapRegion::build(
+                        Some(FileOffset::new(f, 0)),
+                        size,
+                        libc::PROT_READ | libc::PROT_WRITE,
+                        mmap_flags,
+                    )
+                    .map_err(Error::GuestMemoryRegion)?,
                     start_addr,
                 )
-            .map_err(Error::GuestMemory)?,
+                .map_err(Error::GuestMemory)?
+            }
         }))
     }
 
@@ -555,8 +604,15 @@ impl MemoryManager {
         }
 
         // Allocate memory for the region
-        let region =
+        let region = MemoryManager::create_ram_region(
-            MemoryManager::create_ram_region(&self.backing_file, start_addr, size, false, false)?;
+            &self.backing_file,
+            start_addr,
+            size,
+            false,
+            false,
+            self.shared,
+            self.hugepages,
+        )?;
 
         // Map it into the guest
         self.create_userspace_mapping(

vmm/src/seccomp_filters.rs

@@ -222,6 +222,7 @@ pub fn vmm_thread_filter() -> Result<SeccompFilter, Error> {
             allow_syscall(libc::SYS_listen),
             allow_syscall(libc::SYS_lseek),
             allow_syscall(libc::SYS_madvise),
+            allow_syscall(libc::SYS_memfd_create),
             allow_syscall(libc::SYS_mmap),
             allow_syscall(libc::SYS_mprotect),
             allow_syscall(libc::SYS_mremap),