vmm: Add virtio-pmem support to cloud-hypervisor

This patch plumbs the virtio-pmem device to the VMM. By adding a new
command line option "--pmem", we can now expose some persistent memory
to the guest OS, backed by the provided source.

The point of having such support in cloud-hypervisor is to be able to
share some memory between the host and the guest as DAXable.
One interesting use case is to boot directly from an image passed
through virtio-pmem, instead of going through virtio-blk. This can
allow good performances while avoiding the guest cache, which would
prevent the VM memory footprint from growing too much.

Fixes #68

Signed-off-by: Sebastien Boeuf <sebastien.boeuf@intel.com>

src/main.rs

@@ -77,6 +77,15 @@ fn main() {
                 .takes_value(true)
                 .min_values(1),
         )
+        .arg(
+            Arg::with_name("pmem")
+                .long("pmem")
+                .help(
+                    "Persistent memory parameters \"file=<backing_file_path>,\
+                     size=<persistent_memory_size>\"",
+                )
+                .takes_value(true),
+        )
         .get_matches();
 
     // These .unwrap()s cannot fail as there is a default value defined
@@ -101,6 +110,8 @@ fn main() {
 
     let fs: Option<Vec<&str>> = cmd_arguments.values_of("fs").map(|x| x.collect());
 
+    let pmem = cmd_arguments.value_of("pmem");
+
     let vm_config = match config::VmConfig::parse(config::VmParams {
         cpus,
         memory,
@@ -110,6 +121,7 @@ fn main() {
         net,
         rng,
         fs,
+        pmem,
     }) {
         Ok(config) => config,
         Err(e) => {

vmm/src/config.rs

@@ -48,6 +48,10 @@ pub enum Error<'a> {
     ParseFsNumQueuesParam(std::num::ParseIntError),
     /// Failed parsing fs queue size parameter.
     ParseFsQueueSizeParam(std::num::ParseIntError),
+    /// Failed parsing persitent memory file parameter.
+    ParsePmemFileParam,
+    /// Failed parsing persitent memory size parameter.
+    ParsePmemSizeParam(std::num::ParseIntError),
 }
 pub type Result<'a, T> = result::Result<T, Error<'a>>;
 
@@ -60,6 +64,7 @@ pub struct VmParams<'a> {
     pub net: Option<&'a str>,
     pub rng: &'a str,
     pub fs: Option<Vec<&'a str>>,
+    pub pmem: Option<&'a str>,
 }
 
 pub struct CpusConfig(pub u8);
@@ -291,6 +296,42 @@ impl<'a> FsConfig<'a> {
     }
 }
 
+pub struct PmemConfig<'a> {
+    pub file: &'a Path,
+    pub size: u64,
+}
+
+impl<'a> PmemConfig<'a> {
+    pub fn parse(pmem: Option<&'a str>) -> Result<Option<Self>> {
+        if pmem.is_none() {
+            return Ok(None);
+        }
+
+        // Split the parameters based on the comma delimiter
+        let params_list: Vec<&str> = pmem.unwrap().split(',').collect();
+
+        let mut file_str: &str = "";
+        let mut size_str: &str = "";
+
+        for param in params_list.iter() {
+            if param.starts_with("file=") {
+                file_str = &param[5..];
+            } else if param.starts_with("size=") {
+                size_str = &param[5..];
+            }
+        }
+
+        if file_str.is_empty() {
+            return Err(Error::ParsePmemFileParam);
+        }
+
+        Ok(Some(PmemConfig {
+            file: Path::new(file_str),
+            size: size_str.parse::<u64>().map_err(Error::ParsePmemSizeParam)?,
+        }))
+    }
+}
+
 pub struct VmConfig<'a> {
     pub cpus: CpusConfig,
     pub memory: MemoryConfig<'a>,
@@ -300,6 +341,7 @@ pub struct VmConfig<'a> {
     pub net: Option<NetConfig<'a>>,
     pub rng: RngConfig<'a>,
     pub fs: Option<Vec<FsConfig<'a>>>,
+    pub pmem: Option<PmemConfig<'a>>,
 }
 
 impl<'a> VmConfig<'a> {
@@ -327,6 +369,7 @@ impl<'a> VmConfig<'a> {
             net: NetConfig::parse(vm_params.net)?,
             rng: RngConfig::parse(vm_params.rng)?,
             fs,
+            pmem: PmemConfig::parse(vm_params.pmem)?,
         })
     }
 }

vmm/src/vm.rs

@@ -41,6 +41,7 @@ use std::fs::{File, OpenOptions};
 use std::io::{self, stdout};
 use std::os::unix::fs::OpenOptionsExt;
 use std::os::unix::io::{AsRawFd, RawFd};
+use std::ptr::null_mut;
 use std::sync::{Arc, Barrier, Mutex};
 use std::{result, str, thread};
 use vm_allocator::SystemAllocator;
@@ -183,6 +184,9 @@ pub enum DeviceManagerError {
     /// Cannot create virtio-fs device
     CreateVirtioFs(vm_virtio::fs::Error),
 
+    /// Cannot create virtio-pmem device
+    CreateVirtioPmem(io::Error),
+
     /// Failed parsing disk image format
     DetectImageType(qcow::Error),
 
@@ -209,6 +213,15 @@ pub enum DeviceManagerError {
 
     /// Cannot add PCI device
     AddPciDevice(pci::PciRootError),
+
+    /// Cannot open persistent memory file
+    PmemFileOpen(io::Error),
+
+    /// Cannot set persistent memory file size
+    PmemFileSetLen(io::Error),
+
+    /// Cannot find a memory range for persistent memory
+    PmemRangeAllocation,
 }
 pub type DeviceManagerResult<T> = result::Result<T, DeviceManagerError>;
 
@@ -579,7 +592,6 @@ impl DeviceManager {
 
         // Add virtio-rng if required
         if let Some(rng_path) = vm_cfg.rng.src.to_str() {
-            println!("VIRTIO_RNG PATH {}", rng_path);
             let virtio_rng_device =
                 vm_virtio::Rng::new(rng_path).map_err(DeviceManagerError::CreateVirtioRng)?;
 
@@ -620,6 +632,68 @@ impl DeviceManager {
             }
         }
 
+        // Add virtio-pmem if required
+        if let Some(pmem_cfg) = &vm_cfg.pmem {
+            let size = pmem_cfg.size;
+
+            let pmem_guest_addr = allocator
+                .allocate_mmio_addresses(None, size as GuestUsize)
+                .ok_or(DeviceManagerError::PmemRangeAllocation)?;
+
+            let (custom_flags, set_len) = if pmem_cfg.file.is_dir() {
+                (O_TMPFILE, true)
+            } else {
+                (0, false)
+            };
+
+            let file = OpenOptions::new()
+                .read(true)
+                .write(true)
+                .custom_flags(custom_flags)
+                .open(pmem_cfg.file)
+                .map_err(DeviceManagerError::PmemFileOpen)?;
+
+            if set_len {
+                file.set_len(size)
+                    .map_err(DeviceManagerError::PmemFileSetLen)?;
+            }
+
+            let addr = unsafe {
+                libc::mmap(
+                    null_mut(),
+                    size as usize,
+                    libc::PROT_READ | libc::PROT_WRITE,
+                    libc::MAP_NORESERVE | libc::MAP_SHARED,
+                    file.as_raw_fd(),
+                    0 as libc::off_t,
+                ) as *mut u8
+            };
+
+            let mem_region = kvm_userspace_memory_region {
+                slot: memory.num_regions() as u32,
+                guest_phys_addr: pmem_guest_addr.raw_value(),
+                memory_size: size,
+                userspace_addr: addr as u64,
+                flags: 0,
+            };
+            // Safe because the guest regions are guaranteed not to overlap.
+            let _ = unsafe { vm_fd.set_user_memory_region(mem_region) };
+
+            let virtio_pmem_device =
+                vm_virtio::Pmem::new(file, pmem_guest_addr, size as GuestUsize)
+                    .map_err(DeviceManagerError::CreateVirtioPmem)?;
+
+            DeviceManager::add_virtio_pci_device(
+                Box::new(virtio_pmem_device),
+                memory.clone(),
+                allocator,
+                vm_fd,
+                &mut pci_root,
+                &mut mmio_bus,
+                &interrupt_info,
+            )?;
+        }
+
         let pci = Arc::new(Mutex::new(PciConfigIo::new(pci_root)));
 
         Ok(DeviceManager {