From 294c26bfb72097d6a230200b3ee7c4e283d0f00f Mon Sep 17 00:00:00 2001 From: Sebastien Boeuf Date: Wed, 19 Jun 2019 13:48:37 -0700 Subject: [PATCH] vmm: Add virtio-pmem support to cloud-hypervisor This patch plumbs the virtio-pmem device to the VMM. By adding a new command line option "--pmem", we can now expose some persistent memory to the guest OS, backed by the provided source. The point of having such support in cloud-hypervisor is to be able to share some memory between the host and the guest as DAXable. One interesting use case is to boot directly from an image passed through virtio-pmem, instead of going through virtio-blk. This can allow good performances while avoiding the guest cache, which would prevent the VM memory footprint from growing too much. Fixes #68 Signed-off-by: Sebastien Boeuf --- src/main.rs | 12 ++++++++ vmm/src/config.rs | 43 +++++++++++++++++++++++++++ vmm/src/vm.rs | 76 ++++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 130 insertions(+), 1 deletion(-) diff --git a/src/main.rs b/src/main.rs index 5be5a4e82..8049aca88 100755 --- a/src/main.rs +++ b/src/main.rs @@ -77,6 +77,15 @@ fn main() { .takes_value(true) .min_values(1), ) + .arg( + Arg::with_name("pmem") + .long("pmem") + .help( + "Persistent memory parameters \"file=,\ + size=\"", + ) + .takes_value(true), + ) .get_matches(); // These .unwrap()s cannot fail as there is a default value defined @@ -101,6 +110,8 @@ fn main() { let fs: Option> = cmd_arguments.values_of("fs").map(|x| x.collect()); + let pmem = cmd_arguments.value_of("pmem"); + let vm_config = match config::VmConfig::parse(config::VmParams { cpus, memory, @@ -110,6 +121,7 @@ fn main() { net, rng, fs, + pmem, }) { Ok(config) => config, Err(e) => { diff --git a/vmm/src/config.rs b/vmm/src/config.rs index 498d0e1e8..6a2095705 100644 --- a/vmm/src/config.rs +++ b/vmm/src/config.rs @@ -48,6 +48,10 @@ pub enum Error<'a> { ParseFsNumQueuesParam(std::num::ParseIntError), /// Failed parsing fs queue size parameter. ParseFsQueueSizeParam(std::num::ParseIntError), + /// Failed parsing persitent memory file parameter. + ParsePmemFileParam, + /// Failed parsing persitent memory size parameter. + ParsePmemSizeParam(std::num::ParseIntError), } pub type Result<'a, T> = result::Result>; @@ -60,6 +64,7 @@ pub struct VmParams<'a> { pub net: Option<&'a str>, pub rng: &'a str, pub fs: Option>, + pub pmem: Option<&'a str>, } pub struct CpusConfig(pub u8); @@ -291,6 +296,42 @@ impl<'a> FsConfig<'a> { } } +pub struct PmemConfig<'a> { + pub file: &'a Path, + pub size: u64, +} + +impl<'a> PmemConfig<'a> { + pub fn parse(pmem: Option<&'a str>) -> Result> { + if pmem.is_none() { + return Ok(None); + } + + // Split the parameters based on the comma delimiter + let params_list: Vec<&str> = pmem.unwrap().split(',').collect(); + + let mut file_str: &str = ""; + let mut size_str: &str = ""; + + for param in params_list.iter() { + if param.starts_with("file=") { + file_str = ¶m[5..]; + } else if param.starts_with("size=") { + size_str = ¶m[5..]; + } + } + + if file_str.is_empty() { + return Err(Error::ParsePmemFileParam); + } + + Ok(Some(PmemConfig { + file: Path::new(file_str), + size: size_str.parse::().map_err(Error::ParsePmemSizeParam)?, + })) + } +} + pub struct VmConfig<'a> { pub cpus: CpusConfig, pub memory: MemoryConfig<'a>, @@ -300,6 +341,7 @@ pub struct VmConfig<'a> { pub net: Option>, pub rng: RngConfig<'a>, pub fs: Option>>, + pub pmem: Option>, } impl<'a> VmConfig<'a> { @@ -327,6 +369,7 @@ impl<'a> VmConfig<'a> { net: NetConfig::parse(vm_params.net)?, rng: RngConfig::parse(vm_params.rng)?, fs, + pmem: PmemConfig::parse(vm_params.pmem)?, }) } } diff --git a/vmm/src/vm.rs b/vmm/src/vm.rs index af7e38675..1ffa1685f 100755 --- a/vmm/src/vm.rs +++ b/vmm/src/vm.rs @@ -41,6 +41,7 @@ use std::fs::{File, OpenOptions}; use std::io::{self, stdout}; use std::os::unix::fs::OpenOptionsExt; use std::os::unix::io::{AsRawFd, RawFd}; +use std::ptr::null_mut; use std::sync::{Arc, Barrier, Mutex}; use std::{result, str, thread}; use vm_allocator::SystemAllocator; @@ -183,6 +184,9 @@ pub enum DeviceManagerError { /// Cannot create virtio-fs device CreateVirtioFs(vm_virtio::fs::Error), + /// Cannot create virtio-pmem device + CreateVirtioPmem(io::Error), + /// Failed parsing disk image format DetectImageType(qcow::Error), @@ -209,6 +213,15 @@ pub enum DeviceManagerError { /// Cannot add PCI device AddPciDevice(pci::PciRootError), + + /// Cannot open persistent memory file + PmemFileOpen(io::Error), + + /// Cannot set persistent memory file size + PmemFileSetLen(io::Error), + + /// Cannot find a memory range for persistent memory + PmemRangeAllocation, } pub type DeviceManagerResult = result::Result; @@ -579,7 +592,6 @@ impl DeviceManager { // Add virtio-rng if required if let Some(rng_path) = vm_cfg.rng.src.to_str() { - println!("VIRTIO_RNG PATH {}", rng_path); let virtio_rng_device = vm_virtio::Rng::new(rng_path).map_err(DeviceManagerError::CreateVirtioRng)?; @@ -620,6 +632,68 @@ impl DeviceManager { } } + // Add virtio-pmem if required + if let Some(pmem_cfg) = &vm_cfg.pmem { + let size = pmem_cfg.size; + + let pmem_guest_addr = allocator + .allocate_mmio_addresses(None, size as GuestUsize) + .ok_or(DeviceManagerError::PmemRangeAllocation)?; + + let (custom_flags, set_len) = if pmem_cfg.file.is_dir() { + (O_TMPFILE, true) + } else { + (0, false) + }; + + let file = OpenOptions::new() + .read(true) + .write(true) + .custom_flags(custom_flags) + .open(pmem_cfg.file) + .map_err(DeviceManagerError::PmemFileOpen)?; + + if set_len { + file.set_len(size) + .map_err(DeviceManagerError::PmemFileSetLen)?; + } + + let addr = unsafe { + libc::mmap( + null_mut(), + size as usize, + libc::PROT_READ | libc::PROT_WRITE, + libc::MAP_NORESERVE | libc::MAP_SHARED, + file.as_raw_fd(), + 0 as libc::off_t, + ) as *mut u8 + }; + + let mem_region = kvm_userspace_memory_region { + slot: memory.num_regions() as u32, + guest_phys_addr: pmem_guest_addr.raw_value(), + memory_size: size, + userspace_addr: addr as u64, + flags: 0, + }; + // Safe because the guest regions are guaranteed not to overlap. + let _ = unsafe { vm_fd.set_user_memory_region(mem_region) }; + + let virtio_pmem_device = + vm_virtio::Pmem::new(file, pmem_guest_addr, size as GuestUsize) + .map_err(DeviceManagerError::CreateVirtioPmem)?; + + DeviceManager::add_virtio_pci_device( + Box::new(virtio_pmem_device), + memory.clone(), + allocator, + vm_fd, + &mut pci_root, + &mut mmio_bus, + &interrupt_info, + )?; + } + let pci = Arc::new(Mutex::new(PciConfigIo::new(pci_root))); Ok(DeviceManager {