From e6b934a56a7c2e6fcc925cf6572109efc1e4b574 Mon Sep 17 00:00:00 2001 From: Hui Zhu Date: Wed, 4 Mar 2020 10:16:07 +0800 Subject: [PATCH] vmm: Add support for virtio-mem This commit adds new option hotplug_method to memory config. It can set the hotplug method to "acpi" or "virtio-mem". Signed-off-by: Hui Zhu --- src/main.rs | 3 + vmm/src/config.rs | 28 ++++++++++ vmm/src/memory_manager.rs | 113 +++++++++++++++++++++++++++++++++----- vmm/src/vm.rs | 4 ++ 4 files changed, 133 insertions(+), 15 deletions(-) diff --git a/src/main.rs b/src/main.rs index 609444263..5665aceaa 100755 --- a/src/main.rs +++ b/src/main.rs @@ -100,6 +100,7 @@ fn create_app<'a, 'b>( .help( "Memory parameters \ \"size=,file=,mergeable=on|off,\ + hotplug_method=acpi|virtio-mem,\ hotplug_size=\"", ) .default_value(&default_memory) @@ -412,6 +413,7 @@ extern crate credibility; #[cfg(test)] mod unit_tests { + use crate::config::HotplugMethod; use crate::{create_app, prepare_default_values}; use std::path::PathBuf; use vmm::config::{ @@ -476,6 +478,7 @@ mod unit_tests { size: 536_870_912, file: None, mergeable: false, + hotplug_method: HotplugMethod::Acpi, hotplug_size: None, }, kernel: None, diff --git a/vmm/src/config.rs b/vmm/src/config.rs index c0b6847b6..7b838e2c0 100644 --- a/vmm/src/config.rs +++ b/vmm/src/config.rs @@ -31,6 +31,8 @@ pub enum Error { ParseCpusUnknownParam, /// Max is less than boot ParseCpusMaxLowerThanBoot, + /// Failed parsing memory hotplug_method parameter. + ParseMemoryHotplugMethodParam, /// Failed parsing memory file parameter. ParseMemoryFileParam, /// Failed parsing kernel parameters. @@ -246,6 +248,18 @@ impl Default for CpusConfig { } } +#[derive(Clone, Debug, PartialEq, Deserialize, Serialize)] +pub enum HotplugMethod { + Acpi, + VirtioMem, +} + +impl Default for HotplugMethod { + fn default() -> Self { + HotplugMethod::Acpi + } +} + #[derive(Clone, Debug, PartialEq, Deserialize, Serialize)] pub struct MemoryConfig { pub size: u64, @@ -254,6 +268,8 @@ pub struct MemoryConfig { #[serde(default)] pub mergeable: bool, #[serde(default)] + pub hotplug_method: HotplugMethod, + #[serde(default)] pub hotplug_size: Option, } @@ -266,6 +282,7 @@ impl MemoryConfig { let mut file_str: &str = ""; let mut mergeable_str: &str = ""; let mut backed = false; + let mut hotplug_method_str: &str = "acpi"; let mut hotplug_str: &str = ""; for param in params_list.iter() { @@ -276,6 +293,8 @@ impl MemoryConfig { file_str = ¶m[5..]; } else if param.starts_with("mergeable=") { mergeable_str = ¶m[10..]; + } else if param.starts_with("hotplug_method=") { + hotplug_method_str = ¶m[15..]; } else if param.starts_with("hotplug_size=") { hotplug_str = ¶m[13..] } @@ -291,10 +310,18 @@ impl MemoryConfig { None }; + let hotplug_method_str = hotplug_method_str.to_string().to_lowercase(); + let hotplug_method = match &hotplug_method_str[..] { + "acpi" => HotplugMethod::Acpi, + "virtio-mem" => HotplugMethod::VirtioMem, + _ => return Err(Error::ParseMemoryHotplugMethodParam), + }; + Ok(MemoryConfig { size: parse_size(size_str)?, file, mergeable: parse_on_off(mergeable_str)?, + hotplug_method, hotplug_size: if hotplug_str == "" { None } else { @@ -310,6 +337,7 @@ impl Default for MemoryConfig { size: DEFAULT_MEMORY_MB << 20, file: None, mergeable: false, + hotplug_method: HotplugMethod::Acpi, hotplug_size: None, } } diff --git a/vmm/src/memory_manager.rs b/vmm/src/memory_manager.rs index 8f359eb3e..232ff86fb 100644 --- a/vmm/src/memory_manager.rs +++ b/vmm/src/memory_manager.rs @@ -3,6 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 // +use crate::config::HotplugMethod; #[cfg(feature = "acpi")] use acpi_tables::{aml, aml::Aml}; use arch::RegionType; @@ -45,8 +46,12 @@ pub struct MemoryManager { backing_file: Option, mergeable: bool, allocator: Arc>, + hotplug_method: HotplugMethod, + boot_ram: u64, current_ram: u64, next_hotplug_slot: usize, + pub virtiomem_region: Option>, + pub virtiomem_resize: Option, } #[derive(Debug)] @@ -80,6 +85,15 @@ pub enum Error { /// Failed to set the user memory region. SetUserMemoryRegion(kvm_ioctls::Error), + + /// Failed to EventFd. + EventFdFail(std::io::Error), + + /// Eventfd write error + EventfdError(std::io::Error), + + /// Failed to virtio-mem resize + VirtioMemResizeFail(vm_virtio::mem::Error), } pub fn get_host_cpu_phys_bits() -> u8 { @@ -195,6 +209,7 @@ impl MemoryManager { allocator: Arc>, fd: Arc, boot_ram: u64, + hotplug_method: HotplugMethod, hotplug_size: Option, backing_file: &Option, mergeable: bool, @@ -228,8 +243,26 @@ impl MemoryManager { mem_end.unchecked_add(1) }; + let mut virtiomem_region = None; + let mut virtiomem_resize = None; if let Some(size) = hotplug_size { - start_of_device_area = start_of_device_area.unchecked_add(size); + if hotplug_method == HotplugMethod::VirtioMem { + let start_addr = GuestAddress( + (start_of_device_area.0 + vm_virtio::VIRTIO_MEM_DEFAULT_BLOCK_SIZE - 1) + & (!(vm_virtio::VIRTIO_MEM_DEFAULT_BLOCK_SIZE - 1)), + ); + virtiomem_region = Some(MemoryManager::create_ram_region( + backing_file, + start_addr, + size as usize, + )?); + + virtiomem_resize = Some(vm_virtio::Resize::new().map_err(Error::EventFdFail)?); + + start_of_device_area = start_addr.unchecked_add(size); + } else { + start_of_device_area = start_of_device_area.unchecked_add(size); + } } let guest_memory = GuestMemoryAtomic::new(guest_memory); @@ -248,8 +281,12 @@ impl MemoryManager { backing_file: backing_file.clone(), mergeable, allocator: allocator.clone(), + hotplug_method, + boot_ram, current_ram: boot_ram, next_hotplug_slot: 0, + virtiomem_region: virtiomem_region.clone(), + virtiomem_resize, })); guest_memory.memory().with_regions(|_, region| { @@ -263,6 +300,21 @@ impl MemoryManager { Ok(()) })?; + if let Some(region) = virtiomem_region { + memory_manager.lock().unwrap().create_userspace_mapping( + region.start_addr().raw_value(), + region.len() as u64, + region.as_ptr() as u64, + mergeable, + false, + )?; + allocator + .lock() + .unwrap() + .allocate_mmio_addresses(Some(region.start_addr()), region.len(), None) + .ok_or(Error::MemoryRangeAllocation)?; + } + // Allocate RAM and Reserved address ranges. for region in arch_mem_regions.iter() { allocator @@ -315,6 +367,18 @@ impl MemoryManager { })) } + // Update the GuestMemoryMmap with the new range + fn add_region(&mut self, region: Arc) -> Result<(), Error> { + let guest_memory = self + .guest_memory + .memory() + .insert_region(region) + .map_err(Error::GuestMemory)?; + self.guest_memory.lock().unwrap().replace(guest_memory); + + Ok(()) + } + fn hotplug_ram_region(&mut self, size: usize) -> Result<(), Error> { info!("Hotplugging new RAM: {}", size); @@ -370,13 +434,7 @@ impl MemoryManager { self.next_hotplug_slot += 1; - // Update the GuestMemoryMmap with the new range - let guest_memory = self - .guest_memory - .memory() - .insert_region(region) - .map_err(Error::GuestMemory)?; - self.guest_memory.lock().unwrap().replace(guest_memory); + self.add_region(region)?; Ok(()) } @@ -453,14 +511,39 @@ impl MemoryManager { Ok(slot) } - pub fn resize(&mut self, desired_ram: u64) -> Result { - if desired_ram > self.current_ram { - self.hotplug_ram_region((desired_ram - self.current_ram) as usize)?; - self.current_ram = desired_ram; - Ok(true) - } else { - Ok(false) + pub fn virtiomem_resize(&mut self, size: u64) -> Result<(), Error> { + let region = self.virtiomem_region.take(); + if let Some(region) = region { + self.add_region(region)?; } + + if let Some(resize) = &self.virtiomem_resize { + resize.work(size).map_err(Error::VirtioMemResizeFail)?; + } else { + panic!("should not fail here"); + } + + Ok(()) + } + + pub fn resize(&mut self, desired_ram: u64) -> Result { + let mut notify_hotplug = false; + match self.hotplug_method { + HotplugMethod::VirtioMem => { + if desired_ram >= self.boot_ram { + self.virtiomem_resize(desired_ram - self.boot_ram)?; + self.current_ram = desired_ram; + } + } + HotplugMethod::Acpi => { + if desired_ram >= self.current_ram { + self.hotplug_ram_region((desired_ram - self.current_ram) as usize)?; + self.current_ram = desired_ram; + notify_hotplug = true + } + } + } + Ok(notify_hotplug) } } diff --git a/vmm/src/vm.rs b/vmm/src/vm.rs index 629c0c3e8..2603d534d 100755 --- a/vmm/src/vm.rs +++ b/vmm/src/vm.rs @@ -169,6 +169,9 @@ pub enum Error { /// No PCI support NoPciSupport, + + /// Eventfd write error + EventfdError(std::io::Error), } pub type Result = result::Result; @@ -337,6 +340,7 @@ impl Vm { allocator.clone(), fd.clone(), memory_config.size, + memory_config.hotplug_method, memory_config.hotplug_size, &memory_config.file, memory_config.mergeable,