vmm: Add support for virtio-mem

This commit adds new option hotplug_method to memory config.
It can set the hotplug method to "acpi" or "virtio-mem".

Signed-off-by: Hui Zhu <teawater@antfin.com>

src/main.rs

@@ -100,6 +100,7 @@ fn create_app<'a, 'b>(
                 .help(
                     "Memory parameters \
                      \"size=<guest_memory_size>,file=<backing_file_path>,mergeable=on|off,\
+                     hotplug_method=acpi|virtio-mem,\
                      hotplug_size=<hotpluggable_memory_size>\"",
                 )
                 .default_value(&default_memory)
@@ -412,6 +413,7 @@ extern crate credibility;
 
 #[cfg(test)]
 mod unit_tests {
+    use crate::config::HotplugMethod;
     use crate::{create_app, prepare_default_values};
     use std::path::PathBuf;
     use vmm::config::{
@@ -476,6 +478,7 @@ mod unit_tests {
                     size: 536_870_912,
                     file: None,
                     mergeable: false,
+                    hotplug_method: HotplugMethod::Acpi,
                     hotplug_size: None,
                 },
                 kernel: None,

vmm/src/config.rs

@@ -31,6 +31,8 @@ pub enum Error {
     ParseCpusUnknownParam,
     /// Max is less than boot
     ParseCpusMaxLowerThanBoot,
+    /// Failed parsing memory hotplug_method parameter.
+    ParseMemoryHotplugMethodParam,
     /// Failed parsing memory file parameter.
     ParseMemoryFileParam,
     /// Failed parsing kernel parameters.
@@ -246,6 +248,18 @@ impl Default for CpusConfig {
     }
 }
 
+#[derive(Clone, Debug, PartialEq, Deserialize, Serialize)]
+pub enum HotplugMethod {
+    Acpi,
+    VirtioMem,
+}
+
+impl Default for HotplugMethod {
+    fn default() -> Self {
+        HotplugMethod::Acpi
+    }
+}
+
 #[derive(Clone, Debug, PartialEq, Deserialize, Serialize)]
 pub struct MemoryConfig {
     pub size: u64,
@@ -254,6 +268,8 @@ pub struct MemoryConfig {
     #[serde(default)]
     pub mergeable: bool,
     #[serde(default)]
+    pub hotplug_method: HotplugMethod,
+    #[serde(default)]
     pub hotplug_size: Option<u64>,
 }
 
@@ -266,6 +282,7 @@ impl MemoryConfig {
         let mut file_str: &str = "";
         let mut mergeable_str: &str = "";
         let mut backed = false;
+        let mut hotplug_method_str: &str = "acpi";
         let mut hotplug_str: &str = "";
 
         for param in params_list.iter() {
@@ -276,6 +293,8 @@ impl MemoryConfig {
                 file_str = &param[5..];
             } else if param.starts_with("mergeable=") {
                 mergeable_str = &param[10..];
+            } else if param.starts_with("hotplug_method=") {
+                hotplug_method_str = &param[15..];
             } else if param.starts_with("hotplug_size=") {
                 hotplug_str = &param[13..]
             }
@@ -291,10 +310,18 @@ impl MemoryConfig {
             None
         };
 
+        let hotplug_method_str = hotplug_method_str.to_string().to_lowercase();
+        let hotplug_method = match &hotplug_method_str[..] {
+            "acpi" => HotplugMethod::Acpi,
+            "virtio-mem" => HotplugMethod::VirtioMem,
+            _ => return Err(Error::ParseMemoryHotplugMethodParam),
+        };
+
         Ok(MemoryConfig {
             size: parse_size(size_str)?,
             file,
             mergeable: parse_on_off(mergeable_str)?,
+            hotplug_method,
             hotplug_size: if hotplug_str == "" {
                 None
             } else {
@@ -310,6 +337,7 @@ impl Default for MemoryConfig {
             size: DEFAULT_MEMORY_MB << 20,
             file: None,
             mergeable: false,
+            hotplug_method: HotplugMethod::Acpi,
             hotplug_size: None,
         }
     }

vmm/src/memory_manager.rs

@@ -3,6 +3,7 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
+use crate::config::HotplugMethod;
 #[cfg(feature = "acpi")]
 use acpi_tables::{aml, aml::Aml};
 use arch::RegionType;
@@ -45,8 +46,12 @@ pub struct MemoryManager {
     backing_file: Option<PathBuf>,
     mergeable: bool,
     allocator: Arc<Mutex<SystemAllocator>>,
+    hotplug_method: HotplugMethod,
+    boot_ram: u64,
     current_ram: u64,
     next_hotplug_slot: usize,
+    pub virtiomem_region: Option<Arc<GuestRegionMmap>>,
+    pub virtiomem_resize: Option<vm_virtio::Resize>,
 }
 
 #[derive(Debug)]
@@ -80,6 +85,15 @@ pub enum Error {
 
     /// Failed to set the user memory region.
     SetUserMemoryRegion(kvm_ioctls::Error),
+
+    /// Failed to EventFd.
+    EventFdFail(std::io::Error),
+
+    /// Eventfd write error
+    EventfdError(std::io::Error),
+
+    /// Failed to virtio-mem resize
+    VirtioMemResizeFail(vm_virtio::mem::Error),
 }
 
 pub fn get_host_cpu_phys_bits() -> u8 {
@@ -195,6 +209,7 @@ impl MemoryManager {
         allocator: Arc<Mutex<SystemAllocator>>,
         fd: Arc<VmFd>,
         boot_ram: u64,
+        hotplug_method: HotplugMethod,
         hotplug_size: Option<u64>,
         backing_file: &Option<PathBuf>,
         mergeable: bool,
@@ -228,8 +243,26 @@ impl MemoryManager {
             mem_end.unchecked_add(1)
         };
 
+        let mut virtiomem_region = None;
+        let mut virtiomem_resize = None;
         if let Some(size) = hotplug_size {
-            start_of_device_area = start_of_device_area.unchecked_add(size);
+            if hotplug_method == HotplugMethod::VirtioMem {
+                let start_addr = GuestAddress(
+                    (start_of_device_area.0 + vm_virtio::VIRTIO_MEM_DEFAULT_BLOCK_SIZE - 1)
+                        & (!(vm_virtio::VIRTIO_MEM_DEFAULT_BLOCK_SIZE - 1)),
+                );
+                virtiomem_region = Some(MemoryManager::create_ram_region(
+                    backing_file,
+                    start_addr,
+                    size as usize,
+                )?);
+
+                virtiomem_resize = Some(vm_virtio::Resize::new().map_err(Error::EventFdFail)?);
+
+                start_of_device_area = start_addr.unchecked_add(size);
+            } else {
+                start_of_device_area = start_of_device_area.unchecked_add(size);
+            }
         }
 
         let guest_memory = GuestMemoryAtomic::new(guest_memory);
@@ -248,8 +281,12 @@ impl MemoryManager {
             backing_file: backing_file.clone(),
             mergeable,
             allocator: allocator.clone(),
+            hotplug_method,
+            boot_ram,
             current_ram: boot_ram,
             next_hotplug_slot: 0,
+            virtiomem_region: virtiomem_region.clone(),
+            virtiomem_resize,
         }));
 
         guest_memory.memory().with_regions(|_, region| {
@@ -263,6 +300,21 @@ impl MemoryManager {
             Ok(())
         })?;
 
+        if let Some(region) = virtiomem_region {
+            memory_manager.lock().unwrap().create_userspace_mapping(
+                region.start_addr().raw_value(),
+                region.len() as u64,
+                region.as_ptr() as u64,
+                mergeable,
+                false,
+            )?;
+            allocator
+                .lock()
+                .unwrap()
+                .allocate_mmio_addresses(Some(region.start_addr()), region.len(), None)
+                .ok_or(Error::MemoryRangeAllocation)?;
+        }
+
         // Allocate RAM and Reserved address ranges.
         for region in arch_mem_regions.iter() {
             allocator
@@ -315,6 +367,18 @@ impl MemoryManager {
         }))
     }
 
+    // Update the GuestMemoryMmap with the new range
+    fn add_region(&mut self, region: Arc<GuestRegionMmap>) -> Result<(), Error> {
+        let guest_memory = self
+            .guest_memory
+            .memory()
+            .insert_region(region)
+            .map_err(Error::GuestMemory)?;
+        self.guest_memory.lock().unwrap().replace(guest_memory);
+
+        Ok(())
+    }
+
     fn hotplug_ram_region(&mut self, size: usize) -> Result<(), Error> {
         info!("Hotplugging new RAM: {}", size);
 
@@ -370,13 +434,7 @@ impl MemoryManager {
 
         self.next_hotplug_slot += 1;
 
-        // Update the GuestMemoryMmap with the new range
-        let guest_memory = self
-            .guest_memory
-            .memory()
-            .insert_region(region)
-            .map_err(Error::GuestMemory)?;
-        self.guest_memory.lock().unwrap().replace(guest_memory);
+        self.add_region(region)?;
 
         Ok(())
     }
@@ -453,14 +511,39 @@ impl MemoryManager {
         Ok(slot)
     }
 
-    pub fn resize(&mut self, desired_ram: u64) -> Result<bool, Error> {
-        if desired_ram > self.current_ram {
-            self.hotplug_ram_region((desired_ram - self.current_ram) as usize)?;
-            self.current_ram = desired_ram;
-            Ok(true)
-        } else {
-            Ok(false)
+    pub fn virtiomem_resize(&mut self, size: u64) -> Result<(), Error> {
+        let region = self.virtiomem_region.take();
+        if let Some(region) = region {
+            self.add_region(region)?;
         }
+
+        if let Some(resize) = &self.virtiomem_resize {
+            resize.work(size).map_err(Error::VirtioMemResizeFail)?;
+        } else {
+            panic!("should not fail here");
+        }
+
+        Ok(())
+    }
+
+    pub fn resize(&mut self, desired_ram: u64) -> Result<bool, Error> {
+        let mut notify_hotplug = false;
+        match self.hotplug_method {
+            HotplugMethod::VirtioMem => {
+                if desired_ram >= self.boot_ram {
+                    self.virtiomem_resize(desired_ram - self.boot_ram)?;
+                    self.current_ram = desired_ram;
+                }
+            }
+            HotplugMethod::Acpi => {
+                if desired_ram >= self.current_ram {
+                    self.hotplug_ram_region((desired_ram - self.current_ram) as usize)?;
+                    self.current_ram = desired_ram;
+                    notify_hotplug = true
+                }
+            }
+        }
+        Ok(notify_hotplug)
     }
 }
 

vmm/src/vm.rs

@@ -169,6 +169,9 @@ pub enum Error {
 
     /// No PCI support
     NoPciSupport,
+
+    /// Eventfd write error
+    EventfdError(std::io::Error),
 }
 pub type Result<T> = result::Result<T, Error>;
 
@@ -337,6 +340,7 @@ impl Vm {
             allocator.clone(),
             fd.clone(),
             memory_config.size,
+            memory_config.hotplug_method,
             memory_config.hotplug_size,
             &memory_config.file,
             memory_config.mergeable,