vmm: Split VM config and VM state for snapshot/restore

In order to allow for human readable output for the VM configuration, we
pull it out of the snapshot, which becomes effectively the list of
states from the VM. The configuration is stored through a dedicated file
in JSON format (not including any binary output).

Having the ability to read and modify the VM configuration manually
between the snapshot and restore phases makes debugging easier, as well
as empowers users for extending the use cases relying on the
snapshot/restore feature.

Signed-off-by: Sebastien Boeuf <sebastien.boeuf@intel.com>

docs/snapshot_restore.md

@@ -43,17 +43,22 @@ ll /home/foo/snapshot/
 total 4194536
 drwxrwxr-x  2 foo bar       4096 Jul 22 11:50 ./
 drwxr-xr-x 47 foo bar       4096 Jul 22 11:47 ../
+-rw-------  1 foo bar       1084 Jul 22 11:19 config.json
 -rw-------  1 foo bar 4294967296 Jul 22 11:19 memory-ranges
--rw-------  1 foo bar     217853 Jul 22 11:19 vm.json
+-rw-------  1 foo bar     217853 Jul 22 11:19 state.json
 ```
 
+`config.json` contains the virtual machine configuration. It is used to create
+a similar virtual machine with the correct amount of CPUs, RAM, and other
+expected devices. It is stored in a human readable format so that it could be
+modified between the snapshot and restore phases to achieve some very special
+use cases. But for most cases, manually modifying the configuration should not
+be needed.
+
 `memory-ranges` stores the content of the guest RAM.
 
-`vm.json` gathers all information related to the virtual machine configuration
+`state.json` contains the virtual machine state. It is used to restore each
-and state. The configuration bits are used to create a similar virtual machine
+component in the state it was left before the snapshot occurred.
-with the correct amount of CPUs, RAM, and other expected devices. The state
-bits are used to restore each component in the state it was left before the
-snapshot occurred.
 
 ## Restore a Cloud Hypervisor VM
 

vmm/src/lib.rs

@@ -20,7 +20,9 @@ use crate::config::{
     DeviceConfig, DiskConfig, FsConfig, NetConfig, PmemConfig, RestoreConfig, UserDeviceConfig,
     VmConfig, VsockConfig,
 };
-use crate::migration::{get_vm_snapshot, recv_vm_snapshot};
+#[cfg(all(feature = "kvm", target_arch = "x86_64"))]
+use crate::migration::get_vm_snapshot;
+use crate::migration::{recv_vm_config, recv_vm_state};
 use crate::seccomp_filters::{get_seccomp_filter, Thread};
 use crate::vm::{Error as VmError, Vm, VmState};
 use anyhow::anyhow;
@@ -443,14 +445,18 @@ impl Vmm {
         // Safe to unwrap as we checked it was Some(&str).
         let source_url = source_url.unwrap();
 
-        let snapshot = recv_vm_snapshot(source_url).map_err(VmError::Restore)?;
+        let vm_config = Arc::new(Mutex::new(
+            recv_vm_config(source_url).map_err(VmError::Restore)?,
+        ));
+        let snapshot = recv_vm_state(source_url).map_err(VmError::Restore)?;
+        #[cfg(all(feature = "kvm", target_arch = "x86_64"))]
         let vm_snapshot = get_vm_snapshot(&snapshot).map_err(VmError::Restore)?;
 
         #[cfg(all(feature = "kvm", target_arch = "x86_64"))]
-        self.vm_check_cpuid_compatibility(&vm_snapshot.config, &vm_snapshot.common_cpuid)
+        self.vm_check_cpuid_compatibility(&vm_config, &vm_snapshot.common_cpuid)
             .map_err(VmError::Restore)?;
 
-        self.vm_config = Some(Arc::clone(&vm_snapshot.config));
+        self.vm_config = Some(Arc::clone(&vm_config));
 
         let exit_evt = self.exit_evt.try_clone().map_err(VmError::EventFdClone)?;
         let reset_evt = self.reset_evt.try_clone().map_err(VmError::EventFdClone)?;
@@ -461,6 +467,7 @@ impl Vmm {
 
         let vm = Vm::new_from_snapshot(
             &snapshot,
+            vm_config,
             exit_evt,
             reset_evt,
             Some(source_url),

vmm/src/migration.rs

@@ -2,14 +2,18 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 
-use crate::vm::{VmSnapshot, VM_SNAPSHOT_ID};
+use crate::{
+    config::VmConfig,
+    vm::{VmSnapshot, VM_SNAPSHOT_ID},
+};
 use anyhow::anyhow;
 use std::fs::File;
 use std::io::BufReader;
 use std::path::PathBuf;
 use vm_migration::{MigratableError, Snapshot};
 
-pub const VM_SNAPSHOT_FILE: &str = "vm.json";
+pub const SNAPSHOT_STATE_FILE: &str = "state.json";
+pub const SNAPSHOT_CONFIG_FILE: &str = "config.json";
 
 pub fn url_to_path(url: &str) -> std::result::Result<PathBuf, MigratableError> {
     let path: PathBuf = url
@@ -28,19 +32,28 @@ pub fn url_to_path(url: &str) -> std::result::Result<PathBuf, MigratableError> {
     Ok(path)
 }
 
-pub fn recv_vm_snapshot(source_url: &str) -> std::result::Result<Snapshot, MigratableError> {
+pub fn recv_vm_config(source_url: &str) -> std::result::Result<VmConfig, MigratableError> {
-    let mut vm_snapshot_path = url_to_path(source_url)?;
+    let mut vm_config_path = url_to_path(source_url)?;
 
-    vm_snapshot_path.push(VM_SNAPSHOT_FILE);
+    vm_config_path.push(SNAPSHOT_CONFIG_FILE);
 
     // Try opening the snapshot file
-    let vm_snapshot_file =
+    let vm_config_file =
-        File::open(vm_snapshot_path).map_err(|e| MigratableError::MigrateSend(e.into()))?;
+        File::open(vm_config_path).map_err(|e| MigratableError::MigrateSend(e.into()))?;
-    let vm_snapshot_reader = BufReader::new(vm_snapshot_file);
+    let vm_config_reader = BufReader::new(vm_config_file);
-    let vm_snapshot = serde_json::from_reader(vm_snapshot_reader)
+    serde_json::from_reader(vm_config_reader).map_err(|e| MigratableError::MigrateReceive(e.into()))
-        .map_err(|e| MigratableError::MigrateReceive(e.into()))?;
+}
 
-    Ok(vm_snapshot)
+pub fn recv_vm_state(source_url: &str) -> std::result::Result<Snapshot, MigratableError> {
+    let mut vm_state_path = url_to_path(source_url)?;
+
+    vm_state_path.push(SNAPSHOT_STATE_FILE);
+
+    // Try opening the snapshot file
+    let vm_state_file =
+        File::open(vm_state_path).map_err(|e| MigratableError::MigrateSend(e.into()))?;
+    let vm_state_reader = BufReader::new(vm_state_file);
+    serde_json::from_reader(vm_state_reader).map_err(|e| MigratableError::MigrateReceive(e.into()))
 }
 
 pub fn get_vm_snapshot(snapshot: &Snapshot) -> std::result::Result<VmSnapshot, MigratableError> {

vmm/src/vm.rs

@@ -23,7 +23,7 @@ use crate::device_tree::DeviceTree;
 use crate::memory_manager::{
     Error as MemoryManagerError, MemoryManager, MemoryManagerSnapshotData,
 };
-use crate::migration::{get_vm_snapshot, url_to_path, VM_SNAPSHOT_FILE};
+use crate::migration::{get_vm_snapshot, url_to_path, SNAPSHOT_CONFIG_FILE, SNAPSHOT_STATE_FILE};
 use crate::seccomp_filters::{get_seccomp_filter, Thread};
 use crate::GuestMemoryMmap;
 use crate::{
@@ -820,6 +820,7 @@ impl Vm {
     #[allow(clippy::too_many_arguments)]
     pub fn new_from_snapshot(
         snapshot: &Snapshot,
+        vm_config: Arc<Mutex<VmConfig>>,
         exit_evt: EventFd,
         reset_evt: EventFd,
         source_url: Option<&str>,
@@ -840,7 +841,6 @@ impl Vm {
         }
 
         let vm_snapshot = get_vm_snapshot(snapshot).map_err(Error::Restore)?;
-        let config = vm_snapshot.config;
         if let Some(state) = vm_snapshot.state {
             vm.set_state(state)
                 .map_err(|e| Error::Restore(MigratableError::Restore(e.into())))?;
@@ -849,11 +849,11 @@ impl Vm {
         let memory_manager = if let Some(memory_manager_snapshot) =
             snapshot.snapshots.get(MEMORY_MANAGER_SNAPSHOT_ID)
         {
-            let phys_bits = physical_bits(config.lock().unwrap().cpus.max_phys_bits);
+            let phys_bits = physical_bits(vm_config.lock().unwrap().cpus.max_phys_bits);
             MemoryManager::new_from_snapshot(
                 memory_manager_snapshot,
                 vm.clone(),
-                &config.lock().unwrap().memory.clone(),
+                &vm_config.lock().unwrap().memory.clone(),
                 source_url,
                 prefault,
                 phys_bits,
@@ -866,7 +866,7 @@ impl Vm {
         };
 
         Vm::new_from_memory_manager(
-            config,
+            vm_config,
             memory_manager,
             vm,
             exit_evt,
@@ -2524,7 +2524,6 @@ impl Pausable for Vm {
 
 #[derive(Serialize, Deserialize)]
 pub struct VmSnapshot {
-    pub config: Arc<Mutex<VmConfig>>,
     #[cfg(all(feature = "kvm", target_arch = "x86_64"))]
     pub clock: Option<hypervisor::ClockData>,
     pub state: Option<hypervisor::VmState>,
@@ -2582,7 +2581,6 @@ impl Snapshottable for Vm {
             .state()
             .map_err(|e| MigratableError::Snapshot(e.into()))?;
         let vm_snapshot_data = serde_json::to_vec(&VmSnapshot {
-            config: self.get_config(),
             #[cfg(all(feature = "kvm", target_arch = "x86_64"))]
             clock: self.saved_clock,
             state: Some(vm_state),
@@ -2702,23 +2700,42 @@ impl Transportable for Vm {
         snapshot: &Snapshot,
         destination_url: &str,
     ) -> std::result::Result<(), MigratableError> {
-        let mut vm_snapshot_path = url_to_path(destination_url)?;
+        let mut snapshot_config_path = url_to_path(destination_url)?;
-        vm_snapshot_path.push(VM_SNAPSHOT_FILE);
+        snapshot_config_path.push(SNAPSHOT_CONFIG_FILE);
 
-        // Create the snapshot file
+        // Create the snapshot config file
-        let mut vm_snapshot_file = OpenOptions::new()
+        let mut snapshot_config_file = OpenOptions::new()
             .read(true)
             .write(true)
             .create_new(true)
-            .open(vm_snapshot_path)
+            .open(snapshot_config_path)
             .map_err(|e| MigratableError::MigrateSend(e.into()))?;
 
-        // Serialize and write the snapshot
+        // Serialize and write the snapshot config
-        let vm_snapshot =
+        let vm_config = serde_json::to_string(self.config.lock().unwrap().deref())
+            .map_err(|e| MigratableError::MigrateSend(e.into()))?;
+
+        snapshot_config_file
+            .write(vm_config.as_bytes())
+            .map_err(|e| MigratableError::MigrateSend(e.into()))?;
+
+        let mut snapshot_state_path = url_to_path(destination_url)?;
+        snapshot_state_path.push(SNAPSHOT_STATE_FILE);
+
+        // Create the snapshot state file
+        let mut snapshot_state_file = OpenOptions::new()
+            .read(true)
+            .write(true)
+            .create_new(true)
+            .open(snapshot_state_path)
+            .map_err(|e| MigratableError::MigrateSend(e.into()))?;
+
+        // Serialize and write the snapshot state
+        let vm_state =
             serde_json::to_vec(snapshot).map_err(|e| MigratableError::MigrateSend(e.into()))?;
 
-        vm_snapshot_file
+        snapshot_state_file
-            .write(&vm_snapshot)
+            .write(&vm_state)
             .map_err(|e| MigratableError::MigrateSend(e.into()))?;
 
         // Tell the memory manager to also send/write its own snapshot.