From 10676b74dcea89c8d865617bdc9f33d48d0ee748 Mon Sep 17 00:00:00 2001 From: Sebastien Boeuf Date: Tue, 8 Feb 2022 11:32:19 +0100 Subject: [PATCH] vmm: Split VM config and VM state for snapshot/restore In order to allow for human readable output for the VM configuration, we pull it out of the snapshot, which becomes effectively the list of states from the VM. The configuration is stored through a dedicated file in JSON format (not including any binary output). Having the ability to read and modify the VM configuration manually between the snapshot and restore phases makes debugging easier, as well as empowers users for extending the use cases relying on the snapshot/restore feature. Signed-off-by: Sebastien Boeuf --- docs/snapshot_restore.md | 17 +++++++++----- vmm/src/lib.rs | 15 ++++++++---- vmm/src/migration.rs | 35 +++++++++++++++++++--------- vmm/src/vm.rs | 49 +++++++++++++++++++++++++++------------- 4 files changed, 79 insertions(+), 37 deletions(-) diff --git a/docs/snapshot_restore.md b/docs/snapshot_restore.md index 72ce2e8f1..4e89c2dc8 100644 --- a/docs/snapshot_restore.md +++ b/docs/snapshot_restore.md @@ -43,17 +43,22 @@ ll /home/foo/snapshot/ total 4194536 drwxrwxr-x 2 foo bar 4096 Jul 22 11:50 ./ drwxr-xr-x 47 foo bar 4096 Jul 22 11:47 ../ +-rw------- 1 foo bar 1084 Jul 22 11:19 config.json -rw------- 1 foo bar 4294967296 Jul 22 11:19 memory-ranges --rw------- 1 foo bar 217853 Jul 22 11:19 vm.json +-rw------- 1 foo bar 217853 Jul 22 11:19 state.json ``` +`config.json` contains the virtual machine configuration. It is used to create +a similar virtual machine with the correct amount of CPUs, RAM, and other +expected devices. It is stored in a human readable format so that it could be +modified between the snapshot and restore phases to achieve some very special +use cases. But for most cases, manually modifying the configuration should not +be needed. + `memory-ranges` stores the content of the guest RAM. -`vm.json` gathers all information related to the virtual machine configuration -and state. The configuration bits are used to create a similar virtual machine -with the correct amount of CPUs, RAM, and other expected devices. The state -bits are used to restore each component in the state it was left before the -snapshot occurred. +`state.json` contains the virtual machine state. It is used to restore each +component in the state it was left before the snapshot occurred. ## Restore a Cloud Hypervisor VM diff --git a/vmm/src/lib.rs b/vmm/src/lib.rs index 4dae86554..c0daa74ec 100644 --- a/vmm/src/lib.rs +++ b/vmm/src/lib.rs @@ -20,7 +20,9 @@ use crate::config::{ DeviceConfig, DiskConfig, FsConfig, NetConfig, PmemConfig, RestoreConfig, UserDeviceConfig, VmConfig, VsockConfig, }; -use crate::migration::{get_vm_snapshot, recv_vm_snapshot}; +#[cfg(all(feature = "kvm", target_arch = "x86_64"))] +use crate::migration::get_vm_snapshot; +use crate::migration::{recv_vm_config, recv_vm_state}; use crate::seccomp_filters::{get_seccomp_filter, Thread}; use crate::vm::{Error as VmError, Vm, VmState}; use anyhow::anyhow; @@ -443,14 +445,18 @@ impl Vmm { // Safe to unwrap as we checked it was Some(&str). let source_url = source_url.unwrap(); - let snapshot = recv_vm_snapshot(source_url).map_err(VmError::Restore)?; + let vm_config = Arc::new(Mutex::new( + recv_vm_config(source_url).map_err(VmError::Restore)?, + )); + let snapshot = recv_vm_state(source_url).map_err(VmError::Restore)?; + #[cfg(all(feature = "kvm", target_arch = "x86_64"))] let vm_snapshot = get_vm_snapshot(&snapshot).map_err(VmError::Restore)?; #[cfg(all(feature = "kvm", target_arch = "x86_64"))] - self.vm_check_cpuid_compatibility(&vm_snapshot.config, &vm_snapshot.common_cpuid) + self.vm_check_cpuid_compatibility(&vm_config, &vm_snapshot.common_cpuid) .map_err(VmError::Restore)?; - self.vm_config = Some(Arc::clone(&vm_snapshot.config)); + self.vm_config = Some(Arc::clone(&vm_config)); let exit_evt = self.exit_evt.try_clone().map_err(VmError::EventFdClone)?; let reset_evt = self.reset_evt.try_clone().map_err(VmError::EventFdClone)?; @@ -461,6 +467,7 @@ impl Vmm { let vm = Vm::new_from_snapshot( &snapshot, + vm_config, exit_evt, reset_evt, Some(source_url), diff --git a/vmm/src/migration.rs b/vmm/src/migration.rs index 3c2ffa30f..e0e0daa0c 100644 --- a/vmm/src/migration.rs +++ b/vmm/src/migration.rs @@ -2,14 +2,18 @@ // // SPDX-License-Identifier: Apache-2.0 -use crate::vm::{VmSnapshot, VM_SNAPSHOT_ID}; +use crate::{ + config::VmConfig, + vm::{VmSnapshot, VM_SNAPSHOT_ID}, +}; use anyhow::anyhow; use std::fs::File; use std::io::BufReader; use std::path::PathBuf; use vm_migration::{MigratableError, Snapshot}; -pub const VM_SNAPSHOT_FILE: &str = "vm.json"; +pub const SNAPSHOT_STATE_FILE: &str = "state.json"; +pub const SNAPSHOT_CONFIG_FILE: &str = "config.json"; pub fn url_to_path(url: &str) -> std::result::Result { let path: PathBuf = url @@ -28,19 +32,28 @@ pub fn url_to_path(url: &str) -> std::result::Result { Ok(path) } -pub fn recv_vm_snapshot(source_url: &str) -> std::result::Result { - let mut vm_snapshot_path = url_to_path(source_url)?; +pub fn recv_vm_config(source_url: &str) -> std::result::Result { + let mut vm_config_path = url_to_path(source_url)?; - vm_snapshot_path.push(VM_SNAPSHOT_FILE); + vm_config_path.push(SNAPSHOT_CONFIG_FILE); // Try opening the snapshot file - let vm_snapshot_file = - File::open(vm_snapshot_path).map_err(|e| MigratableError::MigrateSend(e.into()))?; - let vm_snapshot_reader = BufReader::new(vm_snapshot_file); - let vm_snapshot = serde_json::from_reader(vm_snapshot_reader) - .map_err(|e| MigratableError::MigrateReceive(e.into()))?; + let vm_config_file = + File::open(vm_config_path).map_err(|e| MigratableError::MigrateSend(e.into()))?; + let vm_config_reader = BufReader::new(vm_config_file); + serde_json::from_reader(vm_config_reader).map_err(|e| MigratableError::MigrateReceive(e.into())) +} - Ok(vm_snapshot) +pub fn recv_vm_state(source_url: &str) -> std::result::Result { + let mut vm_state_path = url_to_path(source_url)?; + + vm_state_path.push(SNAPSHOT_STATE_FILE); + + // Try opening the snapshot file + let vm_state_file = + File::open(vm_state_path).map_err(|e| MigratableError::MigrateSend(e.into()))?; + let vm_state_reader = BufReader::new(vm_state_file); + serde_json::from_reader(vm_state_reader).map_err(|e| MigratableError::MigrateReceive(e.into())) } pub fn get_vm_snapshot(snapshot: &Snapshot) -> std::result::Result { diff --git a/vmm/src/vm.rs b/vmm/src/vm.rs index 509e3b14c..0e60212f4 100644 --- a/vmm/src/vm.rs +++ b/vmm/src/vm.rs @@ -23,7 +23,7 @@ use crate::device_tree::DeviceTree; use crate::memory_manager::{ Error as MemoryManagerError, MemoryManager, MemoryManagerSnapshotData, }; -use crate::migration::{get_vm_snapshot, url_to_path, VM_SNAPSHOT_FILE}; +use crate::migration::{get_vm_snapshot, url_to_path, SNAPSHOT_CONFIG_FILE, SNAPSHOT_STATE_FILE}; use crate::seccomp_filters::{get_seccomp_filter, Thread}; use crate::GuestMemoryMmap; use crate::{ @@ -820,6 +820,7 @@ impl Vm { #[allow(clippy::too_many_arguments)] pub fn new_from_snapshot( snapshot: &Snapshot, + vm_config: Arc>, exit_evt: EventFd, reset_evt: EventFd, source_url: Option<&str>, @@ -840,7 +841,6 @@ impl Vm { } let vm_snapshot = get_vm_snapshot(snapshot).map_err(Error::Restore)?; - let config = vm_snapshot.config; if let Some(state) = vm_snapshot.state { vm.set_state(state) .map_err(|e| Error::Restore(MigratableError::Restore(e.into())))?; @@ -849,11 +849,11 @@ impl Vm { let memory_manager = if let Some(memory_manager_snapshot) = snapshot.snapshots.get(MEMORY_MANAGER_SNAPSHOT_ID) { - let phys_bits = physical_bits(config.lock().unwrap().cpus.max_phys_bits); + let phys_bits = physical_bits(vm_config.lock().unwrap().cpus.max_phys_bits); MemoryManager::new_from_snapshot( memory_manager_snapshot, vm.clone(), - &config.lock().unwrap().memory.clone(), + &vm_config.lock().unwrap().memory.clone(), source_url, prefault, phys_bits, @@ -866,7 +866,7 @@ impl Vm { }; Vm::new_from_memory_manager( - config, + vm_config, memory_manager, vm, exit_evt, @@ -2524,7 +2524,6 @@ impl Pausable for Vm { #[derive(Serialize, Deserialize)] pub struct VmSnapshot { - pub config: Arc>, #[cfg(all(feature = "kvm", target_arch = "x86_64"))] pub clock: Option, pub state: Option, @@ -2582,7 +2581,6 @@ impl Snapshottable for Vm { .state() .map_err(|e| MigratableError::Snapshot(e.into()))?; let vm_snapshot_data = serde_json::to_vec(&VmSnapshot { - config: self.get_config(), #[cfg(all(feature = "kvm", target_arch = "x86_64"))] clock: self.saved_clock, state: Some(vm_state), @@ -2702,23 +2700,42 @@ impl Transportable for Vm { snapshot: &Snapshot, destination_url: &str, ) -> std::result::Result<(), MigratableError> { - let mut vm_snapshot_path = url_to_path(destination_url)?; - vm_snapshot_path.push(VM_SNAPSHOT_FILE); + let mut snapshot_config_path = url_to_path(destination_url)?; + snapshot_config_path.push(SNAPSHOT_CONFIG_FILE); - // Create the snapshot file - let mut vm_snapshot_file = OpenOptions::new() + // Create the snapshot config file + let mut snapshot_config_file = OpenOptions::new() .read(true) .write(true) .create_new(true) - .open(vm_snapshot_path) + .open(snapshot_config_path) .map_err(|e| MigratableError::MigrateSend(e.into()))?; - // Serialize and write the snapshot - let vm_snapshot = + // Serialize and write the snapshot config + let vm_config = serde_json::to_string(self.config.lock().unwrap().deref()) + .map_err(|e| MigratableError::MigrateSend(e.into()))?; + + snapshot_config_file + .write(vm_config.as_bytes()) + .map_err(|e| MigratableError::MigrateSend(e.into()))?; + + let mut snapshot_state_path = url_to_path(destination_url)?; + snapshot_state_path.push(SNAPSHOT_STATE_FILE); + + // Create the snapshot state file + let mut snapshot_state_file = OpenOptions::new() + .read(true) + .write(true) + .create_new(true) + .open(snapshot_state_path) + .map_err(|e| MigratableError::MigrateSend(e.into()))?; + + // Serialize and write the snapshot state + let vm_state = serde_json::to_vec(snapshot).map_err(|e| MigratableError::MigrateSend(e.into()))?; - vm_snapshot_file - .write(&vm_snapshot) + snapshot_state_file + .write(&vm_state) .map_err(|e| MigratableError::MigrateSend(e.into()))?; // Tell the memory manager to also send/write its own snapshot.