From 11a69450ba31b957c1c7e0045e91829b53d0effd Mon Sep 17 00:00:00 2001 From: Rob Bradford Date: Thu, 12 Nov 2020 16:55:55 +0000 Subject: [PATCH] vm-migration, vmm: Send configuration in separate step Prior to sending the memory the full state is not needed only the configuration. This is sufficient to create the appropriate structures in the guest and have the memory allocations ready for filling. Update the protocol documentation to add a separate config step and move the state to after the memory is transferred. As the VM is created in a separate step to restoring it the requires a slightly different constructor as well as saving the VM object for the subsequent commands. Signed-off-by: Rob Bradford --- vm-migration/src/protocol.rs | 14 +++- vmm/src/lib.rs | 157 +++++++++++++++++++++++------------ vmm/src/vm.rs | 34 ++++++++ 3 files changed, 147 insertions(+), 58 deletions(-) diff --git a/vm-migration/src/protocol.rs b/vm-migration/src/protocol.rs index 550e3c76c..6c896fc02 100644 --- a/vm-migration/src/protocol.rs +++ b/vm-migration/src/protocol.rs @@ -10,14 +10,17 @@ use crate::MigratableError; // (The establishment is out of scope.) // 2: Source -> Dest : send "start command" // 3: Dest -> Source : sends "ok response" when read to accept state data -// 4: Source -> Dest : sends "state command" followed by state data, length -// in command is length of state data +// 4: Source -> Dest : sends "config command" followed by config data, length +// in command is length of config data // 5: Dest -> Source : sends "ok response" when ready to accept memory data // 6: Source -> Dest : send "memory command" followed by table of u64 pairs (GPA, size) // followed by the memory described in those pairs. // !! length is size of table i.e. 16 * number of ranges !! // 7: Dest -> Source : sends "ok response" when ready to accept more memory data -// 8..(n-2): Repeat steps 6 and 7 until source has no more memory to send +// 8..(n-4): Repeat steps 6 and 7 until source has no more memory to send +// (n-3): Source -> Dest : sends "state command" followed by state data, length +// in command is length of config data +// (n-2): Dest -> Source : sends "ok response" // (n-1): Source -> Dest : send "complete command" // n: Dest -> Source: sends "ok response" @@ -31,6 +34,7 @@ use std::io::{Read, Write}; pub enum Command { Invalid, Start, + Config, State, Memory, Complete, @@ -82,6 +86,10 @@ impl Request { Self::new(Command::State, length) } + pub fn config(length: u64) -> Self { + Self::new(Command::Config, length) + } + pub fn memory(length: u64) -> Self { Self::new(Command::Memory, length) } diff --git a/vmm/src/lib.rs b/vmm/src/lib.rs index 399c862ce..a70158c54 100644 --- a/vmm/src/lib.rs +++ b/vmm/src/lib.rs @@ -647,10 +647,53 @@ impl Vmm { } } + fn vm_receive_config( + &mut self, + req: &Request, + socket: &mut T, + ) -> std::result::Result + where + T: Read + Write, + { + // Read in config data + let mut data = Vec::with_capacity(req.length() as usize); + unsafe { + data.set_len(req.length() as usize); + } + socket + .read_exact(&mut data) + .map_err(MigratableError::MigrateSocket)?; + let config: VmConfig = serde_json::from_slice(&data).map_err(|e| { + MigratableError::MigrateReceive(anyhow!("Error deserialising config: {}", e)) + })?; + + let exit_evt = self.exit_evt.try_clone().map_err(|e| { + MigratableError::MigrateReceive(anyhow!("Error cloning exit EventFd: {}", e)) + })?; + let reset_evt = self.reset_evt.try_clone().map_err(|e| { + MigratableError::MigrateReceive(anyhow!("Error cloning reset EventFd: {}", e)) + })?; + let vm = Vm::new_from_migration( + Arc::new(Mutex::new(config)), + exit_evt, + reset_evt, + &self.seccomp_action, + self.hypervisor.clone(), + ) + .map_err(|e| { + MigratableError::MigrateReceive(anyhow!("Error creating VM from snapshot: {:?}", e)) + })?; + + Response::ok().write_to(socket)?; + + Ok(vm) + } + fn vm_receive_state( &mut self, req: &Request, socket: &mut T, + mut vm: Vm, ) -> std::result::Result<(), MigratableError> where T: Read + Write, @@ -668,26 +711,6 @@ impl Vmm { })?; // Create VM - let vm_snapshot = get_vm_snapshot(&snapshot)?; - self.vm_config = Some(Arc::clone(&vm_snapshot.config)); - let exit_evt = self.exit_evt.try_clone().map_err(|e| { - MigratableError::MigrateReceive(anyhow!("Error cloning exit EventFd: {}", e)) - })?; - let reset_evt = self.reset_evt.try_clone().map_err(|e| { - MigratableError::MigrateReceive(anyhow!("Error cloning reset EventFd: {}", e)) - })?; - let mut vm = Vm::new_from_snapshot( - &snapshot, - exit_evt, - reset_evt, - None, - false, - &self.seccomp_action, - self.hypervisor.clone(), - ) - .map_err(|e| { - MigratableError::MigrateReceive(anyhow!("Error creating VM from snapshot: {:?}", e)) - })?; vm.restore(snapshot).map_err(|e| { Response::error().write_to(socket).ok(); e @@ -703,6 +726,7 @@ impl Vmm { &mut self, req: &Request, socket: &mut T, + vm: &mut Vm, ) -> std::result::Result<(), MigratableError> where T: Read + Write, @@ -711,14 +735,10 @@ impl Vmm { let table = MemoryRangeTable::read_from(socket, req.length())?; // And then read the memory itself - self.vm - .as_mut() - .unwrap() - .receive_memory_regions(&table, socket) - .map_err(|e| { - Response::error().write_to(socket).ok(); - e - })?; + vm.receive_memory_regions(&table, socket).map_err(|e| { + Response::error().write_to(socket).ok(); + e + })?; Response::ok().write_to(socket)?; Ok(()) } @@ -759,6 +779,7 @@ impl Vmm { }; let mut started = false; + let mut vm: Option = None; loop { let req = Request::read_from(&mut socket)?; @@ -770,6 +791,16 @@ impl Vmm { Response::ok().write_to(&mut socket)?; } + Command::Config => { + info!("Config Command Received"); + + if !started { + warn!("Migration not started yet"); + Response::error().write_to(&mut socket)?; + continue; + } + vm = Some(self.vm_receive_config(&req, &mut socket)?); + } Command::State => { info!("State Command Received"); @@ -778,15 +809,12 @@ impl Vmm { Response::error().write_to(&mut socket)?; continue; } - - // TODO: Remove this when doing live migration - if self.vm.is_some() { - warn!("State already sent"); + if let Some(vm) = vm.take() { + self.vm_receive_state(&req, &mut socket, vm)?; + } else { + warn!("Configuration not sent yet"); Response::error().write_to(&mut socket)?; - continue; } - - self.vm_receive_state(&req, &mut socket)?; } Command::Memory => { info!("Memory Command Received"); @@ -796,8 +824,12 @@ impl Vmm { Response::error().write_to(&mut socket)?; continue; } - - self.vm_receive_memory(&req, &mut socket)?; + if let Some(ref mut vm) = vm.as_mut() { + self.vm_receive_memory(&req, &mut socket, vm)?; + } else { + warn!("Configuration not sent yet"); + Response::error().write_to(&mut socket)?; + } } Command::Complete => { info!("Complete Command Received"); @@ -853,6 +885,39 @@ impl Vmm { ))); } + // Send config + let config_data = serde_json::to_vec(&vm.get_config()).unwrap(); + Request::config(config_data.len() as u64).write_to(&mut socket)?; + socket + .write_all(&config_data) + .map_err(MigratableError::MigrateSocket)?; + let res = Response::read_from(&mut socket)?; + if res.status() != Status::Ok { + warn!("Error during config migration"); + Request::abandon().write_to(&mut socket)?; + Response::read_from(&mut socket).ok(); + return Err(MigratableError::MigrateSend(anyhow!( + "Error during config migration" + ))); + } + + // Send memory table + let table = vm.memory_range_table()?; + Request::memory(table.length()) + .write_to(&mut socket) + .unwrap(); + table.write_to(&mut socket)?; + // And then the memory itself + vm.send_memory_regions(&table, &mut socket)?; + if res.status() != Status::Ok { + warn!("Error during memory migration"); + Request::abandon().write_to(&mut socket)?; + Response::read_from(&mut socket).ok(); + return Err(MigratableError::MigrateSend(anyhow!( + "Error during memory migration" + ))); + } + // Capture snapshot and send it let vm_snapshot = vm.snapshot()?; let snapshot_data = serde_json::to_vec(&vm_snapshot).unwrap(); @@ -870,24 +935,6 @@ impl Vmm { ))); } - // Send memory table - let table = vm.memory_range_table()?; - Request::memory(table.length()) - .write_to(&mut socket) - .unwrap(); - table.write_to(&mut socket)?; - - // And then the memory itself - vm.send_memory_regions(&table, &mut socket)?; - if res.status() != Status::Ok { - warn!("Error during memory migration"); - Request::abandon().write_to(&mut socket)?; - Response::read_from(&mut socket).ok(); - return Err(MigratableError::MigrateSend(anyhow!( - "Error during memory migration" - ))); - } - // Complete the migration Request::complete().write_to(&mut socket)?; let res = Response::read_from(&mut socket)?; diff --git a/vmm/src/vm.rs b/vmm/src/vm.rs index 79d44b034..ee2bf7d21 100644 --- a/vmm/src/vm.rs +++ b/vmm/src/vm.rs @@ -732,6 +732,40 @@ impl Vm { ) } + pub fn new_from_migration( + config: Arc>, + exit_evt: EventFd, + reset_evt: EventFd, + seccomp_action: &SeccompAction, + hypervisor: Arc, + ) -> Result { + #[cfg(target_arch = "x86_64")] + hypervisor.check_required_extensions().unwrap(); + let vm = hypervisor.create_vm().unwrap(); + #[cfg(target_arch = "x86_64")] + vm.enable_split_irq().unwrap(); + let phys_bits = physical_bits(config.lock().unwrap().cpus.max_phys_bits); + + let memory_manager = MemoryManager::new( + vm.clone(), + &config.lock().unwrap().memory.clone(), + false, + phys_bits, + ) + .map_err(Error::MemoryManager)?; + + Vm::new_from_memory_manager( + config, + memory_manager, + vm, + exit_evt, + reset_evt, + seccomp_action, + hypervisor, + None, + ) + } + fn load_initramfs(&mut self, guest_mem: &GuestMemoryMmap) -> Result { let mut initramfs = self.initramfs.as_ref().unwrap(); let size: usize = initramfs