External/cloud-hypervisor
1
0
Fork 0
mirror of https://github.com/cloud-hypervisor/cloud-hypervisor.git synced 2024-10-01 02:55:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

vm-migration, vmm: Send configuration in separate step

Browse Source 
Prior to sending the memory the full state is not needed only the
configuration. This is sufficient to create the appropriate structures
in the guest and have the memory allocations ready for filling.

Update the protocol documentation to add a separate config step and move
the state to after the memory is transferred. As the VM is created in a
separate step to restoring it the requires a slightly different
constructor as well as saving the VM object for the subsequent commands.

Signed-off-by: Rob Bradford <robert.bradford@intel.com>
This commit is contained in:
Rob Bradford 2020-11-12 16:55:55 +00:00
parent c62e409827
commit 11a69450ba
3 changed files with 147 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
vm-migration/src/protocol.rs
View File

@ -10,14 +10,17 @@ use crate::MigratableError;
// (The establishment is out of scope.)
// 2: Source -> Dest : send "start command"
// 3: Dest -> Source : sends "ok response" when read to accept state data
// 4: Source -> Dest : sends "state command" followed by state data, length
// in command is length of state data
// 4: Source -> Dest : sends "config command" followed by config data, length
// in command is length of config data
// 5: Dest -> Source : sends "ok response" when ready to accept memory data
// 6: Source -> Dest : send "memory command" followed by table of u64 pairs (GPA, size)
// followed by the memory described in those pairs.
// !! length is size of table i.e. 16 * number of ranges !!
// 7: Dest -> Source : sends "ok response" when ready to accept more memory data
// 8..(n-2): Repeat steps 6 and 7 until source has no more memory to send
// 8..(n-4): Repeat steps 6 and 7 until source has no more memory to send
// (n-3): Source -> Dest : sends "state command" followed by state data, length
// in command is length of config data
// (n-2): Dest -> Source : sends "ok response"
// (n-1): Source -> Dest : send "complete command"
// n: Dest -> Source: sends "ok response"
@ -31,6 +34,7 @@ use std::io::{Read, Write};
pub enum Command {
Invalid,
Start,
Config,
State,
Memory,
Complete,
@ -82,6 +86,10 @@ impl Request {
Self::new(Command::State, length)
}
pub fn config(length: u64) -> Self {
Self::new(Command::Config, length)
}
pub fn memory(length: u64) -> Self {
Self::new(Command::Memory, length)
}

157
vmm/src/lib.rs
View File

@ -647,10 +647,53 @@ impl Vmm {
}
}
fn vm_receive_config<T>(
&mut self,
req: &Request,
socket: &mut T,
) -> std::result::Result<Vm, MigratableError>
where
T: Read + Write,
{
// Read in config data
let mut data = Vec::with_capacity(req.length() as usize);
unsafe {
data.set_len(req.length() as usize);
}
socket
.read_exact(&mut data)
.map_err(MigratableError::MigrateSocket)?;
let config: VmConfig = serde_json::from_slice(&data).map_err(|e| {
MigratableError::MigrateReceive(anyhow!("Error deserialising config: {}", e))
})?;
let exit_evt = self.exit_evt.try_clone().map_err(|e| {
MigratableError::MigrateReceive(anyhow!("Error cloning exit EventFd: {}", e))
})?;
let reset_evt = self.reset_evt.try_clone().map_err(|e| {
MigratableError::MigrateReceive(anyhow!("Error cloning reset EventFd: {}", e))
})?;
let vm = Vm::new_from_migration(
Arc::new(Mutex::new(config)),
exit_evt,
reset_evt,
&self.seccomp_action,
self.hypervisor.clone(),
)
.map_err(|e| {
MigratableError::MigrateReceive(anyhow!("Error creating VM from snapshot: {:?}", e))
})?;
Response::ok().write_to(socket)?;
Ok(vm)
}
fn vm_receive_state<T>(
&mut self,
req: &Request,
socket: &mut T,
mut vm: Vm,
) -> std::result::Result<(), MigratableError>
where
T: Read + Write,
@ -668,26 +711,6 @@ impl Vmm {
})?;
// Create VM
let vm_snapshot = get_vm_snapshot(&snapshot)?;
self.vm_config = Some(Arc::clone(&vm_snapshot.config));
let exit_evt = self.exit_evt.try_clone().map_err(|e| {
MigratableError::MigrateReceive(anyhow!("Error cloning exit EventFd: {}", e))
})?;
let reset_evt = self.reset_evt.try_clone().map_err(|e| {
MigratableError::MigrateReceive(anyhow!("Error cloning reset EventFd: {}", e))
})?;
let mut vm = Vm::new_from_snapshot(
&snapshot,
exit_evt,
reset_evt,
None,
false,
&self.seccomp_action,
self.hypervisor.clone(),
)
.map_err(|e| {
MigratableError::MigrateReceive(anyhow!("Error creating VM from snapshot: {:?}", e))
})?;
vm.restore(snapshot).map_err(|e| {
Response::error().write_to(socket).ok();
e
@ -703,6 +726,7 @@ impl Vmm {
&mut self,
req: &Request,
socket: &mut T,
vm: &mut Vm,
) -> std::result::Result<(), MigratableError>
where
T: Read + Write,
@ -711,14 +735,10 @@ impl Vmm {
let table = MemoryRangeTable::read_from(socket, req.length())?;
// And then read the memory itself
self.vm
.as_mut()
.unwrap()
.receive_memory_regions(&table, socket)
.map_err(|e| {
Response::error().write_to(socket).ok();
e
})?;
vm.receive_memory_regions(&table, socket).map_err(|e| {
Response::error().write_to(socket).ok();
e
})?;
Response::ok().write_to(socket)?;
Ok(())
}
@ -759,6 +779,7 @@ impl Vmm {
};
let mut started = false;
let mut vm: Option<Vm> = None;
loop {
let req = Request::read_from(&mut socket)?;
@ -770,6 +791,16 @@ impl Vmm {
Response::ok().write_to(&mut socket)?;
}
Command::Config => {
info!("Config Command Received");
if !started {
warn!("Migration not started yet");
Response::error().write_to(&mut socket)?;
continue;
}
vm = Some(self.vm_receive_config(&req, &mut socket)?);
}
Command::State => {
info!("State Command Received");
@ -778,15 +809,12 @@ impl Vmm {
Response::error().write_to(&mut socket)?;
continue;
}
// TODO: Remove this when doing live migration
if self.vm.is_some() {
warn!("State already sent");
if let Some(vm) = vm.take() {
self.vm_receive_state(&req, &mut socket, vm)?;
} else {
warn!("Configuration not sent yet");
Response::error().write_to(&mut socket)?;
continue;
}
self.vm_receive_state(&req, &mut socket)?;
}
Command::Memory => {
info!("Memory Command Received");
@ -796,8 +824,12 @@ impl Vmm {
Response::error().write_to(&mut socket)?;
continue;
}
self.vm_receive_memory(&req, &mut socket)?;
if let Some(ref mut vm) = vm.as_mut() {
self.vm_receive_memory(&req, &mut socket, vm)?;
} else {
warn!("Configuration not sent yet");
Response::error().write_to(&mut socket)?;
}
}
Command::Complete => {
info!("Complete Command Received");
@ -853,6 +885,39 @@ impl Vmm {
)));
}
// Send config
let config_data = serde_json::to_vec(&vm.get_config()).unwrap();
Request::config(config_data.len() as u64).write_to(&mut socket)?;
socket
.write_all(&config_data)
.map_err(MigratableError::MigrateSocket)?;
let res = Response::read_from(&mut socket)?;
if res.status() != Status::Ok {
warn!("Error during config migration");
Request::abandon().write_to(&mut socket)?;
Response::read_from(&mut socket).ok();
return Err(MigratableError::MigrateSend(anyhow!(
"Error during config migration"
)));
}
// Send memory table
let table = vm.memory_range_table()?;
Request::memory(table.length())
.write_to(&mut socket)
.unwrap();
table.write_to(&mut socket)?;
// And then the memory itself
vm.send_memory_regions(&table, &mut socket)?;
if res.status() != Status::Ok {
warn!("Error during memory migration");
Request::abandon().write_to(&mut socket)?;
Response::read_from(&mut socket).ok();
return Err(MigratableError::MigrateSend(anyhow!(
"Error during memory migration"
)));
}
// Capture snapshot and send it
let vm_snapshot = vm.snapshot()?;
let snapshot_data = serde_json::to_vec(&vm_snapshot).unwrap();
@ -870,24 +935,6 @@ impl Vmm {
)));
}
// Send memory table
let table = vm.memory_range_table()?;
Request::memory(table.length())
.write_to(&mut socket)
.unwrap();
table.write_to(&mut socket)?;
// And then the memory itself
vm.send_memory_regions(&table, &mut socket)?;
if res.status() != Status::Ok {
warn!("Error during memory migration");
Request::abandon().write_to(&mut socket)?;
Response::read_from(&mut socket).ok();
return Err(MigratableError::MigrateSend(anyhow!(
"Error during memory migration"
)));
}
// Complete the migration
Request::complete().write_to(&mut socket)?;
let res = Response::read_from(&mut socket)?;

34
vmm/src/vm.rs
View File

@ -732,6 +732,40 @@ impl Vm {
)
}
pub fn new_from_migration(
config: Arc<Mutex<VmConfig>>,
exit_evt: EventFd,
reset_evt: EventFd,
seccomp_action: &SeccompAction,
hypervisor: Arc<dyn hypervisor::Hypervisor>,
) -> Result<Self> {
#[cfg(target_arch = "x86_64")]
hypervisor.check_required_extensions().unwrap();
let vm = hypervisor.create_vm().unwrap();
#[cfg(target_arch = "x86_64")]
vm.enable_split_irq().unwrap();
let phys_bits = physical_bits(config.lock().unwrap().cpus.max_phys_bits);
let memory_manager = MemoryManager::new(
vm.clone(),
&config.lock().unwrap().memory.clone(),
false,
phys_bits,
)
.map_err(Error::MemoryManager)?;
Vm::new_from_memory_manager(
config,
memory_manager,
vm,
exit_evt,
reset_evt,
seccomp_action,
hypervisor,
None,
)
}
fn load_initramfs(&mut self, guest_mem: &GuestMemoryMmap) -> Result<arch::InitramfsConfig> {
let mut initramfs = self.initramfs.as_ref().unwrap();
let size: usize = initramfs