mirror of
https://github.com/cloud-hypervisor/cloud-hypervisor.git
synced 2024-12-22 05:35:20 +00:00
tests: live-migration: Add integration test for tcp live migration
Add integration test for tcp live migration to ensure live migration functions as expected. Signed-off-by: Jinrong Liang <cloudliang@tencent.com>
This commit is contained in:
parent
b9163bf431
commit
bbd8d3b71d
@ -12,12 +12,14 @@ extern crate test_infra;
|
|||||||
|
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
use std::io::{BufRead, Read, Seek, Write};
|
use std::io::{BufRead, Read, Seek, Write};
|
||||||
|
use std::net::TcpListener;
|
||||||
use std::os::unix::io::AsRawFd;
|
use std::os::unix::io::AsRawFd;
|
||||||
use std::path::PathBuf;
|
use std::path::PathBuf;
|
||||||
use std::process::{Child, Command, Stdio};
|
use std::process::{Child, Command, Stdio};
|
||||||
use std::string::String;
|
use std::string::String;
|
||||||
use std::sync::mpsc::Receiver;
|
use std::sync::mpsc::Receiver;
|
||||||
use std::sync::{mpsc, Mutex};
|
use std::sync::{mpsc, Mutex};
|
||||||
|
use std::time::Duration;
|
||||||
use std::{fs, io, thread};
|
use std::{fs, io, thread};
|
||||||
|
|
||||||
use net_util::MacAddr;
|
use net_util::MacAddr;
|
||||||
@ -10295,6 +10297,220 @@ mod live_migration {
|
|||||||
handle_child_output(r, &dest_output);
|
handle_child_output(r, &dest_output);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Function to get an available port
|
||||||
|
fn get_available_port() -> u16 {
|
||||||
|
TcpListener::bind("127.0.0.1:0")
|
||||||
|
.expect("Failed to bind to address")
|
||||||
|
.local_addr()
|
||||||
|
.unwrap()
|
||||||
|
.port()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn start_live_migration_tcp(src_api_socket: &str, dest_api_socket: &str) -> bool {
|
||||||
|
// Get an available TCP port
|
||||||
|
let migration_port = get_available_port();
|
||||||
|
let host_ip = "127.0.0.1";
|
||||||
|
|
||||||
|
// Start the 'receive-migration' command on the destination
|
||||||
|
let mut receive_migration = Command::new(clh_command("ch-remote"))
|
||||||
|
.args([
|
||||||
|
&format!("--api-socket={}", dest_api_socket),
|
||||||
|
"receive-migration",
|
||||||
|
&format!("tcp:0.0.0.0:{}", migration_port),
|
||||||
|
])
|
||||||
|
.stdin(Stdio::null())
|
||||||
|
.stderr(Stdio::piped())
|
||||||
|
.stdout(Stdio::piped())
|
||||||
|
.spawn()
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
// Give the destination some time to start listening
|
||||||
|
thread::sleep(Duration::from_secs(1));
|
||||||
|
|
||||||
|
// Start the 'send-migration' command on the source
|
||||||
|
let mut send_migration = Command::new(clh_command("ch-remote"))
|
||||||
|
.args([
|
||||||
|
&format!("--api-socket={}", src_api_socket),
|
||||||
|
"send-migration",
|
||||||
|
&format!("tcp:{}:{}", host_ip, migration_port),
|
||||||
|
])
|
||||||
|
.stdin(Stdio::null())
|
||||||
|
.stderr(Stdio::piped())
|
||||||
|
.stdout(Stdio::piped())
|
||||||
|
.spawn()
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
// Check if the 'send-migration' command executed successfully
|
||||||
|
let send_success = if let Some(status) = send_migration
|
||||||
|
.wait_timeout(Duration::from_secs(60))
|
||||||
|
.unwrap()
|
||||||
|
{
|
||||||
|
status.success()
|
||||||
|
} else {
|
||||||
|
false
|
||||||
|
};
|
||||||
|
|
||||||
|
if !send_success {
|
||||||
|
let _ = send_migration.kill();
|
||||||
|
let output = send_migration.wait_with_output().unwrap();
|
||||||
|
eprintln!(
|
||||||
|
"\n\n==== Start 'send_migration' output ====\n\n---stdout---\n{}\n\n---stderr---\n{}\n\n==== End 'send_migration' output ====\n\n",
|
||||||
|
String::from_utf8_lossy(&output.stdout),
|
||||||
|
String::from_utf8_lossy(&output.stderr)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check if the 'receive-migration' command executed successfully
|
||||||
|
let receive_success = if let Some(status) = receive_migration
|
||||||
|
.wait_timeout(Duration::from_secs(60))
|
||||||
|
.unwrap()
|
||||||
|
{
|
||||||
|
status.success()
|
||||||
|
} else {
|
||||||
|
false
|
||||||
|
};
|
||||||
|
|
||||||
|
if !receive_success {
|
||||||
|
let _ = receive_migration.kill();
|
||||||
|
let output = receive_migration.wait_with_output().unwrap();
|
||||||
|
eprintln!(
|
||||||
|
"\n\n==== Start 'receive_migration' output ====\n\n---stdout---\n{}\n\n---stderr---\n{}\n\n==== End 'receive_migration' output ====\n\n",
|
||||||
|
String::from_utf8_lossy(&output.stdout),
|
||||||
|
String::from_utf8_lossy(&output.stderr)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
send_success && receive_success
|
||||||
|
}
|
||||||
|
|
||||||
|
fn _test_live_migration_tcp() {
|
||||||
|
let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string());
|
||||||
|
let guest = Guest::new(Box::new(focal));
|
||||||
|
let kernel_path = direct_kernel_boot_path();
|
||||||
|
let console_text = String::from("On a branch floating down river a cricket, singing.");
|
||||||
|
let net_id = "net123";
|
||||||
|
let net_params = format!(
|
||||||
|
"id={},tap=,mac={},ip={},mask=255.255.255.0",
|
||||||
|
net_id, guest.network.guest_mac, guest.network.host_ip
|
||||||
|
);
|
||||||
|
let memory_param: &[&str] = &["--memory", "size=4G,shared=on"];
|
||||||
|
let boot_vcpus = 2;
|
||||||
|
let max_vcpus = 4;
|
||||||
|
let pmem_temp_file = TempFile::new().unwrap();
|
||||||
|
pmem_temp_file.as_file().set_len(128 << 20).unwrap();
|
||||||
|
std::process::Command::new("mkfs.ext4")
|
||||||
|
.arg(pmem_temp_file.as_path())
|
||||||
|
.output()
|
||||||
|
.expect("Expect creating disk image to succeed");
|
||||||
|
let pmem_path = String::from("/dev/pmem0");
|
||||||
|
|
||||||
|
// Start the source VM
|
||||||
|
let src_vm_path = clh_command("cloud-hypervisor");
|
||||||
|
let src_api_socket = temp_api_path(&guest.tmp_dir);
|
||||||
|
let mut src_vm_cmd = GuestCommand::new_with_binary_path(&guest, &src_vm_path);
|
||||||
|
src_vm_cmd
|
||||||
|
.args([
|
||||||
|
"--cpus",
|
||||||
|
format!("boot={},max={}", boot_vcpus, max_vcpus).as_str(),
|
||||||
|
])
|
||||||
|
.args(memory_param)
|
||||||
|
.args(["--kernel", kernel_path.to_str().unwrap()])
|
||||||
|
.args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE])
|
||||||
|
.default_disks()
|
||||||
|
.args(["--net", net_params.as_str()])
|
||||||
|
.args(["--api-socket", &src_api_socket])
|
||||||
|
.args([
|
||||||
|
"--pmem",
|
||||||
|
format!(
|
||||||
|
"file={},discard_writes=on",
|
||||||
|
pmem_temp_file.as_path().to_str().unwrap(),
|
||||||
|
)
|
||||||
|
.as_str(),
|
||||||
|
])
|
||||||
|
.capture_output();
|
||||||
|
let mut src_child = src_vm_cmd.spawn().unwrap();
|
||||||
|
|
||||||
|
// Start the destination VM
|
||||||
|
let mut dest_api_socket = temp_api_path(&guest.tmp_dir);
|
||||||
|
dest_api_socket.push_str(".dest");
|
||||||
|
let mut dest_child = GuestCommand::new(&guest)
|
||||||
|
.args(["--api-socket", &dest_api_socket])
|
||||||
|
.capture_output()
|
||||||
|
.spawn()
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let r = std::panic::catch_unwind(|| {
|
||||||
|
guest.wait_vm_boot(None).unwrap();
|
||||||
|
// Ensure the source VM is running normally
|
||||||
|
assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus);
|
||||||
|
assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000);
|
||||||
|
guest.check_devices_common(None, Some(&console_text), Some(&pmem_path));
|
||||||
|
|
||||||
|
// On x86_64 architecture, remove and re-add the virtio-net device
|
||||||
|
#[cfg(target_arch = "x86_64")]
|
||||||
|
{
|
||||||
|
assert!(remote_command(
|
||||||
|
&src_api_socket,
|
||||||
|
"remove-device",
|
||||||
|
Some(net_id),
|
||||||
|
));
|
||||||
|
thread::sleep(Duration::new(10, 0));
|
||||||
|
// Re-add the virtio-net device
|
||||||
|
assert!(remote_command(
|
||||||
|
&src_api_socket,
|
||||||
|
"add-net",
|
||||||
|
Some(net_params.as_str()),
|
||||||
|
));
|
||||||
|
thread::sleep(Duration::new(10, 0));
|
||||||
|
}
|
||||||
|
// Start TCP live migration
|
||||||
|
assert!(
|
||||||
|
start_live_migration_tcp(&src_api_socket, &dest_api_socket),
|
||||||
|
"Unsuccessful command: 'send-migration' or 'receive-migration'."
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
// Check and report any errors that occurred during live migration
|
||||||
|
if r.is_err() {
|
||||||
|
print_and_panic(
|
||||||
|
src_child,
|
||||||
|
dest_child,
|
||||||
|
None,
|
||||||
|
"Error occurred during live-migration",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check the source vm has been terminated successful (give it '3s' to settle)
|
||||||
|
thread::sleep(std::time::Duration::new(3, 0));
|
||||||
|
if !src_child.try_wait().unwrap().is_some_and(|s| s.success()) {
|
||||||
|
print_and_panic(
|
||||||
|
src_child,
|
||||||
|
dest_child,
|
||||||
|
None,
|
||||||
|
"Source VM was not terminated successfully.",
|
||||||
|
);
|
||||||
|
};
|
||||||
|
|
||||||
|
// After live migration, ensure the destination VM is running normally
|
||||||
|
let r = std::panic::catch_unwind(|| {
|
||||||
|
// Perform the same checks to ensure the VM has migrated correctly
|
||||||
|
assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus);
|
||||||
|
assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000);
|
||||||
|
guest.check_devices_common(None, Some(&console_text), Some(&pmem_path));
|
||||||
|
});
|
||||||
|
|
||||||
|
// Clean up the destination VM and ensure it terminates properly
|
||||||
|
let _ = dest_child.kill();
|
||||||
|
let dest_output = dest_child.wait_with_output().unwrap();
|
||||||
|
handle_child_output(r, &dest_output);
|
||||||
|
|
||||||
|
// Check if the expected `console_text` is present in the destination VM's output
|
||||||
|
let r = std::panic::catch_unwind(|| {
|
||||||
|
assert!(String::from_utf8_lossy(&dest_output.stdout).contains(&console_text));
|
||||||
|
});
|
||||||
|
handle_child_output(r, &dest_output);
|
||||||
|
}
|
||||||
|
|
||||||
mod live_migration_parallel {
|
mod live_migration_parallel {
|
||||||
use super::*;
|
use super::*;
|
||||||
#[test]
|
#[test]
|
||||||
@ -10307,6 +10523,11 @@ mod live_migration {
|
|||||||
_test_live_migration(false, true)
|
_test_live_migration(false, true)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_live_migration_tcp() {
|
||||||
|
_test_live_migration_tcp();
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_live_migration_watchdog() {
|
fn test_live_migration_watchdog() {
|
||||||
_test_live_migration_watchdog(false, false)
|
_test_live_migration_watchdog(false, false)
|
||||||
|
Loading…
Reference in New Issue
Block a user