diff --git a/tests/integration.rs b/tests/integration.rs index fd1f18225..2056dcb96 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -12,12 +12,14 @@ extern crate test_infra; use std::collections::HashMap; use std::io::{BufRead, Read, Seek, Write}; +use std::net::TcpListener; use std::os::unix::io::AsRawFd; use std::path::PathBuf; use std::process::{Child, Command, Stdio}; use std::string::String; use std::sync::mpsc::Receiver; use std::sync::{mpsc, Mutex}; +use std::time::Duration; use std::{fs, io, thread}; use net_util::MacAddr; @@ -10295,6 +10297,220 @@ mod live_migration { handle_child_output(r, &dest_output); } + // Function to get an available port + fn get_available_port() -> u16 { + TcpListener::bind("127.0.0.1:0") + .expect("Failed to bind to address") + .local_addr() + .unwrap() + .port() + } + + fn start_live_migration_tcp(src_api_socket: &str, dest_api_socket: &str) -> bool { + // Get an available TCP port + let migration_port = get_available_port(); + let host_ip = "127.0.0.1"; + + // Start the 'receive-migration' command on the destination + let mut receive_migration = Command::new(clh_command("ch-remote")) + .args([ + &format!("--api-socket={}", dest_api_socket), + "receive-migration", + &format!("tcp:0.0.0.0:{}", migration_port), + ]) + .stdin(Stdio::null()) + .stderr(Stdio::piped()) + .stdout(Stdio::piped()) + .spawn() + .unwrap(); + + // Give the destination some time to start listening + thread::sleep(Duration::from_secs(1)); + + // Start the 'send-migration' command on the source + let mut send_migration = Command::new(clh_command("ch-remote")) + .args([ + &format!("--api-socket={}", src_api_socket), + "send-migration", + &format!("tcp:{}:{}", host_ip, migration_port), + ]) + .stdin(Stdio::null()) + .stderr(Stdio::piped()) + .stdout(Stdio::piped()) + .spawn() + .unwrap(); + + // Check if the 'send-migration' command executed successfully + let send_success = if let Some(status) = send_migration + .wait_timeout(Duration::from_secs(60)) + .unwrap() + { + status.success() + } else { + false + }; + + if !send_success { + let _ = send_migration.kill(); + let output = send_migration.wait_with_output().unwrap(); + eprintln!( + "\n\n==== Start 'send_migration' output ====\n\n---stdout---\n{}\n\n---stderr---\n{}\n\n==== End 'send_migration' output ====\n\n", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + } + + // Check if the 'receive-migration' command executed successfully + let receive_success = if let Some(status) = receive_migration + .wait_timeout(Duration::from_secs(60)) + .unwrap() + { + status.success() + } else { + false + }; + + if !receive_success { + let _ = receive_migration.kill(); + let output = receive_migration.wait_with_output().unwrap(); + eprintln!( + "\n\n==== Start 'receive_migration' output ====\n\n---stdout---\n{}\n\n---stderr---\n{}\n\n==== End 'receive_migration' output ====\n\n", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + } + + send_success && receive_success + } + + fn _test_live_migration_tcp() { + let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); + let guest = Guest::new(Box::new(focal)); + let kernel_path = direct_kernel_boot_path(); + let console_text = String::from("On a branch floating down river a cricket, singing."); + let net_id = "net123"; + let net_params = format!( + "id={},tap=,mac={},ip={},mask=255.255.255.0", + net_id, guest.network.guest_mac, guest.network.host_ip + ); + let memory_param: &[&str] = &["--memory", "size=4G,shared=on"]; + let boot_vcpus = 2; + let max_vcpus = 4; + let pmem_temp_file = TempFile::new().unwrap(); + pmem_temp_file.as_file().set_len(128 << 20).unwrap(); + std::process::Command::new("mkfs.ext4") + .arg(pmem_temp_file.as_path()) + .output() + .expect("Expect creating disk image to succeed"); + let pmem_path = String::from("/dev/pmem0"); + + // Start the source VM + let src_vm_path = clh_command("cloud-hypervisor"); + let src_api_socket = temp_api_path(&guest.tmp_dir); + let mut src_vm_cmd = GuestCommand::new_with_binary_path(&guest, &src_vm_path); + src_vm_cmd + .args([ + "--cpus", + format!("boot={},max={}", boot_vcpus, max_vcpus).as_str(), + ]) + .args(memory_param) + .args(["--kernel", kernel_path.to_str().unwrap()]) + .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE]) + .default_disks() + .args(["--net", net_params.as_str()]) + .args(["--api-socket", &src_api_socket]) + .args([ + "--pmem", + format!( + "file={},discard_writes=on", + pmem_temp_file.as_path().to_str().unwrap(), + ) + .as_str(), + ]) + .capture_output(); + let mut src_child = src_vm_cmd.spawn().unwrap(); + + // Start the destination VM + let mut dest_api_socket = temp_api_path(&guest.tmp_dir); + dest_api_socket.push_str(".dest"); + let mut dest_child = GuestCommand::new(&guest) + .args(["--api-socket", &dest_api_socket]) + .capture_output() + .spawn() + .unwrap(); + + let r = std::panic::catch_unwind(|| { + guest.wait_vm_boot(None).unwrap(); + // Ensure the source VM is running normally + assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); + assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); + guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); + + // On x86_64 architecture, remove and re-add the virtio-net device + #[cfg(target_arch = "x86_64")] + { + assert!(remote_command( + &src_api_socket, + "remove-device", + Some(net_id), + )); + thread::sleep(Duration::new(10, 0)); + // Re-add the virtio-net device + assert!(remote_command( + &src_api_socket, + "add-net", + Some(net_params.as_str()), + )); + thread::sleep(Duration::new(10, 0)); + } + // Start TCP live migration + assert!( + start_live_migration_tcp(&src_api_socket, &dest_api_socket), + "Unsuccessful command: 'send-migration' or 'receive-migration'." + ); + }); + + // Check and report any errors that occurred during live migration + if r.is_err() { + print_and_panic( + src_child, + dest_child, + None, + "Error occurred during live-migration", + ); + } + + // Check the source vm has been terminated successful (give it '3s' to settle) + thread::sleep(std::time::Duration::new(3, 0)); + if !src_child.try_wait().unwrap().is_some_and(|s| s.success()) { + print_and_panic( + src_child, + dest_child, + None, + "Source VM was not terminated successfully.", + ); + }; + + // After live migration, ensure the destination VM is running normally + let r = std::panic::catch_unwind(|| { + // Perform the same checks to ensure the VM has migrated correctly + assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus); + assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); + guest.check_devices_common(None, Some(&console_text), Some(&pmem_path)); + }); + + // Clean up the destination VM and ensure it terminates properly + let _ = dest_child.kill(); + let dest_output = dest_child.wait_with_output().unwrap(); + handle_child_output(r, &dest_output); + + // Check if the expected `console_text` is present in the destination VM's output + let r = std::panic::catch_unwind(|| { + assert!(String::from_utf8_lossy(&dest_output.stdout).contains(&console_text)); + }); + handle_child_output(r, &dest_output); + } + mod live_migration_parallel { use super::*; #[test] @@ -10307,6 +10523,11 @@ mod live_migration { _test_live_migration(false, true) } + #[test] + fn test_live_migration_tcp() { + _test_live_migration_tcp(); + } + #[test] fn test_live_migration_watchdog() { _test_live_migration_watchdog(false, false)