tests: live-migration: Add integration test for tcp live migration

Add integration test for tcp live migration to ensure live migration
functions as expected.

Signed-off-by: Jinrong Liang <cloudliang@tencent.com>

tests/integration.rs

@@ -12,12 +12,14 @@ extern crate test_infra;
 
 use std::collections::HashMap;
 use std::io::{BufRead, Read, Seek, Write};
+use std::net::TcpListener;
 use std::os::unix::io::AsRawFd;
 use std::path::PathBuf;
 use std::process::{Child, Command, Stdio};
 use std::string::String;
 use std::sync::mpsc::Receiver;
 use std::sync::{mpsc, Mutex};
+use std::time::Duration;
 use std::{fs, io, thread};
 
 use net_util::MacAddr;
@@ -10295,6 +10297,220 @@ mod live_migration {
         handle_child_output(r, &dest_output);
     }
 
+    // Function to get an available port
+    fn get_available_port() -> u16 {
+        TcpListener::bind("127.0.0.1:0")
+            .expect("Failed to bind to address")
+            .local_addr()
+            .unwrap()
+            .port()
+    }
+
+    fn start_live_migration_tcp(src_api_socket: &str, dest_api_socket: &str) -> bool {
+        // Get an available TCP port
+        let migration_port = get_available_port();
+        let host_ip = "127.0.0.1";
+
+        // Start the 'receive-migration' command on the destination
+        let mut receive_migration = Command::new(clh_command("ch-remote"))
+            .args([
+                &format!("--api-socket={}", dest_api_socket),
+                "receive-migration",
+                &format!("tcp:0.0.0.0:{}", migration_port),
+            ])
+            .stdin(Stdio::null())
+            .stderr(Stdio::piped())
+            .stdout(Stdio::piped())
+            .spawn()
+            .unwrap();
+
+        // Give the destination some time to start listening
+        thread::sleep(Duration::from_secs(1));
+
+        // Start the 'send-migration' command on the source
+        let mut send_migration = Command::new(clh_command("ch-remote"))
+            .args([
+                &format!("--api-socket={}", src_api_socket),
+                "send-migration",
+                &format!("tcp:{}:{}", host_ip, migration_port),
+            ])
+            .stdin(Stdio::null())
+            .stderr(Stdio::piped())
+            .stdout(Stdio::piped())
+            .spawn()
+            .unwrap();
+
+        // Check if the 'send-migration' command executed successfully
+        let send_success = if let Some(status) = send_migration
+            .wait_timeout(Duration::from_secs(60))
+            .unwrap()
+        {
+            status.success()
+        } else {
+            false
+        };
+
+        if !send_success {
+            let _ = send_migration.kill();
+            let output = send_migration.wait_with_output().unwrap();
+            eprintln!(
+                "\n\n==== Start 'send_migration' output ====\n\n---stdout---\n{}\n\n---stderr---\n{}\n\n==== End 'send_migration' output ====\n\n",
+                String::from_utf8_lossy(&output.stdout),
+                String::from_utf8_lossy(&output.stderr)
+            );
+        }
+
+        // Check if the 'receive-migration' command executed successfully
+        let receive_success = if let Some(status) = receive_migration
+            .wait_timeout(Duration::from_secs(60))
+            .unwrap()
+        {
+            status.success()
+        } else {
+            false
+        };
+
+        if !receive_success {
+            let _ = receive_migration.kill();
+            let output = receive_migration.wait_with_output().unwrap();
+            eprintln!(
+                "\n\n==== Start 'receive_migration' output ====\n\n---stdout---\n{}\n\n---stderr---\n{}\n\n==== End 'receive_migration' output ====\n\n",
+                String::from_utf8_lossy(&output.stdout),
+                String::from_utf8_lossy(&output.stderr)
+            );
+        }
+
+        send_success && receive_success
+    }
+
+    fn _test_live_migration_tcp() {
+        let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string());
+        let guest = Guest::new(Box::new(focal));
+        let kernel_path = direct_kernel_boot_path();
+        let console_text = String::from("On a branch floating down river a cricket, singing.");
+        let net_id = "net123";
+        let net_params = format!(
+            "id={},tap=,mac={},ip={},mask=255.255.255.0",
+            net_id, guest.network.guest_mac, guest.network.host_ip
+        );
+        let memory_param: &[&str] = &["--memory", "size=4G,shared=on"];
+        let boot_vcpus = 2;
+        let max_vcpus = 4;
+        let pmem_temp_file = TempFile::new().unwrap();
+        pmem_temp_file.as_file().set_len(128 << 20).unwrap();
+        std::process::Command::new("mkfs.ext4")
+            .arg(pmem_temp_file.as_path())
+            .output()
+            .expect("Expect creating disk image to succeed");
+        let pmem_path = String::from("/dev/pmem0");
+
+        // Start the source VM
+        let src_vm_path = clh_command("cloud-hypervisor");
+        let src_api_socket = temp_api_path(&guest.tmp_dir);
+        let mut src_vm_cmd = GuestCommand::new_with_binary_path(&guest, &src_vm_path);
+        src_vm_cmd
+            .args([
+                "--cpus",
+                format!("boot={},max={}", boot_vcpus, max_vcpus).as_str(),
+            ])
+            .args(memory_param)
+            .args(["--kernel", kernel_path.to_str().unwrap()])
+            .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE])
+            .default_disks()
+            .args(["--net", net_params.as_str()])
+            .args(["--api-socket", &src_api_socket])
+            .args([
+                "--pmem",
+                format!(
+                    "file={},discard_writes=on",
+                    pmem_temp_file.as_path().to_str().unwrap(),
+                )
+                .as_str(),
+            ])
+            .capture_output();
+        let mut src_child = src_vm_cmd.spawn().unwrap();
+
+        // Start the destination VM
+        let mut dest_api_socket = temp_api_path(&guest.tmp_dir);
+        dest_api_socket.push_str(".dest");
+        let mut dest_child = GuestCommand::new(&guest)
+            .args(["--api-socket", &dest_api_socket])
+            .capture_output()
+            .spawn()
+            .unwrap();
+
+        let r = std::panic::catch_unwind(|| {
+            guest.wait_vm_boot(None).unwrap();
+            // Ensure the source VM is running normally
+            assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus);
+            assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000);
+            guest.check_devices_common(None, Some(&console_text), Some(&pmem_path));
+
+            // On x86_64 architecture, remove and re-add the virtio-net device
+            #[cfg(target_arch = "x86_64")]
+            {
+                assert!(remote_command(
+                    &src_api_socket,
+                    "remove-device",
+                    Some(net_id),
+                ));
+                thread::sleep(Duration::new(10, 0));
+                // Re-add the virtio-net device
+                assert!(remote_command(
+                    &src_api_socket,
+                    "add-net",
+                    Some(net_params.as_str()),
+                ));
+                thread::sleep(Duration::new(10, 0));
+            }
+            // Start TCP live migration
+            assert!(
+                start_live_migration_tcp(&src_api_socket, &dest_api_socket),
+                "Unsuccessful command: 'send-migration' or 'receive-migration'."
+            );
+        });
+
+        // Check and report any errors that occurred during live migration
+        if r.is_err() {
+            print_and_panic(
+                src_child,
+                dest_child,
+                None,
+                "Error occurred during live-migration",
+            );
+        }
+
+        // Check the source vm has been terminated successful (give it '3s' to settle)
+        thread::sleep(std::time::Duration::new(3, 0));
+        if !src_child.try_wait().unwrap().is_some_and(|s| s.success()) {
+            print_and_panic(
+                src_child,
+                dest_child,
+                None,
+                "Source VM was not terminated successfully.",
+            );
+        };
+
+        // After live migration, ensure the destination VM is running normally
+        let r = std::panic::catch_unwind(|| {
+            // Perform the same checks to ensure the VM has migrated correctly
+            assert_eq!(guest.get_cpu_count().unwrap_or_default(), boot_vcpus);
+            assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000);
+            guest.check_devices_common(None, Some(&console_text), Some(&pmem_path));
+        });
+
+        // Clean up the destination VM and ensure it terminates properly
+        let _ = dest_child.kill();
+        let dest_output = dest_child.wait_with_output().unwrap();
+        handle_child_output(r, &dest_output);
+
+        // Check if the expected `console_text` is present in the destination VM's output
+        let r = std::panic::catch_unwind(|| {
+            assert!(String::from_utf8_lossy(&dest_output.stdout).contains(&console_text));
+        });
+        handle_child_output(r, &dest_output);
+    }
+
     mod live_migration_parallel {
         use super::*;
         #[test]
@@ -10307,6 +10523,11 @@ mod live_migration {
             _test_live_migration(false, true)
         }
 
+        #[test]
+        fn test_live_migration_tcp() {
+            _test_live_migration_tcp();
+        }
+
         #[test]
         fn test_live_migration_watchdog() {
             _test_live_migration_watchdog(false, false)