From a181b77bc88048b26b178e1799aae0540195e597 Mon Sep 17 00:00:00 2001 From: Bo Chen Date: Wed, 8 Sep 2021 21:44:27 -0700 Subject: [PATCH] ci: Add integration test for live migration with OVS-DPDK Signed-off-by: Bo Chen --- Jenkinsfile | 2 + .../run_integration_tests_live_migration.sh | 11 +- tests/integration.rs | 193 ++++++++++++++++++ 3 files changed, 205 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index bbe92cb29..8b630bcae 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -228,6 +228,7 @@ pipeline{ timeout(time: 1, unit: 'HOURS') } steps { + sh "sudo modprobe openvswitch" sh "scripts/dev_cli.sh tests --integration-live-migration" } } @@ -236,6 +237,7 @@ pipeline{ timeout(time: 1, unit: 'HOURS') } steps { + sh "sudo modprobe openvswitch" sh "scripts/dev_cli.sh tests --integration-live-migration --libc musl" } } diff --git a/scripts/run_integration_tests_live_migration.sh b/scripts/run_integration_tests_live_migration.sh index 0fc3e7475..42dfd4d32 100755 --- a/scripts/run_integration_tests_live_migration.sh +++ b/scripts/run_integration_tests_live_migration.sh @@ -79,8 +79,17 @@ strip target/$BUILD_TARGET/release/cloud-hypervisor strip target/$BUILD_TARGET/release/vhost_user_net strip target/$BUILD_TARGET/release/ch-remote -export RUST_BACKTRACE=1 +# Test ovs-dpdk relies on hugepages +echo 6144 | sudo tee /proc/sys/vm/nr_hugepages +sudo chmod a+rwX /dev/hugepages +# Setup ovs-dpdk +service openvswitch-switch start +ovs-vsctl init +ovs-vsctl set Open_vSwitch . other_config:dpdk-init=true +service openvswitch-switch restart + +export RUST_BACKTRACE=1 time cargo test $features_test "tests::live_migration::$test_filter" -- --test-threads=1 RES=$? diff --git a/tests/integration.rs b/tests/integration.rs index 795587a6b..83da380b6 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -7019,6 +7019,199 @@ mod tests { fn test_live_migration_numa() { _test_live_migration(true) } + #[test] + fn test_live_migration_ovs_dpdk() { + let ovs_focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); + let ovs_guest = Guest::new(Box::new(ovs_focal)); + + let migration_focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); + let migration_guest = Guest::new(Box::new(migration_focal)); + let src_api_socket = temp_api_path(&migration_guest.tmp_dir); + + // Start two VMs that are connected through ovs-dpdk and one of the VMs is the source VM for live-migration + let (mut ovs_child, mut src_child) = + setup_ovs_dpdk_guests(&ovs_guest, &migration_guest, &src_api_socket); + + // Start the destination VM + let mut dest_api_socket = temp_api_path(&migration_guest.tmp_dir); + dest_api_socket.push_str(".dest"); + let mut dest_child = GuestCommand::new(&migration_guest) + .args(&["--api-socket", &dest_api_socket]) + .capture_output() + .spawn() + .unwrap(); + + let r = std::panic::catch_unwind(|| { + // Give it '1s' to make sure the 'dest_api_socket' file is properly created + thread::sleep(std::time::Duration::new(1, 0)); + + // Start the live-migration + let migration_socket = String::from( + migration_guest + .tmp_dir + .as_path() + .join("live-migration.sock") + .to_str() + .unwrap(), + ); + // Start to receive migration from the destintion VM + let mut receive_migration = Command::new(clh_command("ch-remote")) + .args(&[ + &format!("--api-socket={}", &dest_api_socket), + "receive-migration", + &format! {"unix:{}", migration_socket}, + ]) + .stderr(Stdio::piped()) + .stdout(Stdio::piped()) + .spawn() + .unwrap(); + // Give it '1s' to make sure the 'migration_socket' file is properly created + thread::sleep(std::time::Duration::new(1, 0)); + // Start to send migration from the source VM + let mut send_migration = Command::new(clh_command("ch-remote")) + .args(&[ + &format!("--api-socket={}", &src_api_socket), + "send-migration", + &format! {"unix:{}", migration_socket}, + ]) + .stderr(Stdio::piped()) + .stdout(Stdio::piped()) + .spawn() + .unwrap(); + + let mut success; + // The 'send-migration' command should be executed successfully within the given timeout + if let Some(status) = send_migration + .wait_timeout(std::time::Duration::from_secs(30)) + .unwrap() + { + success = status.success(); + } else { + success = false; + } + if !success { + let _ = send_migration.kill(); + let output = send_migration.wait_with_output().unwrap(); + eprintln!("\n\n==== Start 'send_migration' output ====\n\n---stdout---\n{}\n\n---stderr---\n{}\n\n==== End 'send_migration' output ====\n\n", + String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr)); + } + // The 'receive-migration' command should be executed successfully within the given timeout + if let Some(status) = receive_migration + .wait_timeout(std::time::Duration::from_secs(30)) + .unwrap() + { + success = status.success(); + } else { + success = false; + } + if !success { + let _ = receive_migration.kill(); + let output = receive_migration.wait_with_output().unwrap(); + eprintln!("\n\n==== Start 'receive_migration' output ====\n\n---stdout---\n{}\n\n---stderr---\n{}\n\n==== End 'receive_migration' output ====\n\n", + String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr)); + } + assert!( + success, + "Unsuccessful command: 'send-migration' or 'receive-migration'." + ); + }); + + let print_and_panic = |src_vm: Child, + dest_vm: Child, + ovs_vm: Child, + message: &str| + -> ! { + let mut src_vm = src_vm; + let mut dest_vm = dest_vm; + let mut ovs_vm = ovs_vm; + + let _ = src_vm.kill(); + let src_output = src_vm.wait_with_output().unwrap(); + eprintln!( + "\n\n==== Start 'source_vm' stdout ====\n\n{}\n\n==== End 'source_vm' stdout ====", + String::from_utf8_lossy(&src_output.stdout) + ); + eprintln!( + "\n\n==== Start 'source_vm' stderr ====\n\n{}\n\n==== End 'source_vm' stderr ====", + String::from_utf8_lossy(&src_output.stderr) + ); + let _ = dest_vm.kill(); + let dest_output = dest_vm.wait_with_output().unwrap(); + eprintln!( + "\n\n==== Start 'destination_vm' stdout ====\n\n{}\n\n==== End 'destination_vm' stdout ====", + String::from_utf8_lossy(&dest_output.stdout) + ); + eprintln!( + "\n\n==== Start 'destination_vm' stderr ====\n\n{}\n\n==== End 'destination_vm' stderr ====", + String::from_utf8_lossy(&dest_output.stderr) + ); + let _ = ovs_vm.kill(); + let ovs_output = ovs_vm.wait_with_output().unwrap(); + eprintln!( + "\n\n==== Start 'ovs_vm' stdout ====\n\n{}\n\n==== End 'ovs_vm' stdout ====", + String::from_utf8_lossy(&ovs_output.stdout) + ); + eprintln!( + "\n\n==== Start 'ovs_vm' stderr ====\n\n{}\n\n==== End 'ovs_vm' stderr ====", + String::from_utf8_lossy(&ovs_output.stderr) + ); + + panic!("Test failed: {}", message) + }; + + // Check and report any errors occured during the live-migration + if r.is_err() { + print_and_panic( + src_child, + dest_child, + ovs_child, + "Error occured during live-migration", + ); + } + + // Check the source vm has been terminated successful (give it '3s' to settle) + thread::sleep(std::time::Duration::new(3, 0)); + if !src_child.try_wait().unwrap().map_or(false, |s| s.success()) { + print_and_panic( + src_child, + dest_child, + ovs_child, + "source VM was not terminated successfully.", + ); + }; + + // Post live-migration check to make sure the destination VM is funcational + let r = std::panic::catch_unwind(|| { + // Perform same checks to validate VM has been properly migrated + // Spawn a new netcat listener in the OVS VM + let guest_ip = ovs_guest.network.guest_ip.clone(); + thread::spawn(move || { + ssh_command_ip( + "nc -l 12345", + &guest_ip, + DEFAULT_SSH_RETRIES, + DEFAULT_SSH_TIMEOUT, + ) + .unwrap(); + }); + + // Wait for the server to be listening + thread::sleep(std::time::Duration::new(5, 0)); + + // And check the connection is still functional after live-migration + migration_guest + .ssh_command("nc -vz 172.100.0.1 12345") + .unwrap(); + }); + + // Clean-up the destination VM and OVS VM, and make sure they terminated correctly + let _ = dest_child.kill(); + let _ = ovs_child.kill(); + let dest_output = dest_child.wait_with_output().unwrap(); + handle_child_output(r, &dest_output); + let ovs_output = ovs_child.wait_with_output().unwrap(); + handle_child_output(Ok(()), &ovs_output); + } } #[cfg(all(target_arch = "aarch64", feature = "acpi"))]