From fa0565234c4dd5c6023792ef5651600931864a74 Mon Sep 17 00:00:00 2001 From: Sebastien Boeuf Date: Tue, 2 Aug 2022 12:11:41 +0200 Subject: [PATCH] tests: Make test_api_shutdown and test_api_delete more reliable Both of these tests have been sporadically failing through multiple CI runs. The reason is related to cloud-init which fails to run the "init-local" script during the second boot of the VM. This causes the network interface to not be available, and therefore the test can't SSH into the VM as expected. The root cause is the filesystem and cache corruption that happens on the cloud-init disk. The way to prevent from this issue is to sync the guest filesystem before we shut it down, and as a security harness, we also wait for a few seconds for the shutdown command to complete inside the guest before we trigger the API shutdown or delete. Signed-off-by: Sebastien Boeuf --- tests/integration.rs | 34 ++++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/tests/integration.rs b/tests/integration.rs index 7dc219948..d0d655366 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -4104,26 +4104,31 @@ mod parallel { DIRECT_KERNEL_BOOT_CMDLINE, ); - curl_command( - &api_socket, - "PUT", - "http://localhost/api/v1/vm.create", - Some(&http_body), - ); - - // Then boot it - curl_command(&api_socket, "PUT", "http://localhost/api/v1/vm.boot", None); - let r = std::panic::catch_unwind(|| { + curl_command( + &api_socket, + "PUT", + "http://localhost/api/v1/vm.create", + Some(&http_body), + ); + + // Then boot it + curl_command(&api_socket, "PUT", "http://localhost/api/v1/vm.boot", None); + guest.wait_vm_boot(None).unwrap(); // Check that the VM booted as expected assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); assert!(guest.get_total_memory().unwrap_or_default() > 480_000); - // Shutdown without powering off to prevent filesystem corruption + // Sync and shutdown without powering off to prevent filesystem + // corruption. + guest.ssh_command("sync").unwrap(); guest.ssh_command("sudo shutdown -H now").unwrap(); + // Wait for the guest to be fully shutdown + thread::sleep(std::time::Duration::new(20, 0)); + // Then shut it down curl_command( &api_socket, @@ -4194,9 +4199,14 @@ mod parallel { assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count); assert!(guest.get_total_memory().unwrap_or_default() > 480_000); - // Shutdown without powering off to prevent filesystem corruption + // Sync and shutdown without powering off to prevent filesystem + // corruption. + guest.ssh_command("sync").unwrap(); guest.ssh_command("sudo shutdown -H now").unwrap(); + // Wait for the guest to be fully shutdown + thread::sleep(std::time::Duration::new(20, 0)); + // Then delete it curl_command( &api_socket,