tests: Make test_api_shutdown and test_api_delete more reliable

Both of these tests have been sporadically failing through multiple CI
runs. The reason is related to cloud-init which fails to run the
"init-local" script during the second boot of the VM. This causes the
network interface to not be available, and therefore the test can't SSH
into the VM as expected. The root cause is the filesystem and cache
corruption that happens on the cloud-init disk.

The way to prevent from this issue is to sync the guest filesystem
before we shut it down, and as a security harness, we also wait for a
few seconds for the shutdown command to complete inside the guest before
we trigger the API shutdown or delete.

Signed-off-by: Sebastien Boeuf <sebastien.boeuf@intel.com>
This commit is contained in:
Sebastien Boeuf 2022-08-02 12:11:41 +02:00
parent 755eb80c49
commit fa0565234c

View File

@ -4104,26 +4104,31 @@ mod parallel {
DIRECT_KERNEL_BOOT_CMDLINE,
);
curl_command(
&api_socket,
"PUT",
"http://localhost/api/v1/vm.create",
Some(&http_body),
);
// Then boot it
curl_command(&api_socket, "PUT", "http://localhost/api/v1/vm.boot", None);
let r = std::panic::catch_unwind(|| {
curl_command(
&api_socket,
"PUT",
"http://localhost/api/v1/vm.create",
Some(&http_body),
);
// Then boot it
curl_command(&api_socket, "PUT", "http://localhost/api/v1/vm.boot", None);
guest.wait_vm_boot(None).unwrap();
// Check that the VM booted as expected
assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count);
assert!(guest.get_total_memory().unwrap_or_default() > 480_000);
// Shutdown without powering off to prevent filesystem corruption
// Sync and shutdown without powering off to prevent filesystem
// corruption.
guest.ssh_command("sync").unwrap();
guest.ssh_command("sudo shutdown -H now").unwrap();
// Wait for the guest to be fully shutdown
thread::sleep(std::time::Duration::new(20, 0));
// Then shut it down
curl_command(
&api_socket,
@ -4194,9 +4199,14 @@ mod parallel {
assert_eq!(guest.get_cpu_count().unwrap_or_default() as u8, cpu_count);
assert!(guest.get_total_memory().unwrap_or_default() > 480_000);
// Shutdown without powering off to prevent filesystem corruption
// Sync and shutdown without powering off to prevent filesystem
// corruption.
guest.ssh_command("sync").unwrap();
guest.ssh_command("sudo shutdown -H now").unwrap();
// Wait for the guest to be fully shutdown
thread::sleep(std::time::Duration::new(20, 0));
// Then delete it
curl_command(
&api_socket,