From b5be62a16780e757752dff2fe110b8187d834f46 Mon Sep 17 00:00:00 2001 From: Sebastien Boeuf Date: Wed, 28 Apr 2021 12:25:57 +0200 Subject: [PATCH] tests: Extend VFIO baremetal integration tests Relying on guest Ubuntu image 21.04, including a 5.11 kernel, this patch adds some additional tests to the VFIO baremetal integration tests. It adds a test for ACPI memory hotplug, another one for virtio-mem memory hotplug, and finally a test for hotplugging the NVIDIA card. The existing test already taking care of the reboot has been renamed. The script running "cargo test" has been modified to run only one thread at a time, so that each test run sequentially. This is mandatory since the card can't be shared across multiple VMs. Fixes #2404 Signed-off-by: Sebastien Boeuf --- scripts/run_integration_tests_vfio.sh | 2 +- tests/integration.rs | 129 ++++++++++++++++++++++---- 2 files changed, 110 insertions(+), 21 deletions(-) diff --git a/scripts/run_integration_tests_vfio.sh b/scripts/run_integration_tests_vfio.sh index 424b82dbf..1f39a7dbc 100755 --- a/scripts/run_integration_tests_vfio.sh +++ b/scripts/run_integration_tests_vfio.sh @@ -22,7 +22,7 @@ strip target/$BUILD_TARGET/release/cloud-hypervisor export RUST_BACKTRACE=1 -time cargo test $features_test "tests::vfio::$test_filter" +time cargo test $features_test "tests::vfio::$test_filter" -- --test-threads=1 RES=$? exit $RES diff --git a/tests/integration.rs b/tests/integration.rs index ae5b07803..d57ee5c44 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -696,6 +696,14 @@ mod tests { assert!(device_query_result.contains("Device 0: \"NVIDIA Tesla T4\"")); assert!(device_query_result.contains("Result = PASS")); + // Run NVIDIA DCGM Diagnostics to validate the device is functional + assert_eq!( + self.ssh_command("sudo nv-hostengine && echo ok") + .unwrap() + .trim(), + "ok" + ); + assert!(self .ssh_command("sudo dcgmi discovery -l") .unwrap() @@ -5774,16 +5782,20 @@ mod tests { mod vfio { use crate::tests::*; - #[test] - fn test_nvidia_card() { + fn test_nvidia_card_memory_hotplug(hotplug_method: &str) { let mut hirsute = UbuntuDiskConfig::new(HIRSUTE_NVIDIA_IMAGE_NAME.to_string()); let guest = Guest::new(&mut hirsute); + let api_socket = temp_api_path(&guest.tmp_dir); let mut child = GuestCommand::new(&guest) .args(&["--cpus", "boot=4"]) - .args(&["--memory", "size=4G"]) + .args(&[ + "--memory", + format!("size=4G,hotplug_size=4G,hotplug_method={}", hotplug_method).as_str(), + ]) .args(&["--kernel", guest.fw_path.as_str()]) .args(&["--device", "path=/sys/bus/pci/devices/0000:31:00.0/"]) + .args(&["--api-socket", &api_socket]) .default_disks() .default_net() .capture_output() @@ -5793,28 +5805,105 @@ mod tests { let r = std::panic::catch_unwind(|| { guest.wait_vm_boot(None).unwrap(); - // Run NVIDIA DCGM Diagnostics to validate the device is functional - assert_eq!( - guest - .ssh_command("sudo nv-hostengine && echo ok") - .unwrap() - .trim(), - "ok" - ); + assert!(guest.get_total_memory().unwrap_or_default() > 3_840_000); + guest.enable_memory_hotplug(); + + // Add RAM to the VM + let desired_ram = 6 << 30; + resize_command(&api_socket, None, Some(desired_ram), None); + thread::sleep(std::time::Duration::new(10, 0)); + assert!(guest.get_total_memory().unwrap_or_default() > 5_760_000); + + // Check the VFIO device works when RAM is increased to 6GiB + guest.check_nvidia_gpu(); + }); + + let _ = child.kill(); + let output = child.wait_with_output().unwrap(); + + handle_child_output(r, &output); + } + + #[test] + fn test_nvidia_card_memory_hotplug_acpi() { + test_nvidia_card_memory_hotplug("acpi") + } + + #[test] + fn test_nvidia_card_memory_hotplug_virtio_mem() { + test_nvidia_card_memory_hotplug("virtio-mem") + } + + #[test] + fn test_nvidia_card_pci_hotplug() { + let mut hirsute = UbuntuDiskConfig::new(HIRSUTE_NVIDIA_IMAGE_NAME.to_string()); + let guest = Guest::new(&mut hirsute); + let api_socket = temp_api_path(&guest.tmp_dir); + + let mut child = GuestCommand::new(&guest) + .args(&["--cpus", "boot=4"]) + .args(&["--memory", "size=4G"]) + .args(&["--kernel", guest.fw_path.as_str()]) + .args(&["--api-socket", &api_socket]) + .default_disks() + .default_net() + .capture_output() + .spawn() + .unwrap(); + + let r = std::panic::catch_unwind(|| { + guest.wait_vm_boot(None).unwrap(); + + // Hotplug the card to the VM + let (cmd_success, cmd_output) = remote_command_w_output( + &api_socket, + "add-device", + Some("id=vfio0,path=/sys/bus/pci/devices/0000:31:00.0/"), + ); + assert!(cmd_success); + assert!(String::from_utf8_lossy(&cmd_output) + .contains("{\"id\":\"vfio0\",\"bdf\":\"0000:00:06.0\"}")); + + thread::sleep(std::time::Duration::new(10, 0)); + + // Check the VFIO device works after hotplug + guest.check_nvidia_gpu(); + }); + + let _ = child.kill(); + let output = child.wait_with_output().unwrap(); + + handle_child_output(r, &output); + } + + #[test] + fn test_nvidia_card_reboot() { + let mut hirsute = UbuntuDiskConfig::new(HIRSUTE_NVIDIA_IMAGE_NAME.to_string()); + let guest = Guest::new(&mut hirsute); + let api_socket = temp_api_path(&guest.tmp_dir); + + let mut child = GuestCommand::new(&guest) + .args(&["--cpus", "boot=4"]) + .args(&["--memory", "size=4G"]) + .args(&["--kernel", guest.fw_path.as_str()]) + .args(&["--device", "path=/sys/bus/pci/devices/0000:31:00.0/"]) + .args(&["--api-socket", &api_socket]) + .default_disks() + .default_net() + .capture_output() + .spawn() + .unwrap(); + + let r = std::panic::catch_unwind(|| { + guest.wait_vm_boot(None).unwrap(); + + // Check the VFIO device works after boot guest.check_nvidia_gpu(); guest.reboot_linux(0, None); - // Run NVIDIA DCGM Diagnostics to validate the device is functional - assert_eq!( - guest - .ssh_command("sudo nv-hostengine && echo ok") - .unwrap() - .trim(), - "ok" - ); - + // Check the VFIO device works after reboot guest.check_nvidia_gpu(); });