diff --git a/Cargo.lock b/Cargo.lock index 37e1a66b7..2a7eecf3a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -955,6 +955,7 @@ version = "0.1.0" dependencies = [ "dirs 3.0.2", "epoll", + "lazy_static", "libc", "ssh2", "vmm-sys-util", diff --git a/test_infra/Cargo.toml b/test_infra/Cargo.toml index f78116b73..c0358f48e 100644 --- a/test_infra/Cargo.toml +++ b/test_infra/Cargo.toml @@ -7,6 +7,7 @@ edition = "2018" [dependencies] dirs = "3.0.1" epoll = "4.3.1" +lazy_static= "1.4.0" libc = "0.2.91" ssh2 = "0.9.1" vmm-sys-util = "0.9.0" diff --git a/test_infra/src/lib.rs b/test_infra/src/lib.rs index 1493866b6..361b56fa3 100644 --- a/test_infra/src/lib.rs +++ b/test_infra/src/lib.rs @@ -3,7 +3,12 @@ // SPDX-License-Identifier: Apache-2.0 // +#[macro_use] +extern crate lazy_static; + use ssh2::Session; +use std::env; +use std::ffi::OsStr; use std::fs; use std::io; use std::io::{Read, Write}; @@ -11,12 +16,24 @@ use std::net::TcpListener; use std::net::TcpStream; use std::os::unix::io::AsRawFd; use std::path::Path; -use std::process::{ExitStatus, Output}; +use std::process::{Child, Command, ExitStatus, Output, Stdio}; use std::str::FromStr; +use std::sync::Mutex; use std::thread; use vmm_sys_util::tempdir::TempDir; -pub const DEFAULT_TCP_LISTENER_MESSAGE: &str = "booted"; +#[derive(Debug)] +pub enum Error { + Parsing(std::num::ParseIntError), + SshCommand(SshCommandError), + WaitForBoot(WaitForBootError), +} + +impl From for Error { + fn from(e: SshCommandError) -> Self { + Self::SshCommand(e) + } +} pub struct GuestNetworkConfig { pub guest_ip: String, @@ -31,6 +48,7 @@ pub struct GuestNetworkConfig { pub tcp_listener_port: u16, } +pub const DEFAULT_TCP_LISTENER_MESSAGE: &str = "booted"; pub const DEFAULT_TCP_LISTENER_PORT: u16 = 8000; pub const DEFAULT_TCP_LISTENER_TIMEOUT: i32 = 80; @@ -605,3 +623,578 @@ pub fn exec_host_command_output(command: &str) -> Output { .output() .unwrap_or_else(|_| panic!("Expected '{}' to run", command)) } + +pub const PIPE_SIZE: i32 = 32 << 20; + +lazy_static! { + static ref NEXT_VM_ID: Mutex = Mutex::new(1); +} + +pub struct Guest { + pub tmp_dir: TempDir, + pub disk_config: Box, + pub network: GuestNetworkConfig, +} + +// Safe to implement as we know we have no interior mutability +impl std::panic::RefUnwindSafe for Guest {} + +impl Guest { + pub fn new_from_ip_range(mut disk_config: Box, class: &str, id: u8) -> Self { + let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); + + let network = GuestNetworkConfig { + guest_ip: format!("{}.{}.2", class, id), + l2_guest_ip1: format!("{}.{}.3", class, id), + l2_guest_ip2: format!("{}.{}.4", class, id), + l2_guest_ip3: format!("{}.{}.5", class, id), + host_ip: format!("{}.{}.1", class, id), + guest_mac: format!("12:34:56:78:90:{:02x}", id), + l2_guest_mac1: format!("de:ad:be:ef:12:{:02x}", id), + l2_guest_mac2: format!("de:ad:be:ef:34:{:02x}", id), + l2_guest_mac3: format!("de:ad:be:ef:56:{:02x}", id), + tcp_listener_port: DEFAULT_TCP_LISTENER_PORT + id as u16, + }; + + disk_config.prepare_files(&tmp_dir, &network); + + Guest { + tmp_dir, + disk_config, + network, + } + } + + pub fn new(disk_config: Box) -> Self { + let mut guard = NEXT_VM_ID.lock().unwrap(); + let id = *guard; + *guard = id + 1; + + Self::new_from_ip_range(disk_config, "192.168", id) + } + + pub fn default_net_string(&self) -> String { + format!( + "tap=,mac={},ip={},mask=255.255.255.0", + self.network.guest_mac, self.network.host_ip + ) + } + + pub fn default_net_string_w_iommu(&self) -> String { + format!( + "tap=,mac={},ip={},mask=255.255.255.0,iommu=on", + self.network.guest_mac, self.network.host_ip + ) + } + + pub fn ssh_command(&self, command: &str) -> Result { + ssh_command_ip( + command, + &self.network.guest_ip, + DEFAULT_SSH_RETRIES, + DEFAULT_SSH_TIMEOUT, + ) + } + + #[cfg(target_arch = "x86_64")] + pub fn ssh_command_l1(&self, command: &str) -> Result { + ssh_command_ip( + command, + &self.network.guest_ip, + DEFAULT_SSH_RETRIES, + DEFAULT_SSH_TIMEOUT, + ) + } + + #[cfg(target_arch = "x86_64")] + pub fn ssh_command_l2_1(&self, command: &str) -> Result { + ssh_command_ip( + command, + &self.network.l2_guest_ip1, + DEFAULT_SSH_RETRIES, + DEFAULT_SSH_TIMEOUT, + ) + } + + #[cfg(target_arch = "x86_64")] + pub fn ssh_command_l2_2(&self, command: &str) -> Result { + ssh_command_ip( + command, + &self.network.l2_guest_ip2, + DEFAULT_SSH_RETRIES, + DEFAULT_SSH_TIMEOUT, + ) + } + + #[cfg(target_arch = "x86_64")] + pub fn ssh_command_l2_3(&self, command: &str) -> Result { + ssh_command_ip( + command, + &self.network.l2_guest_ip3, + DEFAULT_SSH_RETRIES, + DEFAULT_SSH_TIMEOUT, + ) + } + + pub fn api_create_body( + &self, + cpu_count: u8, + fw_path: &str, + _kernel_path: &str, + _kernel_cmd: &str, + ) -> String { + #[cfg(all(target_arch = "x86_64", not(feature = "mshv")))] + format! {"{{\"cpus\":{{\"boot_vcpus\":{},\"max_vcpus\":{}}},\"kernel\":{{\"path\":\"{}\"}},\"cmdline\":{{\"args\": \"\"}},\"net\":[{{\"ip\":\"{}\", \"mask\":\"255.255.255.0\", \"mac\":\"{}\"}}], \"disks\":[{{\"path\":\"{}\"}}, {{\"path\":\"{}\"}}]}}", + cpu_count, + cpu_count, + fw_path, + self.network.host_ip, + self.network.guest_mac, + self.disk_config.disk(DiskType::OperatingSystem).unwrap().as_str(), + self.disk_config.disk(DiskType::CloudInit).unwrap().as_str(), + } + + #[cfg(any(target_arch = "aarch64", feature = "mshv"))] + format! {"{{\"cpus\":{{\"boot_vcpus\":{},\"max_vcpus\":{}}},\"kernel\":{{\"path\":\"{}\"}},\"cmdline\":{{\"args\": \"{}\"}},\"net\":[{{\"ip\":\"{}\", \"mask\":\"255.255.255.0\", \"mac\":\"{}\"}}], \"disks\":[{{\"path\":\"{}\"}}, {{\"path\":\"{}\"}}]}}", + cpu_count, + cpu_count, + _kernel_path, + _kernel_cmd, + self.network.host_ip, + self.network.guest_mac, + self.disk_config.disk(DiskType::OperatingSystem).unwrap().as_str(), + self.disk_config.disk(DiskType::CloudInit).unwrap().as_str(), + } + } + + pub fn get_cpu_count(&self) -> Result { + self.ssh_command("grep -c processor /proc/cpuinfo")? + .trim() + .parse() + .map_err(Error::Parsing) + } + + #[cfg(target_arch = "x86_64")] + pub fn get_initial_apicid(&self) -> Result { + self.ssh_command("grep \"initial apicid\" /proc/cpuinfo | grep -o \"[0-9]*\"")? + .trim() + .parse() + .map_err(Error::Parsing) + } + + pub fn get_total_memory(&self) -> Result { + self.ssh_command("grep MemTotal /proc/meminfo | grep -o \"[0-9]*\"")? + .trim() + .parse() + .map_err(Error::Parsing) + } + + #[cfg(target_arch = "x86_64")] + pub fn get_total_memory_l2(&self) -> Result { + self.ssh_command_l2_1("grep MemTotal /proc/meminfo | grep -o \"[0-9]*\"")? + .trim() + .parse() + .map_err(Error::Parsing) + } + + pub fn get_numa_node_memory(&self, node_id: usize) -> Result { + self.ssh_command( + format!( + "grep MemTotal /sys/devices/system/node/node{}/meminfo \ + | cut -d \":\" -f 2 | grep -o \"[0-9]*\"", + node_id + ) + .as_str(), + )? + .trim() + .parse() + .map_err(Error::Parsing) + } + + pub fn wait_vm_boot(&self, custom_timeout: Option) -> Result<(), Error> { + self.network + .wait_vm_boot(custom_timeout) + .map_err(Error::WaitForBoot) + } + + pub fn check_numa_node_cpus(&self, node_id: usize, cpus: Vec) -> Result<(), Error> { + for cpu in cpus.iter() { + let cmd = format!( + "[ -d \"/sys/devices/system/node/node{}/cpu{}\" ]", + node_id, cpu + ); + self.ssh_command(cmd.as_str())?; + } + + Ok(()) + } + + pub fn check_numa_node_distances( + &self, + node_id: usize, + distances: &str, + ) -> Result { + let cmd = format!("cat /sys/devices/system/node/node{}/distance", node_id); + if self.ssh_command(cmd.as_str())?.trim() == distances { + Ok(true) + } else { + Ok(false) + } + } + + pub fn check_numa_common( + &self, + mem_ref: Option<&[u32]>, + node_ref: Option<&[Vec]>, + distance_ref: Option<&[&str]>, + ) { + if let Some(mem_ref) = mem_ref { + // Check each NUMA node has been assigned the right amount of + // memory. + for (i, &m) in mem_ref.iter().enumerate() { + assert!(self.get_numa_node_memory(i).unwrap_or_default() > m); + } + } + + if let Some(node_ref) = node_ref { + // Check each NUMA node has been assigned the right CPUs set. + for (i, n) in node_ref.iter().enumerate() { + self.check_numa_node_cpus(i, n.clone()).unwrap(); + } + } + + if let Some(distance_ref) = distance_ref { + // Check each NUMA node has been assigned the right distances. + for (i, &d) in distance_ref.iter().enumerate() { + assert!(self.check_numa_node_distances(i, d).unwrap()); + } + } + } + + #[cfg(target_arch = "x86_64")] + pub fn check_sgx_support(&self) -> Result<(), Error> { + self.ssh_command( + "cpuid -l 0x7 -s 0 | tr -s [:space:] | grep -q 'SGX: \ + Software Guard Extensions supported = true'", + )?; + self.ssh_command( + "cpuid -l 0x7 -s 0 | tr -s [:space:] | grep -q 'SGX_LC: \ + SGX launch config supported = true'", + )?; + self.ssh_command( + "cpuid -l 0x12 -s 0 | tr -s [:space:] | grep -q 'SGX1 \ + supported = true'", + )?; + + Ok(()) + } + + pub fn get_entropy(&self) -> Result { + self.ssh_command("cat /proc/sys/kernel/random/entropy_avail")? + .trim() + .parse() + .map_err(Error::Parsing) + } + + pub fn get_pci_bridge_class(&self) -> Result { + Ok(self + .ssh_command("cat /sys/bus/pci/devices/0000:00:00.0/class")? + .trim() + .to_string()) + } + + pub fn get_pci_device_ids(&self) -> Result { + Ok(self + .ssh_command("cat /sys/bus/pci/devices/*/device")? + .trim() + .to_string()) + } + + pub fn get_pci_vendor_ids(&self) -> Result { + Ok(self + .ssh_command("cat /sys/bus/pci/devices/*/vendor")? + .trim() + .to_string()) + } + + pub fn does_device_vendor_pair_match( + &self, + device_id: &str, + vendor_id: &str, + ) -> Result { + // We are checking if console device's device id and vendor id pair matches + let devices = self.get_pci_device_ids()?; + let devices: Vec<&str> = devices.split('\n').collect(); + let vendors = self.get_pci_vendor_ids()?; + let vendors: Vec<&str> = vendors.split('\n').collect(); + + for (index, d_id) in devices.iter().enumerate() { + if *d_id == device_id { + if let Some(v_id) = vendors.get(index) { + if *v_id == vendor_id { + return Ok(true); + } + } + } + } + + Ok(false) + } + + pub fn valid_virtio_fs_cache_size( + &self, + dax: bool, + cache_size: Option, + ) -> Result { + // SHM region is called different things depending on kernel + let shm_region = self + .ssh_command("sudo grep 'virtio[0-9]\\|virtio-pci-shm' /proc/iomem || true")? + .trim() + .to_string(); + + if shm_region.is_empty() { + return Ok(!dax); + } + + // From this point, the region is not empty, hence it is an error + // if DAX is off. + if !dax { + return Ok(false); + } + + let cache = if let Some(cache) = cache_size { + cache + } else { + // 8Gib by default + 0x0002_0000_0000 + }; + + let args: Vec<&str> = shm_region.split(':').collect(); + if args.is_empty() { + return Ok(false); + } + + let args: Vec<&str> = args[0].trim().split('-').collect(); + if args.len() != 2 { + return Ok(false); + } + + let start_addr = u64::from_str_radix(args[0], 16).map_err(Error::Parsing)?; + let end_addr = u64::from_str_radix(args[1], 16).map_err(Error::Parsing)?; + + Ok(cache == (end_addr - start_addr + 1)) + } + + pub fn check_vsock(&self, socket: &str) { + // Listen from guest on vsock CID=3 PORT=16 + // SOCKET-LISTEN::: + let guest_ip = self.network.guest_ip.clone(); + let listen_socat = thread::spawn(move || { + ssh_command_ip("sudo socat - SOCKET-LISTEN:40:0:x00x00x10x00x00x00x03x00x00x00x00x00x00x00 > vsock_log", &guest_ip, DEFAULT_SSH_RETRIES, DEFAULT_SSH_TIMEOUT).unwrap(); + }); + + // Make sure socat is listening, which might take a few second on slow systems + thread::sleep(std::time::Duration::new(10, 0)); + + // Write something to vsock from the host + assert!(exec_host_command_status(&format!( + "echo -e \"CONNECT 16\\nHelloWorld!\" | socat - UNIX-CONNECT:{}", + socket + )) + .success()); + + // Wait for the thread to terminate. + listen_socat.join().unwrap(); + + assert_eq!( + self.ssh_command("cat vsock_log").unwrap().trim(), + "HelloWorld!" + ); + } + + #[cfg(target_arch = "x86_64")] + pub fn check_nvidia_gpu(&self) { + // Run CUDA sample to validate it can find the device + let device_query_result = self + .ssh_command("sudo /root/NVIDIA_CUDA-11.3_Samples/bin/x86_64/linux/release/deviceQuery") + .unwrap(); + assert!(device_query_result.contains("Detected 1 CUDA Capable device")); + assert!(device_query_result.contains("Device 0: \"NVIDIA Tesla T4\"")); + assert!(device_query_result.contains("Result = PASS")); + + // Run NVIDIA DCGM Diagnostics to validate the device is functional + self.ssh_command("sudo nv-hostengine").unwrap(); + + assert!(self + .ssh_command("sudo dcgmi discovery -l") + .unwrap() + .contains("Name: NVIDIA Tesla T4")); + assert_eq!( + self.ssh_command("sudo dcgmi diag -r 'diagnostic' | grep Pass | wc -l") + .unwrap() + .trim(), + "10" + ); + } + + pub fn reboot_linux(&self, current_reboot_count: u32, custom_timeout: Option) { + let list_boots_cmd = "sudo journalctl --list-boots | wc -l"; + let boot_count = self + .ssh_command(list_boots_cmd) + .unwrap() + .trim() + .parse::() + .unwrap_or_default(); + + assert_eq!(boot_count, current_reboot_count + 1); + self.ssh_command("sudo reboot").unwrap(); + + self.wait_vm_boot(custom_timeout).unwrap(); + let boot_count = self + .ssh_command(list_boots_cmd) + .unwrap() + .trim() + .parse::() + .unwrap_or_default(); + assert_eq!(boot_count, current_reboot_count + 2); + } + + pub fn enable_memory_hotplug(&self) { + self.ssh_command("echo online | sudo tee /sys/devices/system/memory/auto_online_blocks") + .unwrap(); + } + + pub fn check_devices_common(&self, socket: Option<&String>, console_text: Option<&String>) { + // Check block devices are readable + self.ssh_command("sudo dd if=/dev/vda of=/dev/null bs=1M iflag=direct count=1024") + .unwrap(); + self.ssh_command("sudo dd if=/dev/vdb of=/dev/null bs=1M iflag=direct count=8") + .unwrap(); + // Check if the rng device is readable + self.ssh_command("sudo head -c 1000 /dev/hwrng > /dev/null") + .unwrap(); + // Check vsock + if let Some(socket) = socket { + self.check_vsock(socket.as_str()); + } + // Check if the console is usable + if let Some(console_text) = console_text { + let console_cmd = format!("echo {} | sudo tee /dev/hvc0", console_text); + self.ssh_command(&console_cmd).unwrap(); + } + // The net device is 'automatically' exercised through the above 'ssh' commands + } +} + +pub struct GuestCommand<'a> { + command: Command, + guest: &'a Guest, + capture_output: bool, +} + +impl<'a> GuestCommand<'a> { + pub fn new(guest: &'a Guest) -> Self { + Self::new_with_binary_name(guest, "cloud-hypervisor") + } + + pub fn new_with_binary_name(guest: &'a Guest, binary_name: &str) -> Self { + Self { + command: Command::new(clh_command(binary_name)), + guest, + capture_output: false, + } + } + + pub fn capture_output(&mut self) -> &mut Self { + self.capture_output = true; + self + } + + pub fn spawn(&mut self) -> io::Result { + println!( + "\n\n==== Start cloud-hypervisor command-line ====\n\n\ + {:?}\n\ + \n==== End cloud-hypervisor command-line ====\n\n", + self.command + ); + + if self.capture_output { + let child = self + .command + .arg("-v") + .stderr(Stdio::piped()) + .stdout(Stdio::piped()) + .spawn() + .unwrap(); + + let fd = child.stdout.as_ref().unwrap().as_raw_fd(); + let pipesize = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; + let fd = child.stderr.as_ref().unwrap().as_raw_fd(); + let pipesize1 = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; + + if pipesize >= PIPE_SIZE && pipesize1 >= PIPE_SIZE { + Ok(child) + } else { + Err(std::io::Error::new( + std::io::ErrorKind::Other, + "resizing pipe w/ 'fnctl' failed!", + )) + } + } else { + self.command.arg("-v").spawn() + } + } + + pub fn args(&mut self, args: I) -> &mut Self + where + I: IntoIterator, + S: AsRef, + { + self.command.args(args); + self + } + + pub fn default_disks(&mut self) -> &mut Self { + if self.guest.disk_config.disk(DiskType::CloudInit).is_some() { + self.args(&[ + "--disk", + format!( + "path={}", + self.guest + .disk_config + .disk(DiskType::OperatingSystem) + .unwrap() + ) + .as_str(), + format!( + "path={}", + self.guest.disk_config.disk(DiskType::CloudInit).unwrap() + ) + .as_str(), + ]) + } else { + self.args(&[ + "--disk", + format!( + "path={}", + self.guest + .disk_config + .disk(DiskType::OperatingSystem) + .unwrap() + ) + .as_str(), + ]) + } + } + + pub fn default_net(&mut self) -> &mut Self { + self.args(&["--net", self.guest.default_net_string().as_str()]) + } +} + +pub fn clh_command(cmd: &str) -> String { + env::var("BUILD_TARGET").map_or( + format!("target/x86_64-unknown-linux-gnu/release/{}", cmd), + |target| format!("target/{}/release/{}", target, cmd), + ) +} diff --git a/tests/integration.rs b/tests/integration.rs index da3188624..59da787d4 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -15,8 +15,6 @@ extern crate test_infra; use net_util::MacAddr; use std::collections::HashMap; -use std::env; -use std::ffi::OsStr; use std::fs; use std::io; use std::io::BufRead; @@ -27,27 +25,15 @@ use std::os::unix::io::AsRawFd; use std::path::PathBuf; use std::process::{Child, Command, Stdio}; use std::string::String; +use std::sync::mpsc; use std::sync::mpsc::Receiver; -use std::sync::{mpsc, Mutex}; +#[cfg(target_arch = "x86_64")] +use std::sync::Mutex; use std::thread; use test_infra::*; use vmm_sys_util::{tempdir::TempDir, tempfile::TempFile}; -#[cfg_attr(target_arch = "aarch64", allow(unused_imports))] use wait_timeout::ChildExt; -lazy_static! { - static ref NEXT_VM_ID: Mutex = Mutex::new(1); -} - -struct Guest { - tmp_dir: TempDir, - disk_config: Box, - network: GuestNetworkConfig, -} - -// Safe to implement as we know we have no interior mutability -impl std::panic::RefUnwindSafe for Guest {} - #[cfg(target_arch = "x86_64")] mod x86_64 { pub const BIONIC_IMAGE_NAME: &str = "bionic-server-cloudimg-amd64.raw"; @@ -84,17 +70,8 @@ use aarch64::*; const DIRECT_KERNEL_BOOT_CMDLINE: &str = "root=/dev/vda1 console=hvc0 rw systemd.journald.forward_to_console=1"; -const PIPE_SIZE: i32 = 32 << 20; - const CONSOLE_TEST_STRING: &str = "Started OpenBSD Secure Shell server"; -fn clh_command(cmd: &str) -> String { - env::var("BUILD_TARGET").map_or( - format!("target/x86_64-unknown-linux-gnu/release/{}", cmd), - |target| format!("target/{}/release/{}", target, cmd), - ) -} - fn prepare_virtiofsd( tmp_dir: &TempDir, shared_dir: &str, @@ -454,19 +431,6 @@ fn setup_ovs_dpdk_guests(guest1: &Guest, guest2: &Guest, api_socket: &str) -> (C (child1, child2) } -#[derive(Debug)] -enum Error { - Parsing(std::num::ParseIntError), - SshCommand(SshCommandError), - WaitForBoot(WaitForBootError), -} - -impl From for Error { - fn from(e: SshCommandError) -> Self { - Self::SshCommand(e) - } -} - enum FwType { Ovmf, RustHypervisorFirmware, @@ -490,555 +454,6 @@ fn fw_path(fw_type: FwType) -> String { fw_path.to_str().unwrap().to_string() } -impl Guest { - fn new_from_ip_range(mut disk_config: Box, class: &str, id: u8) -> Self { - let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap(); - - let network = GuestNetworkConfig { - guest_ip: format!("{}.{}.2", class, id), - l2_guest_ip1: format!("{}.{}.3", class, id), - l2_guest_ip2: format!("{}.{}.4", class, id), - l2_guest_ip3: format!("{}.{}.5", class, id), - host_ip: format!("{}.{}.1", class, id), - guest_mac: format!("12:34:56:78:90:{:02x}", id), - l2_guest_mac1: format!("de:ad:be:ef:12:{:02x}", id), - l2_guest_mac2: format!("de:ad:be:ef:34:{:02x}", id), - l2_guest_mac3: format!("de:ad:be:ef:56:{:02x}", id), - tcp_listener_port: DEFAULT_TCP_LISTENER_PORT + id as u16, - }; - - disk_config.prepare_files(&tmp_dir, &network); - - Guest { - tmp_dir, - disk_config, - network, - } - } - - fn new(disk_config: Box) -> Self { - let mut guard = NEXT_VM_ID.lock().unwrap(); - let id = *guard; - *guard = id + 1; - - Self::new_from_ip_range(disk_config, "192.168", id) - } - - fn default_net_string(&self) -> String { - format!( - "tap=,mac={},ip={},mask=255.255.255.0", - self.network.guest_mac, self.network.host_ip - ) - } - - fn default_net_string_w_iommu(&self) -> String { - format!( - "tap=,mac={},ip={},mask=255.255.255.0,iommu=on", - self.network.guest_mac, self.network.host_ip - ) - } - - fn ssh_command(&self, command: &str) -> Result { - ssh_command_ip( - command, - &self.network.guest_ip, - DEFAULT_SSH_RETRIES, - DEFAULT_SSH_TIMEOUT, - ) - } - - #[cfg(target_arch = "x86_64")] - fn ssh_command_l1(&self, command: &str) -> Result { - ssh_command_ip( - command, - &self.network.guest_ip, - DEFAULT_SSH_RETRIES, - DEFAULT_SSH_TIMEOUT, - ) - } - - #[cfg(target_arch = "x86_64")] - fn ssh_command_l2_1(&self, command: &str) -> Result { - ssh_command_ip( - command, - &self.network.l2_guest_ip1, - DEFAULT_SSH_RETRIES, - DEFAULT_SSH_TIMEOUT, - ) - } - - #[cfg(target_arch = "x86_64")] - fn ssh_command_l2_2(&self, command: &str) -> Result { - ssh_command_ip( - command, - &self.network.l2_guest_ip2, - DEFAULT_SSH_RETRIES, - DEFAULT_SSH_TIMEOUT, - ) - } - - #[cfg(target_arch = "x86_64")] - fn ssh_command_l2_3(&self, command: &str) -> Result { - ssh_command_ip( - command, - &self.network.l2_guest_ip3, - DEFAULT_SSH_RETRIES, - DEFAULT_SSH_TIMEOUT, - ) - } - - fn api_create_body( - &self, - cpu_count: u8, - fw_path: &str, - _kernel_path: &str, - _kernel_cmd: &str, - ) -> String { - #[cfg(all(target_arch = "x86_64", not(feature = "mshv")))] - format! {"{{\"cpus\":{{\"boot_vcpus\":{},\"max_vcpus\":{}}},\"kernel\":{{\"path\":\"{}\"}},\"cmdline\":{{\"args\": \"\"}},\"net\":[{{\"ip\":\"{}\", \"mask\":\"255.255.255.0\", \"mac\":\"{}\"}}], \"disks\":[{{\"path\":\"{}\"}}, {{\"path\":\"{}\"}}]}}", - cpu_count, - cpu_count, - fw_path, - self.network.host_ip, - self.network.guest_mac, - self.disk_config.disk(DiskType::OperatingSystem).unwrap().as_str(), - self.disk_config.disk(DiskType::CloudInit).unwrap().as_str(), - } - - #[cfg(any(target_arch = "aarch64", feature = "mshv"))] - format! {"{{\"cpus\":{{\"boot_vcpus\":{},\"max_vcpus\":{}}},\"kernel\":{{\"path\":\"{}\"}},\"cmdline\":{{\"args\": \"{}\"}},\"net\":[{{\"ip\":\"{}\", \"mask\":\"255.255.255.0\", \"mac\":\"{}\"}}], \"disks\":[{{\"path\":\"{}\"}}, {{\"path\":\"{}\"}}]}}", - cpu_count, - cpu_count, - _kernel_path, - _kernel_cmd, - self.network.host_ip, - self.network.guest_mac, - self.disk_config.disk(DiskType::OperatingSystem).unwrap().as_str(), - self.disk_config.disk(DiskType::CloudInit).unwrap().as_str(), - } - } - - fn get_cpu_count(&self) -> Result { - self.ssh_command("grep -c processor /proc/cpuinfo")? - .trim() - .parse() - .map_err(Error::Parsing) - } - - #[cfg(target_arch = "x86_64")] - fn get_initial_apicid(&self) -> Result { - self.ssh_command("grep \"initial apicid\" /proc/cpuinfo | grep -o \"[0-9]*\"")? - .trim() - .parse() - .map_err(Error::Parsing) - } - - fn get_total_memory(&self) -> Result { - self.ssh_command("grep MemTotal /proc/meminfo | grep -o \"[0-9]*\"")? - .trim() - .parse() - .map_err(Error::Parsing) - } - - #[cfg(target_arch = "x86_64")] - fn get_total_memory_l2(&self) -> Result { - self.ssh_command_l2_1("grep MemTotal /proc/meminfo | grep -o \"[0-9]*\"")? - .trim() - .parse() - .map_err(Error::Parsing) - } - - fn get_numa_node_memory(&self, node_id: usize) -> Result { - self.ssh_command( - format!( - "grep MemTotal /sys/devices/system/node/node{}/meminfo \ - | cut -d \":\" -f 2 | grep -o \"[0-9]*\"", - node_id - ) - .as_str(), - )? - .trim() - .parse() - .map_err(Error::Parsing) - } - - fn wait_vm_boot(&self, custom_timeout: Option) -> Result<(), Error> { - self.network - .wait_vm_boot(custom_timeout) - .map_err(Error::WaitForBoot) - } - - fn check_numa_node_cpus(&self, node_id: usize, cpus: Vec) -> Result<(), Error> { - for cpu in cpus.iter() { - let cmd = format!( - "[ -d \"/sys/devices/system/node/node{}/cpu{}\" ]", - node_id, cpu - ); - self.ssh_command(cmd.as_str())?; - } - - Ok(()) - } - - fn check_numa_node_distances(&self, node_id: usize, distances: &str) -> Result { - let cmd = format!("cat /sys/devices/system/node/node{}/distance", node_id); - if self.ssh_command(cmd.as_str())?.trim() == distances { - Ok(true) - } else { - Ok(false) - } - } - - fn check_numa_common( - &self, - mem_ref: Option<&[u32]>, - node_ref: Option<&[Vec]>, - distance_ref: Option<&[&str]>, - ) { - if let Some(mem_ref) = mem_ref { - // Check each NUMA node has been assigned the right amount of - // memory. - for (i, &m) in mem_ref.iter().enumerate() { - assert!(self.get_numa_node_memory(i).unwrap_or_default() > m); - } - } - - if let Some(node_ref) = node_ref { - // Check each NUMA node has been assigned the right CPUs set. - for (i, n) in node_ref.iter().enumerate() { - self.check_numa_node_cpus(i, n.clone()).unwrap(); - } - } - - if let Some(distance_ref) = distance_ref { - // Check each NUMA node has been assigned the right distances. - for (i, &d) in distance_ref.iter().enumerate() { - assert!(self.check_numa_node_distances(i, d).unwrap()); - } - } - } - - #[cfg(target_arch = "x86_64")] - fn check_sgx_support(&self) -> Result<(), Error> { - self.ssh_command( - "cpuid -l 0x7 -s 0 | tr -s [:space:] | grep -q 'SGX: \ - Software Guard Extensions supported = true'", - )?; - self.ssh_command( - "cpuid -l 0x7 -s 0 | tr -s [:space:] | grep -q 'SGX_LC: \ - SGX launch config supported = true'", - )?; - self.ssh_command( - "cpuid -l 0x12 -s 0 | tr -s [:space:] | grep -q 'SGX1 \ - supported = true'", - )?; - - Ok(()) - } - - fn get_entropy(&self) -> Result { - self.ssh_command("cat /proc/sys/kernel/random/entropy_avail")? - .trim() - .parse() - .map_err(Error::Parsing) - } - - fn get_pci_bridge_class(&self) -> Result { - Ok(self - .ssh_command("cat /sys/bus/pci/devices/0000:00:00.0/class")? - .trim() - .to_string()) - } - - fn get_pci_device_ids(&self) -> Result { - Ok(self - .ssh_command("cat /sys/bus/pci/devices/*/device")? - .trim() - .to_string()) - } - - fn get_pci_vendor_ids(&self) -> Result { - Ok(self - .ssh_command("cat /sys/bus/pci/devices/*/vendor")? - .trim() - .to_string()) - } - - fn does_device_vendor_pair_match( - &self, - device_id: &str, - vendor_id: &str, - ) -> Result { - // We are checking if console device's device id and vendor id pair matches - let devices = self.get_pci_device_ids()?; - let devices: Vec<&str> = devices.split('\n').collect(); - let vendors = self.get_pci_vendor_ids()?; - let vendors: Vec<&str> = vendors.split('\n').collect(); - - for (index, d_id) in devices.iter().enumerate() { - if *d_id == device_id { - if let Some(v_id) = vendors.get(index) { - if *v_id == vendor_id { - return Ok(true); - } - } - } - } - - Ok(false) - } - - fn valid_virtio_fs_cache_size( - &self, - dax: bool, - cache_size: Option, - ) -> Result { - // SHM region is called different things depending on kernel - let shm_region = self - .ssh_command("sudo grep 'virtio[0-9]\\|virtio-pci-shm' /proc/iomem || true")? - .trim() - .to_string(); - - if shm_region.is_empty() { - return Ok(!dax); - } - - // From this point, the region is not empty, hence it is an error - // if DAX is off. - if !dax { - return Ok(false); - } - - let cache = if let Some(cache) = cache_size { - cache - } else { - // 8Gib by default - 0x0002_0000_0000 - }; - - let args: Vec<&str> = shm_region.split(':').collect(); - if args.is_empty() { - return Ok(false); - } - - let args: Vec<&str> = args[0].trim().split('-').collect(); - if args.len() != 2 { - return Ok(false); - } - - let start_addr = u64::from_str_radix(args[0], 16).map_err(Error::Parsing)?; - let end_addr = u64::from_str_radix(args[1], 16).map_err(Error::Parsing)?; - - Ok(cache == (end_addr - start_addr + 1)) - } - - fn check_vsock(&self, socket: &str) { - // Listen from guest on vsock CID=3 PORT=16 - // SOCKET-LISTEN::: - let guest_ip = self.network.guest_ip.clone(); - let listen_socat = thread::spawn(move || { - ssh_command_ip("sudo socat - SOCKET-LISTEN:40:0:x00x00x10x00x00x00x03x00x00x00x00x00x00x00 > vsock_log", &guest_ip, DEFAULT_SSH_RETRIES, DEFAULT_SSH_TIMEOUT).unwrap(); - }); - - // Make sure socat is listening, which might take a few second on slow systems - thread::sleep(std::time::Duration::new(10, 0)); - - // Write something to vsock from the host - assert!(exec_host_command_status(&format!( - "echo -e \"CONNECT 16\\nHelloWorld!\" | socat - UNIX-CONNECT:{}", - socket - )) - .success()); - - // Wait for the thread to terminate. - listen_socat.join().unwrap(); - - assert_eq!( - self.ssh_command("cat vsock_log").unwrap().trim(), - "HelloWorld!" - ); - } - - #[cfg(target_arch = "x86_64")] - fn check_nvidia_gpu(&self) { - // Run CUDA sample to validate it can find the device - let device_query_result = self - .ssh_command("sudo /root/NVIDIA_CUDA-11.3_Samples/bin/x86_64/linux/release/deviceQuery") - .unwrap(); - assert!(device_query_result.contains("Detected 1 CUDA Capable device")); - assert!(device_query_result.contains("Device 0: \"NVIDIA Tesla T4\"")); - assert!(device_query_result.contains("Result = PASS")); - - // Run NVIDIA DCGM Diagnostics to validate the device is functional - self.ssh_command("sudo nv-hostengine").unwrap(); - - assert!(self - .ssh_command("sudo dcgmi discovery -l") - .unwrap() - .contains("Name: NVIDIA Tesla T4")); - assert_eq!( - self.ssh_command("sudo dcgmi diag -r 'diagnostic' | grep Pass | wc -l") - .unwrap() - .trim(), - "10" - ); - } - - fn reboot_linux(&self, current_reboot_count: u32, custom_timeout: Option) { - let list_boots_cmd = "sudo journalctl --list-boots | wc -l"; - let boot_count = self - .ssh_command(list_boots_cmd) - .unwrap() - .trim() - .parse::() - .unwrap_or_default(); - - assert_eq!(boot_count, current_reboot_count + 1); - self.ssh_command("sudo reboot").unwrap(); - - self.wait_vm_boot(custom_timeout).unwrap(); - let boot_count = self - .ssh_command(list_boots_cmd) - .unwrap() - .trim() - .parse::() - .unwrap_or_default(); - assert_eq!(boot_count, current_reboot_count + 2); - } - - fn enable_memory_hotplug(&self) { - self.ssh_command("echo online | sudo tee /sys/devices/system/memory/auto_online_blocks") - .unwrap(); - } - - fn check_devices_common(&self, socket: Option<&String>, console_text: Option<&String>) { - // Check block devices are readable - self.ssh_command("sudo dd if=/dev/vda of=/dev/null bs=1M iflag=direct count=1024") - .unwrap(); - self.ssh_command("sudo dd if=/dev/vdb of=/dev/null bs=1M iflag=direct count=8") - .unwrap(); - // Check if the rng device is readable - self.ssh_command("sudo head -c 1000 /dev/hwrng > /dev/null") - .unwrap(); - // Check vsock - if let Some(socket) = socket { - self.check_vsock(socket.as_str()); - } - // Check if the console is usable - if let Some(console_text) = console_text { - let console_cmd = format!("echo {} | sudo tee /dev/hvc0", console_text); - self.ssh_command(&console_cmd).unwrap(); - } - // The net device is 'automatically' exercised through the above 'ssh' commands - } -} - -struct GuestCommand<'a> { - command: Command, - guest: &'a Guest, - capture_output: bool, -} - -impl<'a> GuestCommand<'a> { - fn new(guest: &'a Guest) -> Self { - Self::new_with_binary_name(guest, "cloud-hypervisor") - } - - fn new_with_binary_name(guest: &'a Guest, binary_name: &str) -> Self { - Self { - command: Command::new(clh_command(binary_name)), - guest, - capture_output: false, - } - } - - fn capture_output(&mut self) -> &mut Self { - self.capture_output = true; - self - } - - fn spawn(&mut self) -> io::Result { - println!( - "\n\n==== Start cloud-hypervisor command-line ====\n\n\ - {:?}\n\ - \n==== End cloud-hypervisor command-line ====\n\n", - self.command - ); - - if self.capture_output { - let child = self - .command - .arg("-v") - .stderr(Stdio::piped()) - .stdout(Stdio::piped()) - .spawn() - .unwrap(); - - let fd = child.stdout.as_ref().unwrap().as_raw_fd(); - let pipesize = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; - let fd = child.stderr.as_ref().unwrap().as_raw_fd(); - let pipesize1 = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) }; - - if pipesize >= PIPE_SIZE && pipesize1 >= PIPE_SIZE { - Ok(child) - } else { - Err(std::io::Error::new( - std::io::ErrorKind::Other, - "resizing pipe w/ 'fnctl' failed!", - )) - } - } else { - self.command.arg("-v").spawn() - } - } - - fn args(&mut self, args: I) -> &mut Self - where - I: IntoIterator, - S: AsRef, - { - self.command.args(args); - self - } - - fn default_disks(&mut self) -> &mut Self { - if self.guest.disk_config.disk(DiskType::CloudInit).is_some() { - self.args(&[ - "--disk", - format!( - "path={}", - self.guest - .disk_config - .disk(DiskType::OperatingSystem) - .unwrap() - ) - .as_str(), - format!( - "path={}", - self.guest.disk_config.disk(DiskType::CloudInit).unwrap() - ) - .as_str(), - ]) - } else { - self.args(&[ - "--disk", - format!( - "path={}", - self.guest - .disk_config - .disk(DiskType::OperatingSystem) - .unwrap() - ) - .as_str(), - ]) - } - } - - fn default_net(&mut self) -> &mut Self { - self.args(&["--net", self.guest.default_net_string().as_str()]) - } -} - fn test_cpu_topology(threads_per_core: u8, cores_per_package: u8, packages: u8, use_fw: bool) { let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string()); let guest = Guest::new(Box::new(focal));