test_infra: Move struct Guest and struct GuestCommand from tests

In this way, these structs can be reused for performance tests.

Signed-off-by: Bo Chen <chen.bo@intel.com>

Cargo.lock

@@ -955,6 +955,7 @@ version = "0.1.0"
 dependencies = [
  "dirs 3.0.2",
  "epoll",
+ "lazy_static",
  "libc",
  "ssh2",
  "vmm-sys-util",

test_infra/Cargo.toml

@@ -7,6 +7,7 @@ edition = "2018"
 [dependencies]
 dirs = "3.0.1"
 epoll = "4.3.1"
+lazy_static= "1.4.0"
 libc = "0.2.91"
 ssh2 = "0.9.1"
 vmm-sys-util = "0.9.0"

test_infra/src/lib.rs

@@ -3,7 +3,12 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
+#[macro_use]
+extern crate lazy_static;
+
 use ssh2::Session;
+use std::env;
+use std::ffi::OsStr;
 use std::fs;
 use std::io;
 use std::io::{Read, Write};
@@ -11,12 +16,24 @@ use std::net::TcpListener;
 use std::net::TcpStream;
 use std::os::unix::io::AsRawFd;
 use std::path::Path;
-use std::process::{ExitStatus, Output};
+use std::process::{Child, Command, ExitStatus, Output, Stdio};
 use std::str::FromStr;
+use std::sync::Mutex;
 use std::thread;
 use vmm_sys_util::tempdir::TempDir;
 
-pub const DEFAULT_TCP_LISTENER_MESSAGE: &str = "booted";
+#[derive(Debug)]
+pub enum Error {
+    Parsing(std::num::ParseIntError),
+    SshCommand(SshCommandError),
+    WaitForBoot(WaitForBootError),
+}
+
+impl From<SshCommandError> for Error {
+    fn from(e: SshCommandError) -> Self {
+        Self::SshCommand(e)
+    }
+}
 
 pub struct GuestNetworkConfig {
     pub guest_ip: String,
@@ -31,6 +48,7 @@ pub struct GuestNetworkConfig {
     pub tcp_listener_port: u16,
 }
 
+pub const DEFAULT_TCP_LISTENER_MESSAGE: &str = "booted";
 pub const DEFAULT_TCP_LISTENER_PORT: u16 = 8000;
 pub const DEFAULT_TCP_LISTENER_TIMEOUT: i32 = 80;
 
@@ -605,3 +623,578 @@ pub fn exec_host_command_output(command: &str) -> Output {
         .output()
         .unwrap_or_else(|_| panic!("Expected '{}' to run", command))
 }
+
+pub const PIPE_SIZE: i32 = 32 << 20;
+
+lazy_static! {
+    static ref NEXT_VM_ID: Mutex<u8> = Mutex::new(1);
+}
+
+pub struct Guest {
+    pub tmp_dir: TempDir,
+    pub disk_config: Box<dyn DiskConfig>,
+    pub network: GuestNetworkConfig,
+}
+
+// Safe to implement as we know we have no interior mutability
+impl std::panic::RefUnwindSafe for Guest {}
+
+impl Guest {
+    pub fn new_from_ip_range(mut disk_config: Box<dyn DiskConfig>, class: &str, id: u8) -> Self {
+        let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap();
+
+        let network = GuestNetworkConfig {
+            guest_ip: format!("{}.{}.2", class, id),
+            l2_guest_ip1: format!("{}.{}.3", class, id),
+            l2_guest_ip2: format!("{}.{}.4", class, id),
+            l2_guest_ip3: format!("{}.{}.5", class, id),
+            host_ip: format!("{}.{}.1", class, id),
+            guest_mac: format!("12:34:56:78:90:{:02x}", id),
+            l2_guest_mac1: format!("de:ad:be:ef:12:{:02x}", id),
+            l2_guest_mac2: format!("de:ad:be:ef:34:{:02x}", id),
+            l2_guest_mac3: format!("de:ad:be:ef:56:{:02x}", id),
+            tcp_listener_port: DEFAULT_TCP_LISTENER_PORT + id as u16,
+        };
+
+        disk_config.prepare_files(&tmp_dir, &network);
+
+        Guest {
+            tmp_dir,
+            disk_config,
+            network,
+        }
+    }
+
+    pub fn new(disk_config: Box<dyn DiskConfig>) -> Self {
+        let mut guard = NEXT_VM_ID.lock().unwrap();
+        let id = *guard;
+        *guard = id + 1;
+
+        Self::new_from_ip_range(disk_config, "192.168", id)
+    }
+
+    pub fn default_net_string(&self) -> String {
+        format!(
+            "tap=,mac={},ip={},mask=255.255.255.0",
+            self.network.guest_mac, self.network.host_ip
+        )
+    }
+
+    pub fn default_net_string_w_iommu(&self) -> String {
+        format!(
+            "tap=,mac={},ip={},mask=255.255.255.0,iommu=on",
+            self.network.guest_mac, self.network.host_ip
+        )
+    }
+
+    pub fn ssh_command(&self, command: &str) -> Result<String, SshCommandError> {
+        ssh_command_ip(
+            command,
+            &self.network.guest_ip,
+            DEFAULT_SSH_RETRIES,
+            DEFAULT_SSH_TIMEOUT,
+        )
+    }
+
+    #[cfg(target_arch = "x86_64")]
+    pub fn ssh_command_l1(&self, command: &str) -> Result<String, SshCommandError> {
+        ssh_command_ip(
+            command,
+            &self.network.guest_ip,
+            DEFAULT_SSH_RETRIES,
+            DEFAULT_SSH_TIMEOUT,
+        )
+    }
+
+    #[cfg(target_arch = "x86_64")]
+    pub fn ssh_command_l2_1(&self, command: &str) -> Result<String, SshCommandError> {
+        ssh_command_ip(
+            command,
+            &self.network.l2_guest_ip1,
+            DEFAULT_SSH_RETRIES,
+            DEFAULT_SSH_TIMEOUT,
+        )
+    }
+
+    #[cfg(target_arch = "x86_64")]
+    pub fn ssh_command_l2_2(&self, command: &str) -> Result<String, SshCommandError> {
+        ssh_command_ip(
+            command,
+            &self.network.l2_guest_ip2,
+            DEFAULT_SSH_RETRIES,
+            DEFAULT_SSH_TIMEOUT,
+        )
+    }
+
+    #[cfg(target_arch = "x86_64")]
+    pub fn ssh_command_l2_3(&self, command: &str) -> Result<String, SshCommandError> {
+        ssh_command_ip(
+            command,
+            &self.network.l2_guest_ip3,
+            DEFAULT_SSH_RETRIES,
+            DEFAULT_SSH_TIMEOUT,
+        )
+    }
+
+    pub fn api_create_body(
+        &self,
+        cpu_count: u8,
+        fw_path: &str,
+        _kernel_path: &str,
+        _kernel_cmd: &str,
+    ) -> String {
+        #[cfg(all(target_arch = "x86_64", not(feature = "mshv")))]
+        format! {"{{\"cpus\":{{\"boot_vcpus\":{},\"max_vcpus\":{}}},\"kernel\":{{\"path\":\"{}\"}},\"cmdline\":{{\"args\": \"\"}},\"net\":[{{\"ip\":\"{}\", \"mask\":\"255.255.255.0\", \"mac\":\"{}\"}}], \"disks\":[{{\"path\":\"{}\"}}, {{\"path\":\"{}\"}}]}}",
+                 cpu_count,
+                 cpu_count,
+                 fw_path,
+                 self.network.host_ip,
+                 self.network.guest_mac,
+                 self.disk_config.disk(DiskType::OperatingSystem).unwrap().as_str(),
+                 self.disk_config.disk(DiskType::CloudInit).unwrap().as_str(),
+        }
+
+        #[cfg(any(target_arch = "aarch64", feature = "mshv"))]
+        format! {"{{\"cpus\":{{\"boot_vcpus\":{},\"max_vcpus\":{}}},\"kernel\":{{\"path\":\"{}\"}},\"cmdline\":{{\"args\": \"{}\"}},\"net\":[{{\"ip\":\"{}\", \"mask\":\"255.255.255.0\", \"mac\":\"{}\"}}], \"disks\":[{{\"path\":\"{}\"}}, {{\"path\":\"{}\"}}]}}",
+                 cpu_count,
+                 cpu_count,
+                 _kernel_path,
+                 _kernel_cmd,
+                 self.network.host_ip,
+                 self.network.guest_mac,
+                 self.disk_config.disk(DiskType::OperatingSystem).unwrap().as_str(),
+                 self.disk_config.disk(DiskType::CloudInit).unwrap().as_str(),
+        }
+    }
+
+    pub fn get_cpu_count(&self) -> Result<u32, Error> {
+        self.ssh_command("grep -c processor /proc/cpuinfo")?
+            .trim()
+            .parse()
+            .map_err(Error::Parsing)
+    }
+
+    #[cfg(target_arch = "x86_64")]
+    pub fn get_initial_apicid(&self) -> Result<u32, Error> {
+        self.ssh_command("grep \"initial apicid\" /proc/cpuinfo | grep -o \"[0-9]*\"")?
+            .trim()
+            .parse()
+            .map_err(Error::Parsing)
+    }
+
+    pub fn get_total_memory(&self) -> Result<u32, Error> {
+        self.ssh_command("grep MemTotal /proc/meminfo | grep -o \"[0-9]*\"")?
+            .trim()
+            .parse()
+            .map_err(Error::Parsing)
+    }
+
+    #[cfg(target_arch = "x86_64")]
+    pub fn get_total_memory_l2(&self) -> Result<u32, Error> {
+        self.ssh_command_l2_1("grep MemTotal /proc/meminfo | grep -o \"[0-9]*\"")?
+            .trim()
+            .parse()
+            .map_err(Error::Parsing)
+    }
+
+    pub fn get_numa_node_memory(&self, node_id: usize) -> Result<u32, Error> {
+        self.ssh_command(
+            format!(
+                "grep MemTotal /sys/devices/system/node/node{}/meminfo \
+                        | cut -d \":\" -f 2 | grep -o \"[0-9]*\"",
+                node_id
+            )
+            .as_str(),
+        )?
+        .trim()
+        .parse()
+        .map_err(Error::Parsing)
+    }
+
+    pub fn wait_vm_boot(&self, custom_timeout: Option<i32>) -> Result<(), Error> {
+        self.network
+            .wait_vm_boot(custom_timeout)
+            .map_err(Error::WaitForBoot)
+    }
+
+    pub fn check_numa_node_cpus(&self, node_id: usize, cpus: Vec<usize>) -> Result<(), Error> {
+        for cpu in cpus.iter() {
+            let cmd = format!(
+                "[ -d \"/sys/devices/system/node/node{}/cpu{}\" ]",
+                node_id, cpu
+            );
+            self.ssh_command(cmd.as_str())?;
+        }
+
+        Ok(())
+    }
+
+    pub fn check_numa_node_distances(
+        &self,
+        node_id: usize,
+        distances: &str,
+    ) -> Result<bool, Error> {
+        let cmd = format!("cat /sys/devices/system/node/node{}/distance", node_id);
+        if self.ssh_command(cmd.as_str())?.trim() == distances {
+            Ok(true)
+        } else {
+            Ok(false)
+        }
+    }
+
+    pub fn check_numa_common(
+        &self,
+        mem_ref: Option<&[u32]>,
+        node_ref: Option<&[Vec<usize>]>,
+        distance_ref: Option<&[&str]>,
+    ) {
+        if let Some(mem_ref) = mem_ref {
+            // Check each NUMA node has been assigned the right amount of
+            // memory.
+            for (i, &m) in mem_ref.iter().enumerate() {
+                assert!(self.get_numa_node_memory(i).unwrap_or_default() > m);
+            }
+        }
+
+        if let Some(node_ref) = node_ref {
+            // Check each NUMA node has been assigned the right CPUs set.
+            for (i, n) in node_ref.iter().enumerate() {
+                self.check_numa_node_cpus(i, n.clone()).unwrap();
+            }
+        }
+
+        if let Some(distance_ref) = distance_ref {
+            // Check each NUMA node has been assigned the right distances.
+            for (i, &d) in distance_ref.iter().enumerate() {
+                assert!(self.check_numa_node_distances(i, d).unwrap());
+            }
+        }
+    }
+
+    #[cfg(target_arch = "x86_64")]
+    pub fn check_sgx_support(&self) -> Result<(), Error> {
+        self.ssh_command(
+            "cpuid -l 0x7 -s 0 | tr -s [:space:] | grep -q 'SGX: \
+                    Software Guard Extensions supported = true'",
+        )?;
+        self.ssh_command(
+            "cpuid -l 0x7 -s 0 | tr -s [:space:] | grep -q 'SGX_LC: \
+                    SGX launch config supported = true'",
+        )?;
+        self.ssh_command(
+            "cpuid -l 0x12 -s 0 | tr -s [:space:] | grep -q 'SGX1 \
+                    supported = true'",
+        )?;
+
+        Ok(())
+    }
+
+    pub fn get_entropy(&self) -> Result<u32, Error> {
+        self.ssh_command("cat /proc/sys/kernel/random/entropy_avail")?
+            .trim()
+            .parse()
+            .map_err(Error::Parsing)
+    }
+
+    pub fn get_pci_bridge_class(&self) -> Result<String, Error> {
+        Ok(self
+            .ssh_command("cat /sys/bus/pci/devices/0000:00:00.0/class")?
+            .trim()
+            .to_string())
+    }
+
+    pub fn get_pci_device_ids(&self) -> Result<String, Error> {
+        Ok(self
+            .ssh_command("cat /sys/bus/pci/devices/*/device")?
+            .trim()
+            .to_string())
+    }
+
+    pub fn get_pci_vendor_ids(&self) -> Result<String, Error> {
+        Ok(self
+            .ssh_command("cat /sys/bus/pci/devices/*/vendor")?
+            .trim()
+            .to_string())
+    }
+
+    pub fn does_device_vendor_pair_match(
+        &self,
+        device_id: &str,
+        vendor_id: &str,
+    ) -> Result<bool, Error> {
+        // We are checking if console device's device id and vendor id pair matches
+        let devices = self.get_pci_device_ids()?;
+        let devices: Vec<&str> = devices.split('\n').collect();
+        let vendors = self.get_pci_vendor_ids()?;
+        let vendors: Vec<&str> = vendors.split('\n').collect();
+
+        for (index, d_id) in devices.iter().enumerate() {
+            if *d_id == device_id {
+                if let Some(v_id) = vendors.get(index) {
+                    if *v_id == vendor_id {
+                        return Ok(true);
+                    }
+                }
+            }
+        }
+
+        Ok(false)
+    }
+
+    pub fn valid_virtio_fs_cache_size(
+        &self,
+        dax: bool,
+        cache_size: Option<u64>,
+    ) -> Result<bool, Error> {
+        // SHM region is called different things depending on kernel
+        let shm_region = self
+            .ssh_command("sudo grep 'virtio[0-9]\\|virtio-pci-shm' /proc/iomem || true")?
+            .trim()
+            .to_string();
+
+        if shm_region.is_empty() {
+            return Ok(!dax);
+        }
+
+        // From this point, the region is not empty, hence it is an error
+        // if DAX is off.
+        if !dax {
+            return Ok(false);
+        }
+
+        let cache = if let Some(cache) = cache_size {
+            cache
+        } else {
+            // 8Gib by default
+            0x0002_0000_0000
+        };
+
+        let args: Vec<&str> = shm_region.split(':').collect();
+        if args.is_empty() {
+            return Ok(false);
+        }
+
+        let args: Vec<&str> = args[0].trim().split('-').collect();
+        if args.len() != 2 {
+            return Ok(false);
+        }
+
+        let start_addr = u64::from_str_radix(args[0], 16).map_err(Error::Parsing)?;
+        let end_addr = u64::from_str_radix(args[1], 16).map_err(Error::Parsing)?;
+
+        Ok(cache == (end_addr - start_addr + 1))
+    }
+
+    pub fn check_vsock(&self, socket: &str) {
+        // Listen from guest on vsock CID=3 PORT=16
+        // SOCKET-LISTEN:<domain>:<protocol>:<local-address>
+        let guest_ip = self.network.guest_ip.clone();
+        let listen_socat = thread::spawn(move || {
+            ssh_command_ip("sudo socat - SOCKET-LISTEN:40:0:x00x00x10x00x00x00x03x00x00x00x00x00x00x00 > vsock_log", &guest_ip, DEFAULT_SSH_RETRIES, DEFAULT_SSH_TIMEOUT).unwrap();
+        });
+
+        // Make sure socat is listening, which might take a few second on slow systems
+        thread::sleep(std::time::Duration::new(10, 0));
+
+        // Write something to vsock from the host
+        assert!(exec_host_command_status(&format!(
+            "echo -e \"CONNECT 16\\nHelloWorld!\" | socat - UNIX-CONNECT:{}",
+            socket
+        ))
+        .success());
+
+        // Wait for the thread to terminate.
+        listen_socat.join().unwrap();
+
+        assert_eq!(
+            self.ssh_command("cat vsock_log").unwrap().trim(),
+            "HelloWorld!"
+        );
+    }
+
+    #[cfg(target_arch = "x86_64")]
+    pub fn check_nvidia_gpu(&self) {
+        // Run CUDA sample to validate it can find the device
+        let device_query_result = self
+            .ssh_command("sudo /root/NVIDIA_CUDA-11.3_Samples/bin/x86_64/linux/release/deviceQuery")
+            .unwrap();
+        assert!(device_query_result.contains("Detected 1 CUDA Capable device"));
+        assert!(device_query_result.contains("Device 0: \"NVIDIA Tesla T4\""));
+        assert!(device_query_result.contains("Result = PASS"));
+
+        // Run NVIDIA DCGM Diagnostics to validate the device is functional
+        self.ssh_command("sudo nv-hostengine").unwrap();
+
+        assert!(self
+            .ssh_command("sudo dcgmi discovery -l")
+            .unwrap()
+            .contains("Name: NVIDIA Tesla T4"));
+        assert_eq!(
+            self.ssh_command("sudo dcgmi diag -r 'diagnostic' | grep Pass | wc -l")
+                .unwrap()
+                .trim(),
+            "10"
+        );
+    }
+
+    pub fn reboot_linux(&self, current_reboot_count: u32, custom_timeout: Option<i32>) {
+        let list_boots_cmd = "sudo journalctl --list-boots | wc -l";
+        let boot_count = self
+            .ssh_command(list_boots_cmd)
+            .unwrap()
+            .trim()
+            .parse::<u32>()
+            .unwrap_or_default();
+
+        assert_eq!(boot_count, current_reboot_count + 1);
+        self.ssh_command("sudo reboot").unwrap();
+
+        self.wait_vm_boot(custom_timeout).unwrap();
+        let boot_count = self
+            .ssh_command(list_boots_cmd)
+            .unwrap()
+            .trim()
+            .parse::<u32>()
+            .unwrap_or_default();
+        assert_eq!(boot_count, current_reboot_count + 2);
+    }
+
+    pub fn enable_memory_hotplug(&self) {
+        self.ssh_command("echo online | sudo tee /sys/devices/system/memory/auto_online_blocks")
+            .unwrap();
+    }
+
+    pub fn check_devices_common(&self, socket: Option<&String>, console_text: Option<&String>) {
+        // Check block devices are readable
+        self.ssh_command("sudo dd if=/dev/vda of=/dev/null bs=1M iflag=direct count=1024")
+            .unwrap();
+        self.ssh_command("sudo dd if=/dev/vdb of=/dev/null bs=1M iflag=direct count=8")
+            .unwrap();
+        // Check if the rng device is readable
+        self.ssh_command("sudo head -c 1000 /dev/hwrng > /dev/null")
+            .unwrap();
+        // Check vsock
+        if let Some(socket) = socket {
+            self.check_vsock(socket.as_str());
+        }
+        // Check if the console is usable
+        if let Some(console_text) = console_text {
+            let console_cmd = format!("echo {} | sudo tee /dev/hvc0", console_text);
+            self.ssh_command(&console_cmd).unwrap();
+        }
+        // The net device is 'automatically' exercised through the above 'ssh' commands
+    }
+}
+
+pub struct GuestCommand<'a> {
+    command: Command,
+    guest: &'a Guest,
+    capture_output: bool,
+}
+
+impl<'a> GuestCommand<'a> {
+    pub fn new(guest: &'a Guest) -> Self {
+        Self::new_with_binary_name(guest, "cloud-hypervisor")
+    }
+
+    pub fn new_with_binary_name(guest: &'a Guest, binary_name: &str) -> Self {
+        Self {
+            command: Command::new(clh_command(binary_name)),
+            guest,
+            capture_output: false,
+        }
+    }
+
+    pub fn capture_output(&mut self) -> &mut Self {
+        self.capture_output = true;
+        self
+    }
+
+    pub fn spawn(&mut self) -> io::Result<Child> {
+        println!(
+            "\n\n==== Start cloud-hypervisor command-line ====\n\n\
+                 {:?}\n\
+                 \n==== End cloud-hypervisor command-line ====\n\n",
+            self.command
+        );
+
+        if self.capture_output {
+            let child = self
+                .command
+                .arg("-v")
+                .stderr(Stdio::piped())
+                .stdout(Stdio::piped())
+                .spawn()
+                .unwrap();
+
+            let fd = child.stdout.as_ref().unwrap().as_raw_fd();
+            let pipesize = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) };
+            let fd = child.stderr.as_ref().unwrap().as_raw_fd();
+            let pipesize1 = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) };
+
+            if pipesize >= PIPE_SIZE && pipesize1 >= PIPE_SIZE {
+                Ok(child)
+            } else {
+                Err(std::io::Error::new(
+                    std::io::ErrorKind::Other,
+                    "resizing pipe w/ 'fnctl' failed!",
+                ))
+            }
+        } else {
+            self.command.arg("-v").spawn()
+        }
+    }
+
+    pub fn args<I, S>(&mut self, args: I) -> &mut Self
+    where
+        I: IntoIterator<Item = S>,
+        S: AsRef<OsStr>,
+    {
+        self.command.args(args);
+        self
+    }
+
+    pub fn default_disks(&mut self) -> &mut Self {
+        if self.guest.disk_config.disk(DiskType::CloudInit).is_some() {
+            self.args(&[
+                "--disk",
+                format!(
+                    "path={}",
+                    self.guest
+                        .disk_config
+                        .disk(DiskType::OperatingSystem)
+                        .unwrap()
+                )
+                .as_str(),
+                format!(
+                    "path={}",
+                    self.guest.disk_config.disk(DiskType::CloudInit).unwrap()
+                )
+                .as_str(),
+            ])
+        } else {
+            self.args(&[
+                "--disk",
+                format!(
+                    "path={}",
+                    self.guest
+                        .disk_config
+                        .disk(DiskType::OperatingSystem)
+                        .unwrap()
+                )
+                .as_str(),
+            ])
+        }
+    }
+
+    pub fn default_net(&mut self) -> &mut Self {
+        self.args(&["--net", self.guest.default_net_string().as_str()])
+    }
+}
+
+pub fn clh_command(cmd: &str) -> String {
+    env::var("BUILD_TARGET").map_or(
+        format!("target/x86_64-unknown-linux-gnu/release/{}", cmd),
+        |target| format!("target/{}/release/{}", target, cmd),
+    )
+}

tests/integration.rs

@@ -15,8 +15,6 @@ extern crate test_infra;
 
 use net_util::MacAddr;
 use std::collections::HashMap;
-use std::env;
-use std::ffi::OsStr;
 use std::fs;
 use std::io;
 use std::io::BufRead;
@@ -27,27 +25,15 @@ use std::os::unix::io::AsRawFd;
 use std::path::PathBuf;
 use std::process::{Child, Command, Stdio};
 use std::string::String;
+use std::sync::mpsc;
 use std::sync::mpsc::Receiver;
-use std::sync::{mpsc, Mutex};
+#[cfg(target_arch = "x86_64")]
+use std::sync::Mutex;
 use std::thread;
 use test_infra::*;
 use vmm_sys_util::{tempdir::TempDir, tempfile::TempFile};
-#[cfg_attr(target_arch = "aarch64", allow(unused_imports))]
 use wait_timeout::ChildExt;
 
-lazy_static! {
-    static ref NEXT_VM_ID: Mutex<u8> = Mutex::new(1);
-}
-
-struct Guest {
-    tmp_dir: TempDir,
-    disk_config: Box<dyn DiskConfig>,
-    network: GuestNetworkConfig,
-}
-
-// Safe to implement as we know we have no interior mutability
-impl std::panic::RefUnwindSafe for Guest {}
-
 #[cfg(target_arch = "x86_64")]
 mod x86_64 {
     pub const BIONIC_IMAGE_NAME: &str = "bionic-server-cloudimg-amd64.raw";
@@ -84,17 +70,8 @@ use aarch64::*;
 const DIRECT_KERNEL_BOOT_CMDLINE: &str =
     "root=/dev/vda1 console=hvc0 rw systemd.journald.forward_to_console=1";
 
-const PIPE_SIZE: i32 = 32 << 20;
-
 const CONSOLE_TEST_STRING: &str = "Started OpenBSD Secure Shell server";
 
-fn clh_command(cmd: &str) -> String {
-    env::var("BUILD_TARGET").map_or(
-        format!("target/x86_64-unknown-linux-gnu/release/{}", cmd),
-        |target| format!("target/{}/release/{}", target, cmd),
-    )
-}
-
 fn prepare_virtiofsd(
     tmp_dir: &TempDir,
     shared_dir: &str,
@@ -454,19 +431,6 @@ fn setup_ovs_dpdk_guests(guest1: &Guest, guest2: &Guest, api_socket: &str) -> (C
     (child1, child2)
 }
 
-#[derive(Debug)]
-enum Error {
-    Parsing(std::num::ParseIntError),
-    SshCommand(SshCommandError),
-    WaitForBoot(WaitForBootError),
-}
-
-impl From<SshCommandError> for Error {
-    fn from(e: SshCommandError) -> Self {
-        Self::SshCommand(e)
-    }
-}
-
 enum FwType {
     Ovmf,
     RustHypervisorFirmware,
@@ -490,555 +454,6 @@ fn fw_path(fw_type: FwType) -> String {
     fw_path.to_str().unwrap().to_string()
 }
 
-impl Guest {
-    fn new_from_ip_range(mut disk_config: Box<dyn DiskConfig>, class: &str, id: u8) -> Self {
-        let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap();
-
-        let network = GuestNetworkConfig {
-            guest_ip: format!("{}.{}.2", class, id),
-            l2_guest_ip1: format!("{}.{}.3", class, id),
-            l2_guest_ip2: format!("{}.{}.4", class, id),
-            l2_guest_ip3: format!("{}.{}.5", class, id),
-            host_ip: format!("{}.{}.1", class, id),
-            guest_mac: format!("12:34:56:78:90:{:02x}", id),
-            l2_guest_mac1: format!("de:ad:be:ef:12:{:02x}", id),
-            l2_guest_mac2: format!("de:ad:be:ef:34:{:02x}", id),
-            l2_guest_mac3: format!("de:ad:be:ef:56:{:02x}", id),
-            tcp_listener_port: DEFAULT_TCP_LISTENER_PORT + id as u16,
-        };
-
-        disk_config.prepare_files(&tmp_dir, &network);
-
-        Guest {
-            tmp_dir,
-            disk_config,
-            network,
-        }
-    }
-
-    fn new(disk_config: Box<dyn DiskConfig>) -> Self {
-        let mut guard = NEXT_VM_ID.lock().unwrap();
-        let id = *guard;
-        *guard = id + 1;
-
-        Self::new_from_ip_range(disk_config, "192.168", id)
-    }
-
-    fn default_net_string(&self) -> String {
-        format!(
-            "tap=,mac={},ip={},mask=255.255.255.0",
-            self.network.guest_mac, self.network.host_ip
-        )
-    }
-
-    fn default_net_string_w_iommu(&self) -> String {
-        format!(
-            "tap=,mac={},ip={},mask=255.255.255.0,iommu=on",
-            self.network.guest_mac, self.network.host_ip
-        )
-    }
-
-    fn ssh_command(&self, command: &str) -> Result<String, SshCommandError> {
-        ssh_command_ip(
-            command,
-            &self.network.guest_ip,
-            DEFAULT_SSH_RETRIES,
-            DEFAULT_SSH_TIMEOUT,
-        )
-    }
-
-    #[cfg(target_arch = "x86_64")]
-    fn ssh_command_l1(&self, command: &str) -> Result<String, SshCommandError> {
-        ssh_command_ip(
-            command,
-            &self.network.guest_ip,
-            DEFAULT_SSH_RETRIES,
-            DEFAULT_SSH_TIMEOUT,
-        )
-    }
-
-    #[cfg(target_arch = "x86_64")]
-    fn ssh_command_l2_1(&self, command: &str) -> Result<String, SshCommandError> {
-        ssh_command_ip(
-            command,
-            &self.network.l2_guest_ip1,
-            DEFAULT_SSH_RETRIES,
-            DEFAULT_SSH_TIMEOUT,
-        )
-    }
-
-    #[cfg(target_arch = "x86_64")]
-    fn ssh_command_l2_2(&self, command: &str) -> Result<String, SshCommandError> {
-        ssh_command_ip(
-            command,
-            &self.network.l2_guest_ip2,
-            DEFAULT_SSH_RETRIES,
-            DEFAULT_SSH_TIMEOUT,
-        )
-    }
-
-    #[cfg(target_arch = "x86_64")]
-    fn ssh_command_l2_3(&self, command: &str) -> Result<String, SshCommandError> {
-        ssh_command_ip(
-            command,
-            &self.network.l2_guest_ip3,
-            DEFAULT_SSH_RETRIES,
-            DEFAULT_SSH_TIMEOUT,
-        )
-    }
-
-    fn api_create_body(
-        &self,
-        cpu_count: u8,
-        fw_path: &str,
-        _kernel_path: &str,
-        _kernel_cmd: &str,
-    ) -> String {
-        #[cfg(all(target_arch = "x86_64", not(feature = "mshv")))]
-        format! {"{{\"cpus\":{{\"boot_vcpus\":{},\"max_vcpus\":{}}},\"kernel\":{{\"path\":\"{}\"}},\"cmdline\":{{\"args\": \"\"}},\"net\":[{{\"ip\":\"{}\", \"mask\":\"255.255.255.0\", \"mac\":\"{}\"}}], \"disks\":[{{\"path\":\"{}\"}}, {{\"path\":\"{}\"}}]}}",
-                 cpu_count,
-                 cpu_count,
-                 fw_path,
-                 self.network.host_ip,
-                 self.network.guest_mac,
-                 self.disk_config.disk(DiskType::OperatingSystem).unwrap().as_str(),
-                 self.disk_config.disk(DiskType::CloudInit).unwrap().as_str(),
-        }
-
-        #[cfg(any(target_arch = "aarch64", feature = "mshv"))]
-        format! {"{{\"cpus\":{{\"boot_vcpus\":{},\"max_vcpus\":{}}},\"kernel\":{{\"path\":\"{}\"}},\"cmdline\":{{\"args\": \"{}\"}},\"net\":[{{\"ip\":\"{}\", \"mask\":\"255.255.255.0\", \"mac\":\"{}\"}}], \"disks\":[{{\"path\":\"{}\"}}, {{\"path\":\"{}\"}}]}}",
-                 cpu_count,
-                 cpu_count,
-                 _kernel_path,
-                 _kernel_cmd,
-                 self.network.host_ip,
-                 self.network.guest_mac,
-                 self.disk_config.disk(DiskType::OperatingSystem).unwrap().as_str(),
-                 self.disk_config.disk(DiskType::CloudInit).unwrap().as_str(),
-        }
-    }
-
-    fn get_cpu_count(&self) -> Result<u32, Error> {
-        self.ssh_command("grep -c processor /proc/cpuinfo")?
-            .trim()
-            .parse()
-            .map_err(Error::Parsing)
-    }
-
-    #[cfg(target_arch = "x86_64")]
-    fn get_initial_apicid(&self) -> Result<u32, Error> {
-        self.ssh_command("grep \"initial apicid\" /proc/cpuinfo | grep -o \"[0-9]*\"")?
-            .trim()
-            .parse()
-            .map_err(Error::Parsing)
-    }
-
-    fn get_total_memory(&self) -> Result<u32, Error> {
-        self.ssh_command("grep MemTotal /proc/meminfo | grep -o \"[0-9]*\"")?
-            .trim()
-            .parse()
-            .map_err(Error::Parsing)
-    }
-
-    #[cfg(target_arch = "x86_64")]
-    fn get_total_memory_l2(&self) -> Result<u32, Error> {
-        self.ssh_command_l2_1("grep MemTotal /proc/meminfo | grep -o \"[0-9]*\"")?
-            .trim()
-            .parse()
-            .map_err(Error::Parsing)
-    }
-
-    fn get_numa_node_memory(&self, node_id: usize) -> Result<u32, Error> {
-        self.ssh_command(
-            format!(
-                "grep MemTotal /sys/devices/system/node/node{}/meminfo \
-                        | cut -d \":\" -f 2 | grep -o \"[0-9]*\"",
-                node_id
-            )
-            .as_str(),
-        )?
-        .trim()
-        .parse()
-        .map_err(Error::Parsing)
-    }
-
-    fn wait_vm_boot(&self, custom_timeout: Option<i32>) -> Result<(), Error> {
-        self.network
-            .wait_vm_boot(custom_timeout)
-            .map_err(Error::WaitForBoot)
-    }
-
-    fn check_numa_node_cpus(&self, node_id: usize, cpus: Vec<usize>) -> Result<(), Error> {
-        for cpu in cpus.iter() {
-            let cmd = format!(
-                "[ -d \"/sys/devices/system/node/node{}/cpu{}\" ]",
-                node_id, cpu
-            );
-            self.ssh_command(cmd.as_str())?;
-        }
-
-        Ok(())
-    }
-
-    fn check_numa_node_distances(&self, node_id: usize, distances: &str) -> Result<bool, Error> {
-        let cmd = format!("cat /sys/devices/system/node/node{}/distance", node_id);
-        if self.ssh_command(cmd.as_str())?.trim() == distances {
-            Ok(true)
-        } else {
-            Ok(false)
-        }
-    }
-
-    fn check_numa_common(
-        &self,
-        mem_ref: Option<&[u32]>,
-        node_ref: Option<&[Vec<usize>]>,
-        distance_ref: Option<&[&str]>,
-    ) {
-        if let Some(mem_ref) = mem_ref {
-            // Check each NUMA node has been assigned the right amount of
-            // memory.
-            for (i, &m) in mem_ref.iter().enumerate() {
-                assert!(self.get_numa_node_memory(i).unwrap_or_default() > m);
-            }
-        }
-
-        if let Some(node_ref) = node_ref {
-            // Check each NUMA node has been assigned the right CPUs set.
-            for (i, n) in node_ref.iter().enumerate() {
-                self.check_numa_node_cpus(i, n.clone()).unwrap();
-            }
-        }
-
-        if let Some(distance_ref) = distance_ref {
-            // Check each NUMA node has been assigned the right distances.
-            for (i, &d) in distance_ref.iter().enumerate() {
-                assert!(self.check_numa_node_distances(i, d).unwrap());
-            }
-        }
-    }
-
-    #[cfg(target_arch = "x86_64")]
-    fn check_sgx_support(&self) -> Result<(), Error> {
-        self.ssh_command(
-            "cpuid -l 0x7 -s 0 | tr -s [:space:] | grep -q 'SGX: \
-                    Software Guard Extensions supported = true'",
-        )?;
-        self.ssh_command(
-            "cpuid -l 0x7 -s 0 | tr -s [:space:] | grep -q 'SGX_LC: \
-                    SGX launch config supported = true'",
-        )?;
-        self.ssh_command(
-            "cpuid -l 0x12 -s 0 | tr -s [:space:] | grep -q 'SGX1 \
-                    supported = true'",
-        )?;
-
-        Ok(())
-    }
-
-    fn get_entropy(&self) -> Result<u32, Error> {
-        self.ssh_command("cat /proc/sys/kernel/random/entropy_avail")?
-            .trim()
-            .parse()
-            .map_err(Error::Parsing)
-    }
-
-    fn get_pci_bridge_class(&self) -> Result<String, Error> {
-        Ok(self
-            .ssh_command("cat /sys/bus/pci/devices/0000:00:00.0/class")?
-            .trim()
-            .to_string())
-    }
-
-    fn get_pci_device_ids(&self) -> Result<String, Error> {
-        Ok(self
-            .ssh_command("cat /sys/bus/pci/devices/*/device")?
-            .trim()
-            .to_string())
-    }
-
-    fn get_pci_vendor_ids(&self) -> Result<String, Error> {
-        Ok(self
-            .ssh_command("cat /sys/bus/pci/devices/*/vendor")?
-            .trim()
-            .to_string())
-    }
-
-    fn does_device_vendor_pair_match(
-        &self,
-        device_id: &str,
-        vendor_id: &str,
-    ) -> Result<bool, Error> {
-        // We are checking if console device's device id and vendor id pair matches
-        let devices = self.get_pci_device_ids()?;
-        let devices: Vec<&str> = devices.split('\n').collect();
-        let vendors = self.get_pci_vendor_ids()?;
-        let vendors: Vec<&str> = vendors.split('\n').collect();
-
-        for (index, d_id) in devices.iter().enumerate() {
-            if *d_id == device_id {
-                if let Some(v_id) = vendors.get(index) {
-                    if *v_id == vendor_id {
-                        return Ok(true);
-                    }
-                }
-            }
-        }
-
-        Ok(false)
-    }
-
-    fn valid_virtio_fs_cache_size(
-        &self,
-        dax: bool,
-        cache_size: Option<u64>,
-    ) -> Result<bool, Error> {
-        // SHM region is called different things depending on kernel
-        let shm_region = self
-            .ssh_command("sudo grep 'virtio[0-9]\\|virtio-pci-shm' /proc/iomem || true")?
-            .trim()
-            .to_string();
-
-        if shm_region.is_empty() {
-            return Ok(!dax);
-        }
-
-        // From this point, the region is not empty, hence it is an error
-        // if DAX is off.
-        if !dax {
-            return Ok(false);
-        }
-
-        let cache = if let Some(cache) = cache_size {
-            cache
-        } else {
-            // 8Gib by default
-            0x0002_0000_0000
-        };
-
-        let args: Vec<&str> = shm_region.split(':').collect();
-        if args.is_empty() {
-            return Ok(false);
-        }
-
-        let args: Vec<&str> = args[0].trim().split('-').collect();
-        if args.len() != 2 {
-            return Ok(false);
-        }
-
-        let start_addr = u64::from_str_radix(args[0], 16).map_err(Error::Parsing)?;
-        let end_addr = u64::from_str_radix(args[1], 16).map_err(Error::Parsing)?;
-
-        Ok(cache == (end_addr - start_addr + 1))
-    }
-
-    fn check_vsock(&self, socket: &str) {
-        // Listen from guest on vsock CID=3 PORT=16
-        // SOCKET-LISTEN:<domain>:<protocol>:<local-address>
-        let guest_ip = self.network.guest_ip.clone();
-        let listen_socat = thread::spawn(move || {
-            ssh_command_ip("sudo socat - SOCKET-LISTEN:40:0:x00x00x10x00x00x00x03x00x00x00x00x00x00x00 > vsock_log", &guest_ip, DEFAULT_SSH_RETRIES, DEFAULT_SSH_TIMEOUT).unwrap();
-        });
-
-        // Make sure socat is listening, which might take a few second on slow systems
-        thread::sleep(std::time::Duration::new(10, 0));
-
-        // Write something to vsock from the host
-        assert!(exec_host_command_status(&format!(
-            "echo -e \"CONNECT 16\\nHelloWorld!\" | socat - UNIX-CONNECT:{}",
-            socket
-        ))
-        .success());
-
-        // Wait for the thread to terminate.
-        listen_socat.join().unwrap();
-
-        assert_eq!(
-            self.ssh_command("cat vsock_log").unwrap().trim(),
-            "HelloWorld!"
-        );
-    }
-
-    #[cfg(target_arch = "x86_64")]
-    fn check_nvidia_gpu(&self) {
-        // Run CUDA sample to validate it can find the device
-        let device_query_result = self
-            .ssh_command("sudo /root/NVIDIA_CUDA-11.3_Samples/bin/x86_64/linux/release/deviceQuery")
-            .unwrap();
-        assert!(device_query_result.contains("Detected 1 CUDA Capable device"));
-        assert!(device_query_result.contains("Device 0: \"NVIDIA Tesla T4\""));
-        assert!(device_query_result.contains("Result = PASS"));
-
-        // Run NVIDIA DCGM Diagnostics to validate the device is functional
-        self.ssh_command("sudo nv-hostengine").unwrap();
-
-        assert!(self
-            .ssh_command("sudo dcgmi discovery -l")
-            .unwrap()
-            .contains("Name: NVIDIA Tesla T4"));
-        assert_eq!(
-            self.ssh_command("sudo dcgmi diag -r 'diagnostic' | grep Pass | wc -l")
-                .unwrap()
-                .trim(),
-            "10"
-        );
-    }
-
-    fn reboot_linux(&self, current_reboot_count: u32, custom_timeout: Option<i32>) {
-        let list_boots_cmd = "sudo journalctl --list-boots | wc -l";
-        let boot_count = self
-            .ssh_command(list_boots_cmd)
-            .unwrap()
-            .trim()
-            .parse::<u32>()
-            .unwrap_or_default();
-
-        assert_eq!(boot_count, current_reboot_count + 1);
-        self.ssh_command("sudo reboot").unwrap();
-
-        self.wait_vm_boot(custom_timeout).unwrap();
-        let boot_count = self
-            .ssh_command(list_boots_cmd)
-            .unwrap()
-            .trim()
-            .parse::<u32>()
-            .unwrap_or_default();
-        assert_eq!(boot_count, current_reboot_count + 2);
-    }
-
-    fn enable_memory_hotplug(&self) {
-        self.ssh_command("echo online | sudo tee /sys/devices/system/memory/auto_online_blocks")
-            .unwrap();
-    }
-
-    fn check_devices_common(&self, socket: Option<&String>, console_text: Option<&String>) {
-        // Check block devices are readable
-        self.ssh_command("sudo dd if=/dev/vda of=/dev/null bs=1M iflag=direct count=1024")
-            .unwrap();
-        self.ssh_command("sudo dd if=/dev/vdb of=/dev/null bs=1M iflag=direct count=8")
-            .unwrap();
-        // Check if the rng device is readable
-        self.ssh_command("sudo head -c 1000 /dev/hwrng > /dev/null")
-            .unwrap();
-        // Check vsock
-        if let Some(socket) = socket {
-            self.check_vsock(socket.as_str());
-        }
-        // Check if the console is usable
-        if let Some(console_text) = console_text {
-            let console_cmd = format!("echo {} | sudo tee /dev/hvc0", console_text);
-            self.ssh_command(&console_cmd).unwrap();
-        }
-        // The net device is 'automatically' exercised through the above 'ssh' commands
-    }
-}
-
-struct GuestCommand<'a> {
-    command: Command,
-    guest: &'a Guest,
-    capture_output: bool,
-}
-
-impl<'a> GuestCommand<'a> {
-    fn new(guest: &'a Guest) -> Self {
-        Self::new_with_binary_name(guest, "cloud-hypervisor")
-    }
-
-    fn new_with_binary_name(guest: &'a Guest, binary_name: &str) -> Self {
-        Self {
-            command: Command::new(clh_command(binary_name)),
-            guest,
-            capture_output: false,
-        }
-    }
-
-    fn capture_output(&mut self) -> &mut Self {
-        self.capture_output = true;
-        self
-    }
-
-    fn spawn(&mut self) -> io::Result<Child> {
-        println!(
-            "\n\n==== Start cloud-hypervisor command-line ====\n\n\
-                 {:?}\n\
-                 \n==== End cloud-hypervisor command-line ====\n\n",
-            self.command
-        );
-
-        if self.capture_output {
-            let child = self
-                .command
-                .arg("-v")
-                .stderr(Stdio::piped())
-                .stdout(Stdio::piped())
-                .spawn()
-                .unwrap();
-
-            let fd = child.stdout.as_ref().unwrap().as_raw_fd();
-            let pipesize = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) };
-            let fd = child.stderr.as_ref().unwrap().as_raw_fd();
-            let pipesize1 = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) };
-
-            if pipesize >= PIPE_SIZE && pipesize1 >= PIPE_SIZE {
-                Ok(child)
-            } else {
-                Err(std::io::Error::new(
-                    std::io::ErrorKind::Other,
-                    "resizing pipe w/ 'fnctl' failed!",
-                ))
-            }
-        } else {
-            self.command.arg("-v").spawn()
-        }
-    }
-
-    fn args<I, S>(&mut self, args: I) -> &mut Self
-    where
-        I: IntoIterator<Item = S>,
-        S: AsRef<OsStr>,
-    {
-        self.command.args(args);
-        self
-    }
-
-    fn default_disks(&mut self) -> &mut Self {
-        if self.guest.disk_config.disk(DiskType::CloudInit).is_some() {
-            self.args(&[
-                "--disk",
-                format!(
-                    "path={}",
-                    self.guest
-                        .disk_config
-                        .disk(DiskType::OperatingSystem)
-                        .unwrap()
-                )
-                .as_str(),
-                format!(
-                    "path={}",
-                    self.guest.disk_config.disk(DiskType::CloudInit).unwrap()
-                )
-                .as_str(),
-            ])
-        } else {
-            self.args(&[
-                "--disk",
-                format!(
-                    "path={}",
-                    self.guest
-                        .disk_config
-                        .disk(DiskType::OperatingSystem)
-                        .unwrap()
-                )
-                .as_str(),
-            ])
-        }
-    }
-
-    fn default_net(&mut self) -> &mut Self {
-        self.args(&["--net", self.guest.default_net_string().as_str()])
-    }
-}
-
 fn test_cpu_topology(threads_per_core: u8, cores_per_package: u8, packages: u8, use_fw: bool) {
     let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string());
     let guest = Guest::new(Box::new(focal));