From 16cb40733a9596f292e81cbe6ec4962574989ba4 Mon Sep 17 00:00:00 2001 From: Rob Bradford Date: Wed, 17 Mar 2021 09:33:49 +0000 Subject: [PATCH] tests: Extract out generic integration test code This includes: * OS disk image management * Cloud init creation * SSH to guest access * Waiting for guest to boot This will be useful in other projects that want to do similar things in their integration tests. Signed-off-by: Rob Bradford --- tests/integration.rs | 582 ++-------------------------------------- tests/test_infra/mod.rs | 582 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 601 insertions(+), 563 deletions(-) create mode 100644 tests/test_infra/mod.rs diff --git a/tests/integration.rs b/tests/integration.rs index a39c98b3c..9e2df520c 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -8,12 +8,15 @@ #[macro_use] extern crate lazy_static; +#[cfg(test)] +#[cfg(feature = "integration_tests")] +mod test_infra; + #[cfg(test)] #[cfg(feature = "integration_tests")] mod tests { - #![allow(dead_code)] + use crate::test_infra::*; use net_util::MacAddr; - use ssh2::Session; use std::collections::HashMap; use std::env; use std::ffi::OsStr; @@ -21,11 +24,9 @@ mod tests { use std::io; use std::io::BufRead; use std::io::{Read, Write}; - use std::net::{TcpListener, TcpStream}; use std::os::unix::io::AsRawFd; - use std::path::{Path, PathBuf}; + use std::path::PathBuf; use std::process::{Child, Command, Stdio}; - use std::str::FromStr; use std::string::String; use std::sync::mpsc::Receiver; use std::sync::{mpsc, Mutex}; @@ -38,23 +39,6 @@ mod tests { static ref NEXT_VM_ID: Mutex = Mutex::new(1); } - struct GuestNetworkConfig { - guest_ip: String, - l2_guest_ip1: String, - l2_guest_ip2: String, - l2_guest_ip3: String, - host_ip: String, - guest_mac: String, - l2_guest_mac1: String, - l2_guest_mac2: String, - l2_guest_mac3: String, - tcp_listener_port: u16, - } - - const DEFAULT_TCP_LISTENER_MESSAGE: &str = "booted"; - const DEFAULT_TCP_LISTENER_PORT: u16 = 8000; - const DEFAULT_TCP_LISTENER_TIMEOUT: i32 = 80; - struct Guest<'a> { tmp_dir: TempDir, disk_config: &'a dyn DiskConfig, @@ -65,23 +49,6 @@ mod tests { // Safe to implement as we know we have no interior mutability impl<'a> std::panic::RefUnwindSafe for Guest<'a> {} - enum DiskType { - OperatingSystem, - CloudInit, - } - - trait DiskConfig { - fn prepare_files(&mut self, tmp_dir: &TempDir, network: &GuestNetworkConfig); - fn prepare_cloudinit(&self, tmp_dir: &TempDir, network: &GuestNetworkConfig) -> String; - fn disk(&self, disk_type: DiskType) -> Option; - } - - struct UbuntuDiskConfig { - osdisk_path: String, - cloudinit_path: String, - image_name: String, - } - #[cfg(target_arch = "x86_64")] const BIONIC_IMAGE_NAME: &str = "bionic-server-cloudimg-amd64.raw"; #[cfg(target_arch = "x86_64")] @@ -117,138 +84,6 @@ mod tests { const PIPE_SIZE: i32 = 32 << 20; - impl UbuntuDiskConfig { - fn new(image_name: String) -> Self { - UbuntuDiskConfig { - image_name, - osdisk_path: String::new(), - cloudinit_path: String::new(), - } - } - } - - struct WindowsDiskConfig { - image_name: String, - osdisk_path: String, - loopback_device: String, - windows_snapshot_cow: String, - windows_snapshot: String, - } - - impl WindowsDiskConfig { - fn new(image_name: String) -> Self { - WindowsDiskConfig { - image_name, - osdisk_path: String::new(), - loopback_device: String::new(), - windows_snapshot_cow: String::new(), - windows_snapshot: String::new(), - } - } - } - - impl Drop for WindowsDiskConfig { - fn drop(&mut self) { - // dmsetup remove windows-snapshot-1 - std::process::Command::new("dmsetup") - .arg("remove") - .arg(self.windows_snapshot.as_str()) - .output() - .expect("Expect removing Windows snapshot with 'dmsetup' to succeed"); - - // dmsetup remove windows-snapshot-cow-1 - std::process::Command::new("dmsetup") - .arg("remove") - .arg(self.windows_snapshot_cow.as_str()) - .output() - .expect("Expect removing Windows snapshot CoW with 'dmsetup' to succeed"); - - // losetup -d - std::process::Command::new("losetup") - .args(&["-d", self.loopback_device.as_str()]) - .output() - .expect("Expect removing loopback device to succeed"); - } - } - - fn handle_child_output( - r: Result<(), std::boxed::Box>, - output: &std::process::Output, - ) { - use std::os::unix::process::ExitStatusExt; - if r.is_ok() && output.status.success() { - return; - } - - match output.status.code() { - None => { - // Don't treat child.kill() as a problem - if output.status.signal() == Some(9) && r.is_ok() { - return; - } - - eprintln!( - "==== child killed by signal: {} ====", - output.status.signal().unwrap() - ); - } - Some(code) => { - eprintln!("\n\n==== child exit code: {} ====", code); - } - } - - eprintln!( - "\n\n==== Start child stdout ====\n\n{}\n\n==== End child stdout ====", - String::from_utf8_lossy(&output.stdout) - ); - eprintln!( - "\n\n==== Start child stderr ====\n\n{}\n\n==== End child stderr ====", - String::from_utf8_lossy(&output.stderr) - ); - - panic!("Test failed") - } - - fn rate_limited_copy, Q: AsRef>(from: P, to: Q) -> io::Result { - for i in 0..10 { - let free_bytes = unsafe { - let mut stats = std::mem::MaybeUninit::zeroed(); - let fs_name = std::ffi::CString::new("/tmp").unwrap(); - libc::statvfs(fs_name.as_ptr(), stats.as_mut_ptr()); - - let free_blocks = stats.assume_init().f_bfree; - let block_size = stats.assume_init().f_bsize; - - free_blocks * block_size - }; - - // Make sure there is at least 6 GiB of space - if free_bytes < 6 << 30 { - eprintln!( - "Not enough space on disk ({}). Attempt {} of 10. Sleeping.", - free_bytes, i - ); - thread::sleep(std::time::Duration::new(60, 0)); - continue; - } - - match fs::copy(&from, &to) { - Err(e) => { - if let Some(errno) = e.raw_os_error() { - if errno == libc::ENOSPC { - eprintln!("Copy returned ENOSPC. Attempt {} of 10. Sleeping.", i); - thread::sleep(std::time::Duration::new(60, 0)); - continue; - } - } - return Err(e); - } - Ok(i) => return Ok(i), - } - } - Err(io::Error::last_os_error()) - } - fn clh_command(cmd: &str) -> String { env::var("BUILD_TARGET").map_or( format!("target/x86_64-unknown-linux-gnu/release/{}", cmd), @@ -256,222 +91,6 @@ mod tests { ) } - impl DiskConfig for UbuntuDiskConfig { - fn prepare_cloudinit(&self, tmp_dir: &TempDir, network: &GuestNetworkConfig) -> String { - let cloudinit_file_path = - String::from(tmp_dir.as_path().join("cloudinit").to_str().unwrap()); - - let cloud_init_directory = tmp_dir.as_path().join("cloud-init").join("ubuntu"); - - fs::create_dir_all(&cloud_init_directory) - .expect("Expect creating cloud-init directory to succeed"); - - let source_file_dir = std::env::current_dir() - .unwrap() - .join("test_data") - .join("cloud-init") - .join("ubuntu"); - - vec!["meta-data"].iter().for_each(|x| { - rate_limited_copy(source_file_dir.join(x), cloud_init_directory.join(x)) - .expect("Expect copying cloud-init meta-data to succeed"); - }); - - let mut user_data_string = String::new(); - fs::File::open(source_file_dir.join("user-data")) - .unwrap() - .read_to_string(&mut user_data_string) - .expect("Expected reading user-data file in to succeed"); - user_data_string = user_data_string.replace( - "@DEFAULT_TCP_LISTENER_MESSAGE", - &DEFAULT_TCP_LISTENER_MESSAGE, - ); - user_data_string = user_data_string.replace("@HOST_IP", &network.host_ip); - user_data_string = user_data_string - .replace("@TCP_LISTENER_PORT", &network.tcp_listener_port.to_string()); - - fs::File::create(cloud_init_directory.join("user-data")) - .unwrap() - .write_all(&user_data_string.as_bytes()) - .expect("Expected writing out user-data to succeed"); - - let mut network_config_string = String::new(); - - fs::File::open(source_file_dir.join("network-config")) - .unwrap() - .read_to_string(&mut network_config_string) - .expect("Expected reading network-config file in to succeed"); - - network_config_string = network_config_string.replace("192.168.2.1", &network.host_ip); - network_config_string = network_config_string.replace("192.168.2.2", &network.guest_ip); - network_config_string = - network_config_string.replace("192.168.2.3", &network.l2_guest_ip1); - network_config_string = - network_config_string.replace("192.168.2.4", &network.l2_guest_ip2); - network_config_string = - network_config_string.replace("192.168.2.5", &network.l2_guest_ip3); - network_config_string = - network_config_string.replace("12:34:56:78:90:ab", &network.guest_mac); - network_config_string = - network_config_string.replace("de:ad:be:ef:12:34", &network.l2_guest_mac1); - network_config_string = - network_config_string.replace("de:ad:be:ef:34:56", &network.l2_guest_mac2); - network_config_string = - network_config_string.replace("de:ad:be:ef:56:78", &network.l2_guest_mac3); - - fs::File::create(cloud_init_directory.join("network-config")) - .unwrap() - .write_all(&network_config_string.as_bytes()) - .expect("Expected writing out network-config to succeed"); - - std::process::Command::new("mkdosfs") - .args(&["-n", "cidata"]) - .args(&["-C", cloudinit_file_path.as_str()]) - .arg("8192") - .output() - .expect("Expect creating disk image to succeed"); - - vec!["user-data", "meta-data", "network-config"] - .iter() - .for_each(|x| { - std::process::Command::new("mcopy") - .arg("-o") - .args(&["-i", cloudinit_file_path.as_str()]) - .args(&["-s", cloud_init_directory.join(x).to_str().unwrap(), "::"]) - .output() - .expect("Expect copying files to disk image to succeed"); - }); - - cloudinit_file_path - } - - fn prepare_files(&mut self, tmp_dir: &TempDir, network: &GuestNetworkConfig) { - let mut workload_path = dirs::home_dir().unwrap(); - workload_path.push("workloads"); - - let mut osdisk_base_path = workload_path; - osdisk_base_path.push(&self.image_name); - - let osdisk_path = String::from(tmp_dir.as_path().join("osdisk.img").to_str().unwrap()); - let cloudinit_path = self.prepare_cloudinit(tmp_dir, network); - - rate_limited_copy(osdisk_base_path, &osdisk_path) - .expect("copying of OS source disk image failed"); - - self.cloudinit_path = cloudinit_path; - self.osdisk_path = osdisk_path; - } - - fn disk(&self, disk_type: DiskType) -> Option { - match disk_type { - DiskType::OperatingSystem => Some(self.osdisk_path.clone()), - DiskType::CloudInit => Some(self.cloudinit_path.clone()), - } - } - } - - impl DiskConfig for WindowsDiskConfig { - fn prepare_cloudinit(&self, _tmp_dir: &TempDir, _network: &GuestNetworkConfig) -> String { - String::new() - } - - fn prepare_files(&mut self, tmp_dir: &TempDir, _network: &GuestNetworkConfig) { - let mut workload_path = dirs::home_dir().unwrap(); - workload_path.push("workloads"); - - let mut osdisk_path = workload_path; - osdisk_path.push(&self.image_name); - - let osdisk_blk_size = fs::metadata(osdisk_path) - .expect("Expect retrieving Windows image metadata") - .len() - >> 9; - - let snapshot_cow_path = - String::from(tmp_dir.as_path().join("snapshot_cow").to_str().unwrap()); - - // Create and truncate CoW file for device mapper - let cow_file_size: u64 = 1 << 30; - let cow_file_blk_size = cow_file_size >> 9; - let cow_file = std::fs::File::create(snapshot_cow_path.as_str()) - .expect("Expect creating CoW image to succeed"); - cow_file - .set_len(cow_file_size) - .expect("Expect truncating CoW image to succeed"); - - // losetup --find --show /tmp/snapshot_cow - let loopback_device = std::process::Command::new("losetup") - .arg("--find") - .arg("--show") - .arg(snapshot_cow_path.as_str()) - .output() - .expect("Expect creating loopback device from snapshot CoW image to succeed"); - - self.loopback_device = String::from_utf8_lossy(&loopback_device.stdout) - .trim() - .to_string(); - - let random_extension = tmp_dir.as_path().file_name().unwrap(); - let windows_snapshot_cow = format!( - "windows-snapshot-cow-{}", - random_extension.to_str().unwrap() - ); - - // dmsetup create windows-snapshot-cow-1 --table '0 2097152 linear /dev/loop1 0' - std::process::Command::new("dmsetup") - .arg("create") - .arg(windows_snapshot_cow.as_str()) - .args(&[ - "--table", - format!("0 {} linear {} 0", cow_file_blk_size, self.loopback_device).as_str(), - ]) - .output() - .expect("Expect creating Windows snapshot CoW with 'dmsetup' to succeed"); - - let windows_snapshot = - format!("windows-snapshot-{}", random_extension.to_str().unwrap()); - - // dmsetup mknodes - std::process::Command::new("dmsetup") - .arg("mknodes") - .output() - .expect("Expect device mapper nodes to be ready"); - - // dmsetup create windows-snapshot-1 --table '0 41943040 snapshot /dev/mapper/windows-base /dev/mapper/windows-snapshot-cow-1 P 8' - std::process::Command::new("dmsetup") - .arg("create") - .arg(windows_snapshot.as_str()) - .args(&[ - "--table", - format!( - "0 {} snapshot /dev/mapper/windows-base /dev/mapper/{} P 8", - osdisk_blk_size, - windows_snapshot_cow.as_str() - ) - .as_str(), - ]) - .output() - .expect("Expect creating Windows snapshot with 'dmsetup' to succeed"); - - // dmsetup mknodes - std::process::Command::new("dmsetup") - .arg("mknodes") - .output() - .expect("Expect device mapper nodes to be ready"); - - self.osdisk_path = format!("/dev/mapper/{}", windows_snapshot); - self.windows_snapshot_cow = windows_snapshot_cow; - self.windows_snapshot = windows_snapshot; - } - - fn disk(&self, disk_type: DiskType) -> Option { - match disk_type { - DiskType::OperatingSystem => Some(self.osdisk_path.clone()), - DiskType::CloudInit => None, - } - } - } - fn prepare_virtiofsd( tmp_dir: &TempDir, shared_dir: &str, @@ -715,102 +334,16 @@ mod tests { cmd.status().expect("Failed to launch ch-remote").success() } - #[derive(Debug)] - struct PasswordAuth { - username: String, - password: String, - } - - const DEFAULT_SSH_RETRIES: u8 = 6; - const DEFAULT_SSH_TIMEOUT: u8 = 10; - fn ssh_command_ip_with_auth( - command: &str, - auth: &PasswordAuth, - ip: &str, - retries: u8, - timeout: u8, - ) -> Result { - let mut s = String::new(); - - let mut counter = 0; - loop { - match (|| -> Result<(), Error> { - let tcp = TcpStream::connect(format!("{}:22", ip)).map_err(Error::Connection)?; - let mut sess = Session::new().unwrap(); - sess.set_tcp_stream(tcp); - sess.handshake().map_err(Error::Handshake)?; - - sess.userauth_password(&auth.username, &auth.password) - .map_err(Error::Authentication)?; - assert!(sess.authenticated()); - - let mut channel = sess.channel_session().map_err(Error::ChannelSession)?; - channel.exec(command).map_err(Error::Command)?; - - // Intentionally ignore these results here as their failure - // does not precipitate a repeat - let _ = channel.read_to_string(&mut s); - let _ = channel.close(); - let _ = channel.wait_close(); - Ok(()) - })() { - Ok(_) => break, - Err(e) => { - counter += 1; - if counter >= retries { - eprintln!( - "\n\n==== Start ssh command output (FAILED) ====\n\n\ - command=\"{}\"\n\ - auth=\"{:#?}\"\n\ - ip=\"{}\"\n\ - output=\"{}\"\n\ - error=\"{}\"\n\ - \n==== End ssh command outout ====\n\n", - command, auth, ip, s, e - ); - - return Err(e); - } - } - }; - thread::sleep(std::time::Duration::new((timeout * counter).into(), 0)); - } - Ok(s) - } - - fn ssh_command_ip(command: &str, ip: &str, retries: u8, timeout: u8) -> Result { - ssh_command_ip_with_auth( - command, - &PasswordAuth { - username: String::from("cloud"), - password: String::from("cloud123"), - }, - ip, - retries, - timeout, - ) - } - #[derive(Debug)] enum Error { - Connection(std::io::Error), - Handshake(ssh2::Error), - Authentication(ssh2::Error), - ChannelSession(ssh2::Error), - Command(ssh2::Error), Parsing(std::num::ParseIntError), - EpollWait(std::io::Error), - EpollWaitTimeout, - ReadToString(std::io::Error), - SetReadTimeout(std::io::Error), - WrongGuestAddr, + SshCommand(SshCommandError), + WaitForBoot(WaitForBootError), } - impl std::error::Error for Error {} - - impl std::fmt::Display for Error { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{:?}", self) + impl From for Error { + fn from(e: SshCommandError) -> Self { + Self::SshCommand(e) } } @@ -872,7 +405,7 @@ mod tests { ) } - fn ssh_command(&self, command: &str) -> Result { + fn ssh_command(&self, command: &str) -> Result { ssh_command_ip( command, &self.network.guest_ip, @@ -881,7 +414,7 @@ mod tests { ) } - fn ssh_command_l1(&self, command: &str) -> Result { + fn ssh_command_l1(&self, command: &str) -> Result { ssh_command_ip( command, &self.network.guest_ip, @@ -890,7 +423,7 @@ mod tests { ) } - fn ssh_command_l2_1(&self, command: &str) -> Result { + fn ssh_command_l2_1(&self, command: &str) -> Result { ssh_command_ip( command, &self.network.l2_guest_ip1, @@ -899,7 +432,7 @@ mod tests { ) } - fn ssh_command_l2_2(&self, command: &str) -> Result { + fn ssh_command_l2_2(&self, command: &str) -> Result { ssh_command_ip( command, &self.network.l2_guest_ip2, @@ -908,7 +441,7 @@ mod tests { ) } - fn ssh_command_l2_3(&self, command: &str) -> Result { + fn ssh_command_l2_3(&self, command: &str) -> Result { ssh_command_ip( command, &self.network.l2_guest_ip3, @@ -989,86 +522,9 @@ mod tests { } fn wait_vm_boot(&self, custom_timeout: Option) -> Result<(), Error> { - let start = std::time::Instant::now(); - // The 'port' is unique per 'GUEST' and listening to wild-card ip avoids retrying on 'TcpListener::bind()' - let listen_addr = format!("0.0.0.0:{}", self.network.tcp_listener_port); - let expected_guest_addr = self.network.guest_ip.as_str(); - let mut s = String::new(); - let timeout = match custom_timeout { - Some(t) => t, - None => DEFAULT_TCP_LISTENER_TIMEOUT, - }; - - match (|| -> Result<(), Error> { - let listener = - TcpListener::bind(&listen_addr.as_str()).map_err(Error::Connection)?; - listener - .set_nonblocking(true) - .expect("Cannot set non-blocking for tcp listener"); - - // Reply on epoll w/ timeout to wait for guest connections faithfully - let epoll_fd = epoll::create(true).expect("Cannot create epoll fd"); - epoll::ctl( - epoll_fd, - epoll::ControlOptions::EPOLL_CTL_ADD, - listener.as_raw_fd(), - epoll::Event::new(epoll::Events::EPOLLIN, 0), - ) - .expect("Cannot add 'tcp_listener' event to epoll"); - let mut events = vec![epoll::Event::new(epoll::Events::empty(), 0); 1]; - loop { - let num_events = - match epoll::wait(epoll_fd, timeout * 1000_i32, &mut events[..]) { - Ok(num_events) => Ok(num_events), - Err(e) => match e.raw_os_error() { - Some(libc::EAGAIN) | Some(libc::EINTR) => continue, - _ => Err(e), - }, - } - .map_err(Error::EpollWait)?; - if num_events == 0 { - return Err(Error::EpollWaitTimeout); - } - break; - } - - match listener.accept() { - Ok((_, addr)) => { - // Make sure the connection is from the expected 'guest_addr' - if addr.ip() != std::net::IpAddr::from_str(expected_guest_addr).unwrap() { - s = format!( - "Expecting the guest ip '{}' while being connected with ip '{}'", - expected_guest_addr, - addr.ip() - ); - return Err(Error::WrongGuestAddr); - } - - Ok(()) - } - Err(e) => { - s = "TcpListener::accept() failed".to_string(); - Err(Error::Connection(e)) - } - } - })() { - Err(e) => { - let duration = start.elapsed(); - eprintln!( - "\n\n==== Start 'wait_vm_boot' (FAILED) ====\n\n\ - duration =\"{:?}, timeout = {}s\"\n\ - listen_addr=\"{}\"\n\ - expected_guest_addr=\"{}\"\n\ - message=\"{}\"\n\ - error=\"{}\"\n\ - \n==== End 'wait_vm_boot' outout ====\n\n", - duration, timeout, listen_addr, expected_guest_addr, s, e - ); - - Err(e) - } - Ok(_) => Ok(()), - } + self.network + .wait_vm_boot(custom_timeout) + .map_err(Error::WaitForBoot) } fn check_numa_node_cpus(&self, node_id: usize, cpus: Vec) -> Result { diff --git a/tests/test_infra/mod.rs b/tests/test_infra/mod.rs new file mode 100644 index 000000000..532f0018f --- /dev/null +++ b/tests/test_infra/mod.rs @@ -0,0 +1,582 @@ +// Copyright © 2021 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +use ssh2::Session; +use std::fs; +use std::io; +use std::io::{Read, Write}; +use std::net::TcpListener; +use std::net::TcpStream; +use std::os::unix::io::AsRawFd; +use std::path::Path; +use std::str::FromStr; +use std::thread; +use vmm_sys_util::tempdir::TempDir; + +pub const DEFAULT_TCP_LISTENER_MESSAGE: &str = "booted"; + +pub struct GuestNetworkConfig { + pub guest_ip: String, + pub l2_guest_ip1: String, + pub l2_guest_ip2: String, + pub l2_guest_ip3: String, + pub host_ip: String, + pub guest_mac: String, + pub l2_guest_mac1: String, + pub l2_guest_mac2: String, + pub l2_guest_mac3: String, + pub tcp_listener_port: u16, +} + +pub const DEFAULT_TCP_LISTENER_PORT: u16 = 8000; +pub const DEFAULT_TCP_LISTENER_TIMEOUT: i32 = 80; + +#[derive(Debug)] +pub enum WaitForBootError { + EpollWait(std::io::Error), + Listen(std::io::Error), + EpollWaitTimeout, + WrongGuestAddr, + Accept(std::io::Error), +} + +impl GuestNetworkConfig { + pub fn wait_vm_boot(&self, custom_timeout: Option) -> Result<(), WaitForBootError> { + let start = std::time::Instant::now(); + // The 'port' is unique per 'GUEST' and listening to wild-card ip avoids retrying on 'TcpListener::bind()' + let listen_addr = format!("0.0.0.0:{}", self.tcp_listener_port); + let expected_guest_addr = self.guest_ip.as_str(); + let mut s = String::new(); + let timeout = match custom_timeout { + Some(t) => t, + None => DEFAULT_TCP_LISTENER_TIMEOUT, + }; + + match (|| -> Result<(), WaitForBootError> { + let listener = + TcpListener::bind(&listen_addr.as_str()).map_err(WaitForBootError::Listen)?; + listener + .set_nonblocking(true) + .expect("Cannot set non-blocking for tcp listener"); + + // Reply on epoll w/ timeout to wait for guest connections faithfully + let epoll_fd = epoll::create(true).expect("Cannot create epoll fd"); + epoll::ctl( + epoll_fd, + epoll::ControlOptions::EPOLL_CTL_ADD, + listener.as_raw_fd(), + epoll::Event::new(epoll::Events::EPOLLIN, 0), + ) + .expect("Cannot add 'tcp_listener' event to epoll"); + let mut events = vec![epoll::Event::new(epoll::Events::empty(), 0); 1]; + loop { + let num_events = match epoll::wait(epoll_fd, timeout * 1000_i32, &mut events[..]) { + Ok(num_events) => Ok(num_events), + Err(e) => match e.raw_os_error() { + Some(libc::EAGAIN) | Some(libc::EINTR) => continue, + _ => Err(e), + }, + } + .map_err(WaitForBootError::EpollWait)?; + if num_events == 0 { + return Err(WaitForBootError::EpollWaitTimeout); + } + break; + } + + match listener.accept() { + Ok((_, addr)) => { + // Make sure the connection is from the expected 'guest_addr' + if addr.ip() != std::net::IpAddr::from_str(expected_guest_addr).unwrap() { + s = format!( + "Expecting the guest ip '{}' while being connected with ip '{}'", + expected_guest_addr, + addr.ip() + ); + return Err(WaitForBootError::WrongGuestAddr); + } + + Ok(()) + } + Err(e) => { + s = "TcpListener::accept() failed".to_string(); + Err(WaitForBootError::Accept(e)) + } + } + })() { + Err(e) => { + let duration = start.elapsed(); + eprintln!( + "\n\n==== Start 'wait_vm_boot' (FAILED) ====\n\n\ + duration =\"{:?}, timeout = {}s\"\n\ + listen_addr=\"{}\"\n\ + expected_guest_addr=\"{}\"\n\ + message=\"{}\"\n\ + error=\"{:?}\"\n\ + \n==== End 'wait_vm_boot' outout ====\n\n", + duration, timeout, listen_addr, expected_guest_addr, s, e + ); + + Err(e) + } + Ok(_) => Ok(()), + } + } +} + +pub enum DiskType { + OperatingSystem, + CloudInit, +} + +pub trait DiskConfig { + fn prepare_files(&mut self, tmp_dir: &TempDir, network: &GuestNetworkConfig); + fn prepare_cloudinit(&self, tmp_dir: &TempDir, network: &GuestNetworkConfig) -> String; + fn disk(&self, disk_type: DiskType) -> Option; +} + +pub struct UbuntuDiskConfig { + osdisk_path: String, + cloudinit_path: String, + image_name: String, +} + +impl UbuntuDiskConfig { + pub fn new(image_name: String) -> Self { + UbuntuDiskConfig { + image_name, + osdisk_path: String::new(), + cloudinit_path: String::new(), + } + } +} + +pub struct WindowsDiskConfig { + image_name: String, + osdisk_path: String, + loopback_device: String, + windows_snapshot_cow: String, + windows_snapshot: String, +} + +impl WindowsDiskConfig { + pub fn new(image_name: String) -> Self { + WindowsDiskConfig { + image_name, + osdisk_path: String::new(), + loopback_device: String::new(), + windows_snapshot_cow: String::new(), + windows_snapshot: String::new(), + } + } +} + +impl Drop for WindowsDiskConfig { + fn drop(&mut self) { + // dmsetup remove windows-snapshot-1 + std::process::Command::new("dmsetup") + .arg("remove") + .arg(self.windows_snapshot.as_str()) + .output() + .expect("Expect removing Windows snapshot with 'dmsetup' to succeed"); + + // dmsetup remove windows-snapshot-cow-1 + std::process::Command::new("dmsetup") + .arg("remove") + .arg(self.windows_snapshot_cow.as_str()) + .output() + .expect("Expect removing Windows snapshot CoW with 'dmsetup' to succeed"); + + // losetup -d + std::process::Command::new("losetup") + .args(&["-d", self.loopback_device.as_str()]) + .output() + .expect("Expect removing loopback device to succeed"); + } +} + +impl DiskConfig for UbuntuDiskConfig { + fn prepare_cloudinit(&self, tmp_dir: &TempDir, network: &GuestNetworkConfig) -> String { + let cloudinit_file_path = + String::from(tmp_dir.as_path().join("cloudinit").to_str().unwrap()); + + let cloud_init_directory = tmp_dir.as_path().join("cloud-init").join("ubuntu"); + + fs::create_dir_all(&cloud_init_directory) + .expect("Expect creating cloud-init directory to succeed"); + + let source_file_dir = std::env::current_dir() + .unwrap() + .join("test_data") + .join("cloud-init") + .join("ubuntu"); + + vec!["meta-data"].iter().for_each(|x| { + rate_limited_copy(source_file_dir.join(x), cloud_init_directory.join(x)) + .expect("Expect copying cloud-init meta-data to succeed"); + }); + + let mut user_data_string = String::new(); + fs::File::open(source_file_dir.join("user-data")) + .unwrap() + .read_to_string(&mut user_data_string) + .expect("Expected reading user-data file in to succeed"); + user_data_string = user_data_string.replace( + "@DEFAULT_TCP_LISTENER_MESSAGE", + &DEFAULT_TCP_LISTENER_MESSAGE, + ); + user_data_string = user_data_string.replace("@HOST_IP", &network.host_ip); + user_data_string = + user_data_string.replace("@TCP_LISTENER_PORT", &network.tcp_listener_port.to_string()); + + fs::File::create(cloud_init_directory.join("user-data")) + .unwrap() + .write_all(&user_data_string.as_bytes()) + .expect("Expected writing out user-data to succeed"); + + let mut network_config_string = String::new(); + + fs::File::open(source_file_dir.join("network-config")) + .unwrap() + .read_to_string(&mut network_config_string) + .expect("Expected reading network-config file in to succeed"); + + network_config_string = network_config_string.replace("192.168.2.1", &network.host_ip); + network_config_string = network_config_string.replace("192.168.2.2", &network.guest_ip); + network_config_string = network_config_string.replace("192.168.2.3", &network.l2_guest_ip1); + network_config_string = network_config_string.replace("192.168.2.4", &network.l2_guest_ip2); + network_config_string = network_config_string.replace("192.168.2.5", &network.l2_guest_ip3); + network_config_string = + network_config_string.replace("12:34:56:78:90:ab", &network.guest_mac); + network_config_string = + network_config_string.replace("de:ad:be:ef:12:34", &network.l2_guest_mac1); + network_config_string = + network_config_string.replace("de:ad:be:ef:34:56", &network.l2_guest_mac2); + network_config_string = + network_config_string.replace("de:ad:be:ef:56:78", &network.l2_guest_mac3); + + fs::File::create(cloud_init_directory.join("network-config")) + .unwrap() + .write_all(&network_config_string.as_bytes()) + .expect("Expected writing out network-config to succeed"); + + std::process::Command::new("mkdosfs") + .args(&["-n", "cidata"]) + .args(&["-C", cloudinit_file_path.as_str()]) + .arg("8192") + .output() + .expect("Expect creating disk image to succeed"); + + vec!["user-data", "meta-data", "network-config"] + .iter() + .for_each(|x| { + std::process::Command::new("mcopy") + .arg("-o") + .args(&["-i", cloudinit_file_path.as_str()]) + .args(&["-s", cloud_init_directory.join(x).to_str().unwrap(), "::"]) + .output() + .expect("Expect copying files to disk image to succeed"); + }); + + cloudinit_file_path + } + + fn prepare_files(&mut self, tmp_dir: &TempDir, network: &GuestNetworkConfig) { + let mut workload_path = dirs::home_dir().unwrap(); + workload_path.push("workloads"); + + let mut osdisk_base_path = workload_path; + osdisk_base_path.push(&self.image_name); + + let osdisk_path = String::from(tmp_dir.as_path().join("osdisk.img").to_str().unwrap()); + let cloudinit_path = self.prepare_cloudinit(tmp_dir, network); + + rate_limited_copy(osdisk_base_path, &osdisk_path) + .expect("copying of OS source disk image failed"); + + self.cloudinit_path = cloudinit_path; + self.osdisk_path = osdisk_path; + } + + fn disk(&self, disk_type: DiskType) -> Option { + match disk_type { + DiskType::OperatingSystem => Some(self.osdisk_path.clone()), + DiskType::CloudInit => Some(self.cloudinit_path.clone()), + } + } +} + +impl DiskConfig for WindowsDiskConfig { + fn prepare_cloudinit(&self, _tmp_dir: &TempDir, _network: &GuestNetworkConfig) -> String { + String::new() + } + + fn prepare_files(&mut self, tmp_dir: &TempDir, _network: &GuestNetworkConfig) { + let mut workload_path = dirs::home_dir().unwrap(); + workload_path.push("workloads"); + + let mut osdisk_path = workload_path; + osdisk_path.push(&self.image_name); + + let osdisk_blk_size = fs::metadata(osdisk_path) + .expect("Expect retrieving Windows image metadata") + .len() + >> 9; + + let snapshot_cow_path = + String::from(tmp_dir.as_path().join("snapshot_cow").to_str().unwrap()); + + // Create and truncate CoW file for device mapper + let cow_file_size: u64 = 1 << 30; + let cow_file_blk_size = cow_file_size >> 9; + let cow_file = std::fs::File::create(snapshot_cow_path.as_str()) + .expect("Expect creating CoW image to succeed"); + cow_file + .set_len(cow_file_size) + .expect("Expect truncating CoW image to succeed"); + + // losetup --find --show /tmp/snapshot_cow + let loopback_device = std::process::Command::new("losetup") + .arg("--find") + .arg("--show") + .arg(snapshot_cow_path.as_str()) + .output() + .expect("Expect creating loopback device from snapshot CoW image to succeed"); + + self.loopback_device = String::from_utf8_lossy(&loopback_device.stdout) + .trim() + .to_string(); + + let random_extension = tmp_dir.as_path().file_name().unwrap(); + let windows_snapshot_cow = format!( + "windows-snapshot-cow-{}", + random_extension.to_str().unwrap() + ); + + // dmsetup create windows-snapshot-cow-1 --table '0 2097152 linear /dev/loop1 0' + std::process::Command::new("dmsetup") + .arg("create") + .arg(windows_snapshot_cow.as_str()) + .args(&[ + "--table", + format!("0 {} linear {} 0", cow_file_blk_size, self.loopback_device).as_str(), + ]) + .output() + .expect("Expect creating Windows snapshot CoW with 'dmsetup' to succeed"); + + let windows_snapshot = format!("windows-snapshot-{}", random_extension.to_str().unwrap()); + + // dmsetup mknodes + std::process::Command::new("dmsetup") + .arg("mknodes") + .output() + .expect("Expect device mapper nodes to be ready"); + + // dmsetup create windows-snapshot-1 --table '0 41943040 snapshot /dev/mapper/windows-base /dev/mapper/windows-snapshot-cow-1 P 8' + std::process::Command::new("dmsetup") + .arg("create") + .arg(windows_snapshot.as_str()) + .args(&[ + "--table", + format!( + "0 {} snapshot /dev/mapper/windows-base /dev/mapper/{} P 8", + osdisk_blk_size, + windows_snapshot_cow.as_str() + ) + .as_str(), + ]) + .output() + .expect("Expect creating Windows snapshot with 'dmsetup' to succeed"); + + // dmsetup mknodes + std::process::Command::new("dmsetup") + .arg("mknodes") + .output() + .expect("Expect device mapper nodes to be ready"); + + self.osdisk_path = format!("/dev/mapper/{}", windows_snapshot); + self.windows_snapshot_cow = windows_snapshot_cow; + self.windows_snapshot = windows_snapshot; + } + + fn disk(&self, disk_type: DiskType) -> Option { + match disk_type { + DiskType::OperatingSystem => Some(self.osdisk_path.clone()), + DiskType::CloudInit => None, + } + } +} + +pub fn rate_limited_copy, Q: AsRef>(from: P, to: Q) -> io::Result { + for i in 0..10 { + let free_bytes = unsafe { + let mut stats = std::mem::MaybeUninit::zeroed(); + let fs_name = std::ffi::CString::new("/tmp").unwrap(); + libc::statvfs(fs_name.as_ptr(), stats.as_mut_ptr()); + + let free_blocks = stats.assume_init().f_bfree; + let block_size = stats.assume_init().f_bsize; + + free_blocks * block_size + }; + + // Make sure there is at least 6 GiB of space + if free_bytes < 6 << 30 { + eprintln!( + "Not enough space on disk ({}). Attempt {} of 10. Sleeping.", + free_bytes, i + ); + thread::sleep(std::time::Duration::new(60, 0)); + continue; + } + + match fs::copy(&from, &to) { + Err(e) => { + if let Some(errno) = e.raw_os_error() { + if errno == libc::ENOSPC { + eprintln!("Copy returned ENOSPC. Attempt {} of 10. Sleeping.", i); + thread::sleep(std::time::Duration::new(60, 0)); + continue; + } + } + return Err(e); + } + Ok(i) => return Ok(i), + } + } + Err(io::Error::last_os_error()) +} + +pub fn handle_child_output( + r: Result<(), std::boxed::Box>, + output: &std::process::Output, +) { + use std::os::unix::process::ExitStatusExt; + if r.is_ok() && output.status.success() { + return; + } + + match output.status.code() { + None => { + // Don't treat child.kill() as a problem + if output.status.signal() == Some(9) && r.is_ok() { + return; + } + + eprintln!( + "==== child killed by signal: {} ====", + output.status.signal().unwrap() + ); + } + Some(code) => { + eprintln!("\n\n==== child exit code: {} ====", code); + } + } + + eprintln!( + "\n\n==== Start child stdout ====\n\n{}\n\n==== End child stdout ====", + String::from_utf8_lossy(&output.stdout) + ); + eprintln!( + "\n\n==== Start child stderr ====\n\n{}\n\n==== End child stderr ====", + String::from_utf8_lossy(&output.stderr) + ); + + panic!("Test failed") +} + +#[derive(Debug)] +pub struct PasswordAuth { + pub username: String, + pub password: String, +} + +pub const DEFAULT_SSH_RETRIES: u8 = 6; +pub const DEFAULT_SSH_TIMEOUT: u8 = 10; + +#[derive(Debug)] +pub enum SshCommandError { + Connection(std::io::Error), + Handshake(ssh2::Error), + Authentication(ssh2::Error), + ChannelSession(ssh2::Error), + Command(ssh2::Error), +} + +pub fn ssh_command_ip_with_auth( + command: &str, + auth: &PasswordAuth, + ip: &str, + retries: u8, + timeout: u8, +) -> Result { + let mut s = String::new(); + + let mut counter = 0; + loop { + match (|| -> Result<(), SshCommandError> { + let tcp = + TcpStream::connect(format!("{}:22", ip)).map_err(SshCommandError::Connection)?; + let mut sess = Session::new().unwrap(); + sess.set_tcp_stream(tcp); + sess.handshake().map_err(SshCommandError::Handshake)?; + + sess.userauth_password(&auth.username, &auth.password) + .map_err(SshCommandError::Authentication)?; + assert!(sess.authenticated()); + + let mut channel = sess + .channel_session() + .map_err(SshCommandError::ChannelSession)?; + channel.exec(command).map_err(SshCommandError::Command)?; + + // Intentionally ignore these results here as their failure + // does not precipitate a repeat + let _ = channel.read_to_string(&mut s); + let _ = channel.close(); + let _ = channel.wait_close(); + Ok(()) + })() { + Ok(_) => break, + Err(e) => { + counter += 1; + if counter >= retries { + eprintln!( + "\n\n==== Start ssh command output (FAILED) ====\n\n\ + command=\"{}\"\n\ + auth=\"{:#?}\"\n\ + ip=\"{}\"\n\ + output=\"{}\"\n\ + error=\"{:?}\"\n\ + \n==== End ssh command outout ====\n\n", + command, auth, ip, s, e + ); + + return Err(e); + } + } + }; + thread::sleep(std::time::Duration::new((timeout * counter).into(), 0)); + } + Ok(s) +} + +pub fn ssh_command_ip( + command: &str, + ip: &str, + retries: u8, + timeout: u8, +) -> Result { + ssh_command_ip_with_auth( + command, + &PasswordAuth { + username: String::from("cloud"), + password: String::from("cloud123"), + }, + ip, + retries, + timeout, + ) +}