External/cloud-hypervisor
1
0
Fork 0
mirror of https://github.com/cloud-hypervisor/cloud-hypervisor.git synced 2024-07-04 17:02:36 +00:00
Code Issues Packages Projects Releases Wiki Activity

test_infra: Move struct Guest and struct GuestCommand from tests

Browse Source 
In this way, these structs can be reused for performance tests.

Signed-off-by: Bo Chen <chen.bo@intel.com>
This commit is contained in:
Bo Chen 2022-02-04 11:14:41 -08:00 committed by Sebastien Boeuf
parent a3a175216a
commit 7f987552ef
4 changed files with 600 additions and 590 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Cargo.lock generated
View File

@ -955,6 +955,7 @@ version = "0.1.0"
dependencies = [
"dirs 3.0.2",
"epoll",
"lazy_static",
"libc",
"ssh2",
"vmm-sys-util",

1
test_infra/Cargo.toml
View File

@ -7,6 +7,7 @@ edition = "2018"
[dependencies]
dirs = "3.0.1"
epoll = "4.3.1"
lazy_static= "1.4.0"
libc = "0.2.91"
ssh2 = "0.9.1"
vmm-sys-util = "0.9.0"

597
test_infra/src/lib.rs
View File

@ -3,7 +3,12 @@
// SPDX-License-Identifier: Apache-2.0
//
#[macro_use]
extern crate lazy_static;
use ssh2::Session;
use std::env;
use std::ffi::OsStr;
use std::fs;
use std::io;
use std::io::{Read, Write};
@ -11,12 +16,24 @@ use std::net::TcpListener;
use std::net::TcpStream;
use std::os::unix::io::AsRawFd;
use std::path::Path;
use std::process::{ExitStatus, Output};
use std::process::{Child, Command, ExitStatus, Output, Stdio};
use std::str::FromStr;
use std::sync::Mutex;
use std::thread;
use vmm_sys_util::tempdir::TempDir;
pub const DEFAULT_TCP_LISTENER_MESSAGE: &str = "booted";
#[derive(Debug)]
pub enum Error {
Parsing(std::num::ParseIntError),
SshCommand(SshCommandError),
WaitForBoot(WaitForBootError),
}
impl From<SshCommandError> for Error {
fn from(e: SshCommandError) -> Self {
Self::SshCommand(e)
}
}
pub struct GuestNetworkConfig {
pub guest_ip: String,
@ -31,6 +48,7 @@ pub struct GuestNetworkConfig {
pub tcp_listener_port: u16,
}
pub const DEFAULT_TCP_LISTENER_MESSAGE: &str = "booted";
pub const DEFAULT_TCP_LISTENER_PORT: u16 = 8000;
pub const DEFAULT_TCP_LISTENER_TIMEOUT: i32 = 80;
@ -605,3 +623,578 @@ pub fn exec_host_command_output(command: &str) -> Output {
.output()
.unwrap_or_else(|_| panic!("Expected '{}' to run", command))
}
pub const PIPE_SIZE: i32 = 32 << 20;
lazy_static! {
static ref NEXT_VM_ID: Mutex<u8> = Mutex::new(1);
}
pub struct Guest {
pub tmp_dir: TempDir,
pub disk_config: Box<dyn DiskConfig>,
pub network: GuestNetworkConfig,
}
// Safe to implement as we know we have no interior mutability
impl std::panic::RefUnwindSafe for Guest {}
impl Guest {
pub fn new_from_ip_range(mut disk_config: Box<dyn DiskConfig>, class: &str, id: u8) -> Self {
let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap();
let network = GuestNetworkConfig {
guest_ip: format!("{}.{}.2", class, id),
l2_guest_ip1: format!("{}.{}.3", class, id),
l2_guest_ip2: format!("{}.{}.4", class, id),
l2_guest_ip3: format!("{}.{}.5", class, id),
host_ip: format!("{}.{}.1", class, id),
guest_mac: format!("12:34:56:78:90:{:02x}", id),
l2_guest_mac1: format!("de:ad:be:ef:12:{:02x}", id),
l2_guest_mac2: format!("de:ad:be:ef:34:{:02x}", id),
l2_guest_mac3: format!("de:ad:be:ef:56:{:02x}", id),
tcp_listener_port: DEFAULT_TCP_LISTENER_PORT + id as u16,
};
disk_config.prepare_files(&tmp_dir, &network);
Guest {
tmp_dir,
disk_config,
network,
}
}
pub fn new(disk_config: Box<dyn DiskConfig>) -> Self {
let mut guard = NEXT_VM_ID.lock().unwrap();
let id = *guard;
*guard = id + 1;
Self::new_from_ip_range(disk_config, "192.168", id)
}
pub fn default_net_string(&self) -> String {
format!(
"tap=,mac={},ip={},mask=255.255.255.0",
self.network.guest_mac, self.network.host_ip
)
}
pub fn default_net_string_w_iommu(&self) -> String {
format!(
"tap=,mac={},ip={},mask=255.255.255.0,iommu=on",
self.network.guest_mac, self.network.host_ip
)
}
pub fn ssh_command(&self, command: &str) -> Result<String, SshCommandError> {
ssh_command_ip(
command,
&self.network.guest_ip,
DEFAULT_SSH_RETRIES,
DEFAULT_SSH_TIMEOUT,
)
}
#[cfg(target_arch = "x86_64")]
pub fn ssh_command_l1(&self, command: &str) -> Result<String, SshCommandError> {
ssh_command_ip(
command,
&self.network.guest_ip,
DEFAULT_SSH_RETRIES,
DEFAULT_SSH_TIMEOUT,
)
}
#[cfg(target_arch = "x86_64")]
pub fn ssh_command_l2_1(&self, command: &str) -> Result<String, SshCommandError> {
ssh_command_ip(
command,
&self.network.l2_guest_ip1,
DEFAULT_SSH_RETRIES,
DEFAULT_SSH_TIMEOUT,
)
}
#[cfg(target_arch = "x86_64")]
pub fn ssh_command_l2_2(&self, command: &str) -> Result<String, SshCommandError> {
ssh_command_ip(
command,
&self.network.l2_guest_ip2,
DEFAULT_SSH_RETRIES,
DEFAULT_SSH_TIMEOUT,
)
}
#[cfg(target_arch = "x86_64")]
pub fn ssh_command_l2_3(&self, command: &str) -> Result<String, SshCommandError> {
ssh_command_ip(
command,
&self.network.l2_guest_ip3,
DEFAULT_SSH_RETRIES,
DEFAULT_SSH_TIMEOUT,
)
}
pub fn api_create_body(
&self,
cpu_count: u8,
fw_path: &str,
_kernel_path: &str,
_kernel_cmd: &str,
) -> String {
#[cfg(all(target_arch = "x86_64", not(feature = "mshv")))]
format! {"{{\"cpus\":{{\"boot_vcpus\":{},\"max_vcpus\":{}}},\"kernel\":{{\"path\":\"{}\"}},\"cmdline\":{{\"args\": \"\"}},\"net\":[{{\"ip\":\"{}\", \"mask\":\"255.255.255.0\", \"mac\":\"{}\"}}], \"disks\":[{{\"path\":\"{}\"}}, {{\"path\":\"{}\"}}]}}",
cpu_count,
cpu_count,
fw_path,
self.network.host_ip,
self.network.guest_mac,
self.disk_config.disk(DiskType::OperatingSystem).unwrap().as_str(),
self.disk_config.disk(DiskType::CloudInit).unwrap().as_str(),
}
#[cfg(any(target_arch = "aarch64", feature = "mshv"))]
format! {"{{\"cpus\":{{\"boot_vcpus\":{},\"max_vcpus\":{}}},\"kernel\":{{\"path\":\"{}\"}},\"cmdline\":{{\"args\": \"{}\"}},\"net\":[{{\"ip\":\"{}\", \"mask\":\"255.255.255.0\", \"mac\":\"{}\"}}], \"disks\":[{{\"path\":\"{}\"}}, {{\"path\":\"{}\"}}]}}",
cpu_count,
cpu_count,
_kernel_path,
_kernel_cmd,
self.network.host_ip,
self.network.guest_mac,
self.disk_config.disk(DiskType::OperatingSystem).unwrap().as_str(),
self.disk_config.disk(DiskType::CloudInit).unwrap().as_str(),
}
}
pub fn get_cpu_count(&self) -> Result<u32, Error> {
self.ssh_command("grep -c processor /proc/cpuinfo")?
.trim()
.parse()
.map_err(Error::Parsing)
}
#[cfg(target_arch = "x86_64")]
pub fn get_initial_apicid(&self) -> Result<u32, Error> {
self.ssh_command("grep \"initial apicid\" /proc/cpuinfo | grep -o \"[0-9]*\"")?
.trim()
.parse()
.map_err(Error::Parsing)
}
pub fn get_total_memory(&self) -> Result<u32, Error> {
self.ssh_command("grep MemTotal /proc/meminfo | grep -o \"[0-9]*\"")?
.trim()
.parse()
.map_err(Error::Parsing)
}
#[cfg(target_arch = "x86_64")]
pub fn get_total_memory_l2(&self) -> Result<u32, Error> {
self.ssh_command_l2_1("grep MemTotal /proc/meminfo | grep -o \"[0-9]*\"")?
.trim()
.parse()
.map_err(Error::Parsing)
}
pub fn get_numa_node_memory(&self, node_id: usize) -> Result<u32, Error> {
self.ssh_command(
format!(
"grep MemTotal /sys/devices/system/node/node{}/meminfo \
| cut -d \":\" -f 2 | grep -o \"[0-9]*\"",
node_id
)
.as_str(),
)?
.trim()
.parse()
.map_err(Error::Parsing)
}
pub fn wait_vm_boot(&self, custom_timeout: Option<i32>) -> Result<(), Error> {
self.network
.wait_vm_boot(custom_timeout)
.map_err(Error::WaitForBoot)
}
pub fn check_numa_node_cpus(&self, node_id: usize, cpus: Vec<usize>) -> Result<(), Error> {
for cpu in cpus.iter() {
let cmd = format!(
"[ -d \"/sys/devices/system/node/node{}/cpu{}\" ]",
node_id, cpu
);
self.ssh_command(cmd.as_str())?;
}
Ok(())
}
pub fn check_numa_node_distances(
&self,
node_id: usize,
distances: &str,
) -> Result<bool, Error> {
let cmd = format!("cat /sys/devices/system/node/node{}/distance", node_id);
if self.ssh_command(cmd.as_str())?.trim() == distances {
Ok(true)
} else {
Ok(false)
}
}
pub fn check_numa_common(
&self,
mem_ref: Option<&[u32]>,
node_ref: Option<&[Vec<usize>]>,
distance_ref: Option<&[&str]>,
) {
if let Some(mem_ref) = mem_ref {
// Check each NUMA node has been assigned the right amount of
// memory.
for (i, &m) in mem_ref.iter().enumerate() {
assert!(self.get_numa_node_memory(i).unwrap_or_default() > m);
}
}
if let Some(node_ref) = node_ref {
// Check each NUMA node has been assigned the right CPUs set.
for (i, n) in node_ref.iter().enumerate() {
self.check_numa_node_cpus(i, n.clone()).unwrap();
}
}
if let Some(distance_ref) = distance_ref {
// Check each NUMA node has been assigned the right distances.
for (i, &d) in distance_ref.iter().enumerate() {
assert!(self.check_numa_node_distances(i, d).unwrap());
}
}
}
#[cfg(target_arch = "x86_64")]
pub fn check_sgx_support(&self) -> Result<(), Error> {
self.ssh_command(
"cpuid -l 0x7 -s 0 | tr -s [:space:] | grep -q 'SGX: \
Software Guard Extensions supported = true'",
)?;
self.ssh_command(
"cpuid -l 0x7 -s 0 | tr -s [:space:] | grep -q 'SGX_LC: \
SGX launch config supported = true'",
)?;
self.ssh_command(
"cpuid -l 0x12 -s 0 | tr -s [:space:] | grep -q 'SGX1 \
supported = true'",
)?;
Ok(())
}
pub fn get_entropy(&self) -> Result<u32, Error> {
self.ssh_command("cat /proc/sys/kernel/random/entropy_avail")?
.trim()
.parse()
.map_err(Error::Parsing)
}
pub fn get_pci_bridge_class(&self) -> Result<String, Error> {
Ok(self
.ssh_command("cat /sys/bus/pci/devices/0000:00:00.0/class")?
.trim()
.to_string())
}
pub fn get_pci_device_ids(&self) -> Result<String, Error> {
Ok(self
.ssh_command("cat /sys/bus/pci/devices/*/device")?
.trim()
.to_string())
}
pub fn get_pci_vendor_ids(&self) -> Result<String, Error> {
Ok(self
.ssh_command("cat /sys/bus/pci/devices/*/vendor")?
.trim()
.to_string())
}
pub fn does_device_vendor_pair_match(
&self,
device_id: &str,
vendor_id: &str,
) -> Result<bool, Error> {
// We are checking if console device's device id and vendor id pair matches
let devices = self.get_pci_device_ids()?;
let devices: Vec<&str> = devices.split('\n').collect();
let vendors = self.get_pci_vendor_ids()?;
let vendors: Vec<&str> = vendors.split('\n').collect();
for (index, d_id) in devices.iter().enumerate() {
if *d_id == device_id {
if let Some(v_id) = vendors.get(index) {
if *v_id == vendor_id {
return Ok(true);
}
}
}
}
Ok(false)
}
pub fn valid_virtio_fs_cache_size(
&self,
dax: bool,
cache_size: Option<u64>,
) -> Result<bool, Error> {
// SHM region is called different things depending on kernel
let shm_region = self
.ssh_command("sudo grep 'virtio[0-9]\\|virtio-pci-shm' /proc/iomem || true")?
.trim()
.to_string();
if shm_region.is_empty() {
return Ok(!dax);
}
// From this point, the region is not empty, hence it is an error
// if DAX is off.
if !dax {
return Ok(false);
}
let cache = if let Some(cache) = cache_size {
cache
} else {
// 8Gib by default
0x0002_0000_0000
};
let args: Vec<&str> = shm_region.split(':').collect();
if args.is_empty() {
return Ok(false);
}
let args: Vec<&str> = args[0].trim().split('-').collect();
if args.len() != 2 {
return Ok(false);
}
let start_addr = u64::from_str_radix(args[0], 16).map_err(Error::Parsing)?;
let end_addr = u64::from_str_radix(args[1], 16).map_err(Error::Parsing)?;
Ok(cache == (end_addr - start_addr + 1))
}
pub fn check_vsock(&self, socket: &str) {
// Listen from guest on vsock CID=3 PORT=16
// SOCKET-LISTEN:<domain>:<protocol>:<local-address>
let guest_ip = self.network.guest_ip.clone();
let listen_socat = thread::spawn(move || {
ssh_command_ip("sudo socat - SOCKET-LISTEN:40:0:x00x00x10x00x00x00x03x00x00x00x00x00x00x00 > vsock_log", &guest_ip, DEFAULT_SSH_RETRIES, DEFAULT_SSH_TIMEOUT).unwrap();
});
// Make sure socat is listening, which might take a few second on slow systems
thread::sleep(std::time::Duration::new(10, 0));
// Write something to vsock from the host
assert!(exec_host_command_status(&format!(
"echo -e \"CONNECT 16\\nHelloWorld!\" | socat - UNIX-CONNECT:{}",
socket
))
.success());
// Wait for the thread to terminate.
listen_socat.join().unwrap();
assert_eq!(
self.ssh_command("cat vsock_log").unwrap().trim(),
"HelloWorld!"
);
}
#[cfg(target_arch = "x86_64")]
pub fn check_nvidia_gpu(&self) {
// Run CUDA sample to validate it can find the device
let device_query_result = self
.ssh_command("sudo /root/NVIDIA_CUDA-11.3_Samples/bin/x86_64/linux/release/deviceQuery")
.unwrap();
assert!(device_query_result.contains("Detected 1 CUDA Capable device"));
assert!(device_query_result.contains("Device 0: \"NVIDIA Tesla T4\""));
assert!(device_query_result.contains("Result = PASS"));
// Run NVIDIA DCGM Diagnostics to validate the device is functional
self.ssh_command("sudo nv-hostengine").unwrap();
assert!(self
.ssh_command("sudo dcgmi discovery -l")
.unwrap()
.contains("Name: NVIDIA Tesla T4"));
assert_eq!(
self.ssh_command("sudo dcgmi diag -r 'diagnostic' | grep Pass | wc -l")
.unwrap()
.trim(),
"10"
);
}
pub fn reboot_linux(&self, current_reboot_count: u32, custom_timeout: Option<i32>) {
let list_boots_cmd = "sudo journalctl --list-boots | wc -l";
let boot_count = self
.ssh_command(list_boots_cmd)
.unwrap()
.trim()
.parse::<u32>()
.unwrap_or_default();
assert_eq!(boot_count, current_reboot_count + 1);
self.ssh_command("sudo reboot").unwrap();
self.wait_vm_boot(custom_timeout).unwrap();
let boot_count = self
.ssh_command(list_boots_cmd)
.unwrap()
.trim()
.parse::<u32>()
.unwrap_or_default();
assert_eq!(boot_count, current_reboot_count + 2);
}
pub fn enable_memory_hotplug(&self) {
self.ssh_command("echo online | sudo tee /sys/devices/system/memory/auto_online_blocks")
.unwrap();
}
pub fn check_devices_common(&self, socket: Option<&String>, console_text: Option<&String>) {
// Check block devices are readable
self.ssh_command("sudo dd if=/dev/vda of=/dev/null bs=1M iflag=direct count=1024")
.unwrap();
self.ssh_command("sudo dd if=/dev/vdb of=/dev/null bs=1M iflag=direct count=8")
.unwrap();
// Check if the rng device is readable
self.ssh_command("sudo head -c 1000 /dev/hwrng > /dev/null")
.unwrap();
// Check vsock
if let Some(socket) = socket {
self.check_vsock(socket.as_str());
}
// Check if the console is usable
if let Some(console_text) = console_text {
let console_cmd = format!("echo {} | sudo tee /dev/hvc0", console_text);
self.ssh_command(&console_cmd).unwrap();
}
// The net device is 'automatically' exercised through the above 'ssh' commands
}
}
pub struct GuestCommand<'a> {
command: Command,
guest: &'a Guest,
capture_output: bool,
}
impl<'a> GuestCommand<'a> {
pub fn new(guest: &'a Guest) -> Self {
Self::new_with_binary_name(guest, "cloud-hypervisor")
}
pub fn new_with_binary_name(guest: &'a Guest, binary_name: &str) -> Self {
Self {
command: Command::new(clh_command(binary_name)),
guest,
capture_output: false,
}
}
pub fn capture_output(&mut self) -> &mut Self {
self.capture_output = true;
self
}
pub fn spawn(&mut self) -> io::Result<Child> {
println!(
"\n\n==== Start cloud-hypervisor command-line ====\n\n\
{:?}\n\
\n==== End cloud-hypervisor command-line ====\n\n",
self.command
);
if self.capture_output {
let child = self
.command
.arg("-v")
.stderr(Stdio::piped())
.stdout(Stdio::piped())
.spawn()
.unwrap();
let fd = child.stdout.as_ref().unwrap().as_raw_fd();
let pipesize = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) };
let fd = child.stderr.as_ref().unwrap().as_raw_fd();
let pipesize1 = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) };
if pipesize >= PIPE_SIZE && pipesize1 >= PIPE_SIZE {
Ok(child)
} else {
Err(std::io::Error::new(
std::io::ErrorKind::Other,
"resizing pipe w/ 'fnctl' failed!",
))
}
} else {
self.command.arg("-v").spawn()
}
}
pub fn args<I, S>(&mut self, args: I) -> &mut Self
where
I: IntoIterator<Item = S>,
S: AsRef<OsStr>,
{
self.command.args(args);
self
}
pub fn default_disks(&mut self) -> &mut Self {
if self.guest.disk_config.disk(DiskType::CloudInit).is_some() {
self.args(&[
"--disk",
format!(
"path={}",
self.guest
.disk_config
.disk(DiskType::OperatingSystem)
.unwrap()
)
.as_str(),
format!(
"path={}",
self.guest.disk_config.disk(DiskType::CloudInit).unwrap()
)
.as_str(),
])
} else {
self.args(&[
"--disk",
format!(
"path={}",
self.guest
.disk_config
.disk(DiskType::OperatingSystem)
.unwrap()
)
.as_str(),
])
}
}
pub fn default_net(&mut self) -> &mut Self {
self.args(&["--net", self.guest.default_net_string().as_str()])
}
}
pub fn clh_command(cmd: &str) -> String {
env::var("BUILD_TARGET").map_or(
format!("target/x86_64-unknown-linux-gnu/release/{}", cmd),
|target| format!("target/{}/release/{}", target, cmd),
)
}

591
tests/integration.rs
View File

@ -15,8 +15,6 @@ extern crate test_infra;
use net_util::MacAddr;
use std::collections::HashMap;
use std::env;
use std::ffi::OsStr;
use std::fs;
use std::io;
use std::io::BufRead;
@ -27,27 +25,15 @@ use std::os::unix::io::AsRawFd;
use std::path::PathBuf;
use std::process::{Child, Command, Stdio};
use std::string::String;
use std::sync::mpsc;
use std::sync::mpsc::Receiver;
use std::sync::{mpsc, Mutex};
#[cfg(target_arch = "x86_64")]
use std::sync::Mutex;
use std::thread;
use test_infra::*;
use vmm_sys_util::{tempdir::TempDir, tempfile::TempFile};
#[cfg_attr(target_arch = "aarch64", allow(unused_imports))]
use wait_timeout::ChildExt;
lazy_static! {
static ref NEXT_VM_ID: Mutex<u8> = Mutex::new(1);
}
struct Guest {
tmp_dir: TempDir,
disk_config: Box<dyn DiskConfig>,
network: GuestNetworkConfig,
}
// Safe to implement as we know we have no interior mutability
impl std::panic::RefUnwindSafe for Guest {}
#[cfg(target_arch = "x86_64")]
mod x86_64 {
pub const BIONIC_IMAGE_NAME: &str = "bionic-server-cloudimg-amd64.raw";
@ -84,17 +70,8 @@ use aarch64::*;
const DIRECT_KERNEL_BOOT_CMDLINE: &str =
"root=/dev/vda1 console=hvc0 rw systemd.journald.forward_to_console=1";
const PIPE_SIZE: i32 = 32 << 20;
const CONSOLE_TEST_STRING: &str = "Started OpenBSD Secure Shell server";
fn clh_command(cmd: &str) -> String {
env::var("BUILD_TARGET").map_or(
format!("target/x86_64-unknown-linux-gnu/release/{}", cmd),
|target| format!("target/{}/release/{}", target, cmd),
)
}
fn prepare_virtiofsd(
tmp_dir: &TempDir,
shared_dir: &str,
@ -454,19 +431,6 @@ fn setup_ovs_dpdk_guests(guest1: &Guest, guest2: &Guest, api_socket: &str) -> (C
(child1, child2)
}
#[derive(Debug)]
enum Error {
Parsing(std::num::ParseIntError),
SshCommand(SshCommandError),
WaitForBoot(WaitForBootError),
}
impl From<SshCommandError> for Error {
fn from(e: SshCommandError) -> Self {
Self::SshCommand(e)
}
}
enum FwType {
Ovmf,
RustHypervisorFirmware,
@ -490,555 +454,6 @@ fn fw_path(fw_type: FwType) -> String {
fw_path.to_str().unwrap().to_string()
}
impl Guest {
fn new_from_ip_range(mut disk_config: Box<dyn DiskConfig>, class: &str, id: u8) -> Self {
let tmp_dir = TempDir::new_with_prefix("/tmp/ch").unwrap();
let network = GuestNetworkConfig {
guest_ip: format!("{}.{}.2", class, id),
l2_guest_ip1: format!("{}.{}.3", class, id),
l2_guest_ip2: format!("{}.{}.4", class, id),
l2_guest_ip3: format!("{}.{}.5", class, id),
host_ip: format!("{}.{}.1", class, id),
guest_mac: format!("12:34:56:78:90:{:02x}", id),
l2_guest_mac1: format!("de:ad:be:ef:12:{:02x}", id),
l2_guest_mac2: format!("de:ad:be:ef:34:{:02x}", id),
l2_guest_mac3: format!("de:ad:be:ef:56:{:02x}", id),
tcp_listener_port: DEFAULT_TCP_LISTENER_PORT + id as u16,
};
disk_config.prepare_files(&tmp_dir, &network);
Guest {
tmp_dir,
disk_config,
network,
}
}
fn new(disk_config: Box<dyn DiskConfig>) -> Self {
let mut guard = NEXT_VM_ID.lock().unwrap();
let id = *guard;
*guard = id + 1;
Self::new_from_ip_range(disk_config, "192.168", id)
}
fn default_net_string(&self) -> String {
format!(
"tap=,mac={},ip={},mask=255.255.255.0",
self.network.guest_mac, self.network.host_ip
)
}
fn default_net_string_w_iommu(&self) -> String {
format!(
"tap=,mac={},ip={},mask=255.255.255.0,iommu=on",
self.network.guest_mac, self.network.host_ip
)
}
fn ssh_command(&self, command: &str) -> Result<String, SshCommandError> {
ssh_command_ip(
command,
&self.network.guest_ip,
DEFAULT_SSH_RETRIES,
DEFAULT_SSH_TIMEOUT,
)
}
#[cfg(target_arch = "x86_64")]
fn ssh_command_l1(&self, command: &str) -> Result<String, SshCommandError> {
ssh_command_ip(
command,
&self.network.guest_ip,
DEFAULT_SSH_RETRIES,
DEFAULT_SSH_TIMEOUT,
)
}
#[cfg(target_arch = "x86_64")]
fn ssh_command_l2_1(&self, command: &str) -> Result<String, SshCommandError> {
ssh_command_ip(
command,
&self.network.l2_guest_ip1,
DEFAULT_SSH_RETRIES,
DEFAULT_SSH_TIMEOUT,
)
}
#[cfg(target_arch = "x86_64")]
fn ssh_command_l2_2(&self, command: &str) -> Result<String, SshCommandError> {
ssh_command_ip(
command,
&self.network.l2_guest_ip2,
DEFAULT_SSH_RETRIES,
DEFAULT_SSH_TIMEOUT,
)
}
#[cfg(target_arch = "x86_64")]
fn ssh_command_l2_3(&self, command: &str) -> Result<String, SshCommandError> {
ssh_command_ip(
command,
&self.network.l2_guest_ip3,
DEFAULT_SSH_RETRIES,
DEFAULT_SSH_TIMEOUT,
)
}
fn api_create_body(
&self,
cpu_count: u8,
fw_path: &str,
_kernel_path: &str,
_kernel_cmd: &str,
) -> String {
#[cfg(all(target_arch = "x86_64", not(feature = "mshv")))]
format! {"{{\"cpus\":{{\"boot_vcpus\":{},\"max_vcpus\":{}}},\"kernel\":{{\"path\":\"{}\"}},\"cmdline\":{{\"args\": \"\"}},\"net\":[{{\"ip\":\"{}\", \"mask\":\"255.255.255.0\", \"mac\":\"{}\"}}], \"disks\":[{{\"path\":\"{}\"}}, {{\"path\":\"{}\"}}]}}",
cpu_count,
cpu_count,
fw_path,
self.network.host_ip,
self.network.guest_mac,
self.disk_config.disk(DiskType::OperatingSystem).unwrap().as_str(),
self.disk_config.disk(DiskType::CloudInit).unwrap().as_str(),
}
#[cfg(any(target_arch = "aarch64", feature = "mshv"))]
format! {"{{\"cpus\":{{\"boot_vcpus\":{},\"max_vcpus\":{}}},\"kernel\":{{\"path\":\"{}\"}},\"cmdline\":{{\"args\": \"{}\"}},\"net\":[{{\"ip\":\"{}\", \"mask\":\"255.255.255.0\", \"mac\":\"{}\"}}], \"disks\":[{{\"path\":\"{}\"}}, {{\"path\":\"{}\"}}]}}",
cpu_count,
cpu_count,
_kernel_path,
_kernel_cmd,
self.network.host_ip,
self.network.guest_mac,
self.disk_config.disk(DiskType::OperatingSystem).unwrap().as_str(),
self.disk_config.disk(DiskType::CloudInit).unwrap().as_str(),
}
}
fn get_cpu_count(&self) -> Result<u32, Error> {
self.ssh_command("grep -c processor /proc/cpuinfo")?
.trim()
.parse()
.map_err(Error::Parsing)
}
#[cfg(target_arch = "x86_64")]
fn get_initial_apicid(&self) -> Result<u32, Error> {
self.ssh_command("grep \"initial apicid\" /proc/cpuinfo | grep -o \"[0-9]*\"")?
.trim()
.parse()
.map_err(Error::Parsing)
}
fn get_total_memory(&self) -> Result<u32, Error> {
self.ssh_command("grep MemTotal /proc/meminfo | grep -o \"[0-9]*\"")?
.trim()
.parse()
.map_err(Error::Parsing)
}
#[cfg(target_arch = "x86_64")]
fn get_total_memory_l2(&self) -> Result<u32, Error> {
self.ssh_command_l2_1("grep MemTotal /proc/meminfo | grep -o \"[0-9]*\"")?
.trim()
.parse()
.map_err(Error::Parsing)
}
fn get_numa_node_memory(&self, node_id: usize) -> Result<u32, Error> {
self.ssh_command(
format!(
"grep MemTotal /sys/devices/system/node/node{}/meminfo \
| cut -d \":\" -f 2 | grep -o \"[0-9]*\"",
node_id
)
.as_str(),
)?
.trim()
.parse()
.map_err(Error::Parsing)
}
fn wait_vm_boot(&self, custom_timeout: Option<i32>) -> Result<(), Error> {
self.network
.wait_vm_boot(custom_timeout)
.map_err(Error::WaitForBoot)
}
fn check_numa_node_cpus(&self, node_id: usize, cpus: Vec<usize>) -> Result<(), Error> {
for cpu in cpus.iter() {
let cmd = format!(
"[ -d \"/sys/devices/system/node/node{}/cpu{}\" ]",
node_id, cpu
);
self.ssh_command(cmd.as_str())?;
}
Ok(())
}
fn check_numa_node_distances(&self, node_id: usize, distances: &str) -> Result<bool, Error> {
let cmd = format!("cat /sys/devices/system/node/node{}/distance", node_id);
if self.ssh_command(cmd.as_str())?.trim() == distances {
Ok(true)
} else {
Ok(false)
}
}
fn check_numa_common(
&self,
mem_ref: Option<&[u32]>,
node_ref: Option<&[Vec<usize>]>,
distance_ref: Option<&[&str]>,
) {
if let Some(mem_ref) = mem_ref {
// Check each NUMA node has been assigned the right amount of
// memory.
for (i, &m) in mem_ref.iter().enumerate() {
assert!(self.get_numa_node_memory(i).unwrap_or_default() > m);
}
}
if let Some(node_ref) = node_ref {
// Check each NUMA node has been assigned the right CPUs set.
for (i, n) in node_ref.iter().enumerate() {
self.check_numa_node_cpus(i, n.clone()).unwrap();
}
}
if let Some(distance_ref) = distance_ref {
// Check each NUMA node has been assigned the right distances.
for (i, &d) in distance_ref.iter().enumerate() {
assert!(self.check_numa_node_distances(i, d).unwrap());
}
}
}
#[cfg(target_arch = "x86_64")]
fn check_sgx_support(&self) -> Result<(), Error> {
self.ssh_command(
"cpuid -l 0x7 -s 0 | tr -s [:space:] | grep -q 'SGX: \
Software Guard Extensions supported = true'",
)?;
self.ssh_command(
"cpuid -l 0x7 -s 0 | tr -s [:space:] | grep -q 'SGX_LC: \
SGX launch config supported = true'",
)?;
self.ssh_command(
"cpuid -l 0x12 -s 0 | tr -s [:space:] | grep -q 'SGX1 \
supported = true'",
)?;
Ok(())
}
fn get_entropy(&self) -> Result<u32, Error> {
self.ssh_command("cat /proc/sys/kernel/random/entropy_avail")?
.trim()
.parse()
.map_err(Error::Parsing)
}
fn get_pci_bridge_class(&self) -> Result<String, Error> {
Ok(self
.ssh_command("cat /sys/bus/pci/devices/0000:00:00.0/class")?
.trim()
.to_string())
}
fn get_pci_device_ids(&self) -> Result<String, Error> {
Ok(self
.ssh_command("cat /sys/bus/pci/devices/*/device")?
.trim()
.to_string())
}
fn get_pci_vendor_ids(&self) -> Result<String, Error> {
Ok(self
.ssh_command("cat /sys/bus/pci/devices/*/vendor")?
.trim()
.to_string())
}
fn does_device_vendor_pair_match(
&self,
device_id: &str,
vendor_id: &str,
) -> Result<bool, Error> {
// We are checking if console device's device id and vendor id pair matches
let devices = self.get_pci_device_ids()?;
let devices: Vec<&str> = devices.split('\n').collect();
let vendors = self.get_pci_vendor_ids()?;
let vendors: Vec<&str> = vendors.split('\n').collect();
for (index, d_id) in devices.iter().enumerate() {
if *d_id == device_id {
if let Some(v_id) = vendors.get(index) {
if *v_id == vendor_id {
return Ok(true);
}
}
}
}
Ok(false)
}
fn valid_virtio_fs_cache_size(
&self,
dax: bool,
cache_size: Option<u64>,
) -> Result<bool, Error> {
// SHM region is called different things depending on kernel
let shm_region = self
.ssh_command("sudo grep 'virtio[0-9]\\|virtio-pci-shm' /proc/iomem || true")?
.trim()
.to_string();
if shm_region.is_empty() {
return Ok(!dax);
}
// From this point, the region is not empty, hence it is an error
// if DAX is off.
if !dax {
return Ok(false);
}
let cache = if let Some(cache) = cache_size {
cache
} else {
// 8Gib by default
0x0002_0000_0000
};
let args: Vec<&str> = shm_region.split(':').collect();
if args.is_empty() {
return Ok(false);
}
let args: Vec<&str> = args[0].trim().split('-').collect();
if args.len() != 2 {
return Ok(false);
}
let start_addr = u64::from_str_radix(args[0], 16).map_err(Error::Parsing)?;
let end_addr = u64::from_str_radix(args[1], 16).map_err(Error::Parsing)?;
Ok(cache == (end_addr - start_addr + 1))
}
fn check_vsock(&self, socket: &str) {
// Listen from guest on vsock CID=3 PORT=16
// SOCKET-LISTEN:<domain>:<protocol>:<local-address>
let guest_ip = self.network.guest_ip.clone();
let listen_socat = thread::spawn(move || {
ssh_command_ip("sudo socat - SOCKET-LISTEN:40:0:x00x00x10x00x00x00x03x00x00x00x00x00x00x00 > vsock_log", &guest_ip, DEFAULT_SSH_RETRIES, DEFAULT_SSH_TIMEOUT).unwrap();
});
// Make sure socat is listening, which might take a few second on slow systems
thread::sleep(std::time::Duration::new(10, 0));
// Write something to vsock from the host
assert!(exec_host_command_status(&format!(
"echo -e \"CONNECT 16\\nHelloWorld!\" | socat - UNIX-CONNECT:{}",
socket
))
.success());
// Wait for the thread to terminate.
listen_socat.join().unwrap();
assert_eq!(
self.ssh_command("cat vsock_log").unwrap().trim(),
"HelloWorld!"
);
}
#[cfg(target_arch = "x86_64")]
fn check_nvidia_gpu(&self) {
// Run CUDA sample to validate it can find the device
let device_query_result = self
.ssh_command("sudo /root/NVIDIA_CUDA-11.3_Samples/bin/x86_64/linux/release/deviceQuery")
.unwrap();
assert!(device_query_result.contains("Detected 1 CUDA Capable device"));
assert!(device_query_result.contains("Device 0: \"NVIDIA Tesla T4\""));
assert!(device_query_result.contains("Result = PASS"));
// Run NVIDIA DCGM Diagnostics to validate the device is functional
self.ssh_command("sudo nv-hostengine").unwrap();
assert!(self
.ssh_command("sudo dcgmi discovery -l")
.unwrap()
.contains("Name: NVIDIA Tesla T4"));
assert_eq!(
self.ssh_command("sudo dcgmi diag -r 'diagnostic' | grep Pass | wc -l")
.unwrap()
.trim(),
"10"
);
}
fn reboot_linux(&self, current_reboot_count: u32, custom_timeout: Option<i32>) {
let list_boots_cmd = "sudo journalctl --list-boots | wc -l";
let boot_count = self
.ssh_command(list_boots_cmd)
.unwrap()
.trim()
.parse::<u32>()
.unwrap_or_default();
assert_eq!(boot_count, current_reboot_count + 1);
self.ssh_command("sudo reboot").unwrap();
self.wait_vm_boot(custom_timeout).unwrap();
let boot_count = self
.ssh_command(list_boots_cmd)
.unwrap()
.trim()
.parse::<u32>()
.unwrap_or_default();
assert_eq!(boot_count, current_reboot_count + 2);
}
fn enable_memory_hotplug(&self) {
self.ssh_command("echo online | sudo tee /sys/devices/system/memory/auto_online_blocks")
.unwrap();
}
fn check_devices_common(&self, socket: Option<&String>, console_text: Option<&String>) {
// Check block devices are readable
self.ssh_command("sudo dd if=/dev/vda of=/dev/null bs=1M iflag=direct count=1024")
.unwrap();
self.ssh_command("sudo dd if=/dev/vdb of=/dev/null bs=1M iflag=direct count=8")
.unwrap();
// Check if the rng device is readable
self.ssh_command("sudo head -c 1000 /dev/hwrng > /dev/null")
.unwrap();
// Check vsock
if let Some(socket) = socket {
self.check_vsock(socket.as_str());
}
// Check if the console is usable
if let Some(console_text) = console_text {
let console_cmd = format!("echo {} | sudo tee /dev/hvc0", console_text);
self.ssh_command(&console_cmd).unwrap();
}
// The net device is 'automatically' exercised through the above 'ssh' commands
}
}
struct GuestCommand<'a> {
command: Command,
guest: &'a Guest,
capture_output: bool,
}
impl<'a> GuestCommand<'a> {
fn new(guest: &'a Guest) -> Self {
Self::new_with_binary_name(guest, "cloud-hypervisor")
}
fn new_with_binary_name(guest: &'a Guest, binary_name: &str) -> Self {
Self {
command: Command::new(clh_command(binary_name)),
guest,
capture_output: false,
}
}
fn capture_output(&mut self) -> &mut Self {
self.capture_output = true;
self
}
fn spawn(&mut self) -> io::Result<Child> {
println!(
"\n\n==== Start cloud-hypervisor command-line ====\n\n\
{:?}\n\
\n==== End cloud-hypervisor command-line ====\n\n",
self.command
);
if self.capture_output {
let child = self
.command
.arg("-v")
.stderr(Stdio::piped())
.stdout(Stdio::piped())
.spawn()
.unwrap();
let fd = child.stdout.as_ref().unwrap().as_raw_fd();
let pipesize = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) };
let fd = child.stderr.as_ref().unwrap().as_raw_fd();
let pipesize1 = unsafe { libc::fcntl(fd, libc::F_SETPIPE_SZ, PIPE_SIZE) };
if pipesize >= PIPE_SIZE && pipesize1 >= PIPE_SIZE {
Ok(child)
} else {
Err(std::io::Error::new(
std::io::ErrorKind::Other,
"resizing pipe w/ 'fnctl' failed!",
))
}
} else {
self.command.arg("-v").spawn()
}
}
fn args<I, S>(&mut self, args: I) -> &mut Self
where
I: IntoIterator<Item = S>,
S: AsRef<OsStr>,
{
self.command.args(args);
self
}
fn default_disks(&mut self) -> &mut Self {
if self.guest.disk_config.disk(DiskType::CloudInit).is_some() {
self.args(&[
"--disk",
format!(
"path={}",
self.guest
.disk_config
.disk(DiskType::OperatingSystem)
.unwrap()
)
.as_str(),
format!(
"path={}",
self.guest.disk_config.disk(DiskType::CloudInit).unwrap()
)
.as_str(),
])
} else {
self.args(&[
"--disk",
format!(
"path={}",
self.guest
.disk_config
.disk(DiskType::OperatingSystem)
.unwrap()
)
.as_str(),
])
}
}
fn default_net(&mut self) -> &mut Self {
self.args(&["--net", self.guest.default_net_string().as_str()])
}
}
fn test_cpu_topology(threads_per_core: u8, cores_per_package: u8, packages: u8, use_fw: bool) {
let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string());
let guest = Guest::new(Box::new(focal));