From 5ae3144f5b4fed9effa809e7f78f69567e75874c Mon Sep 17 00:00:00 2001 From: Samuel Ortiz Date: Fri, 19 Jul 2019 10:26:13 +0200 Subject: [PATCH] tests: Add VFIO integration test The VFIO integration test first boots a QEMU guest and then assigns the QEMU virtio-pci networking device into a nested cloud-hypervisor guest. We then check that we can ssh into the nested guest and verify that it's running with the right kernel command line. Signed-off-by: Samuel Ortiz --- Jenkinsfile | 3 +- scripts/run_integration_tests.sh | 43 +++++ src/main.rs | 161 ++++++++++++++++-- .../cloud-init/openstack/latest/user_data | 41 ++++- 4 files changed, 235 insertions(+), 13 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index f6966cd7b..0468ad3a6 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -5,7 +5,7 @@ stage ("Builds") { } stage ('Install system packages') { sh "sudo DEBIAN_FRONTEND=noninteractive apt-get install -yq build-essential mtools libssl-dev pkg-config" - sh "sudo apt-get install -yq flex bison libelf-dev qemu-utils" + sh "sudo apt-get install -yq flex bison libelf-dev qemu-utils qemu-system" } stage ('Install Rust') { sh "nohup curl https://sh.rustup.rs -sSf | sh -s -- -y" @@ -16,7 +16,6 @@ stage ("Builds") { } stage ('Run integration tests') { sh "sudo mount -t tmpfs tmpfs /tmp" - sh "sudo chmod a+rw /dev/kvm" sh "scripts/run_integration_tests.sh" } } diff --git a/scripts/run_integration_tests.sh b/scripts/run_integration_tests.sh index 41b85cf9d..b733bac99 100755 --- a/scripts/run_integration_tests.sh +++ b/scripts/run_integration_tests.sh @@ -14,6 +14,14 @@ if [ ! -f "$FW" ]; then popd fi +OVMF_URL="https://cdn.download.clearlinux.org/image/OVMF.fd" +OVMF="$WORKLOADS_DIR/OVMF.fd" +if [ ! -f "$OVMF" ]; then + pushd $WORKLOADS_DIR + wget --quiet $OVMF_URL + popd +fi + OS_IMAGE_NAME="clear-29810-cloud.img" OS_IMAGE_URL="https://cloudhypervisorstorage.blob.core.windows.net/images/$OS_IMAGE_NAME.xz" OS_IMAGE="$WORKLOADS_DIR/$OS_IMAGE_NAME" @@ -86,7 +94,42 @@ if [ ! -d "$SHARED_DIR" ]; then echo "bar" > "$SHARED_DIR/file3" fi +VFIO_DIR="$WORKLOADS_DIR/vfio" +if [ ! -d "$VFIO_DIR" ]; then + mkdir -p $VFIO_DIR + cp $OS_IMAGE $VFIO_DIR + cp $FW $VFIO_DIR + cp $VMLINUX_IMAGE $VFIO_DIR +fi + +# VFIO test network setup. +# We reserve a different IP class for it: 172.16.0.0/24. +sudo ip link add name vfio-br0 type bridge +sudo ip link set vfio-br0 up +sudo ip addr add 172.16.0.1/24 dev vfio-br0 + +sudo ip tuntap add vfio-tap0 mode tap +sudo ip link set vfio-tap0 master vfio-br0 +sudo ip link set vfio-tap0 up + +sudo ip tuntap add vfio-tap1 mode tap +sudo ip link set vfio-tap1 master vfio-br0 +sudo ip link set vfio-tap1 up + cargo build sudo setcap cap_net_admin+ep target/debug/cloud-hypervisor +# We always copy a fresh version of our binary for our L2 guest. +cp target/debug/cloud-hypervisor $VFIO_DIR +# We need qemu to have NET_ADMIN as well. +sudo setcap cap_net_admin+ep /usr/bin/qemu-system-x86_64 + +sudo adduser $USER kvm +newgrp kvm << EOF cargo test --features "integration_tests" +EOF + +# Tear VFIO test network down +sudo ip link del vfio-br0 +sudo ip link del vfio-tap0 +sudo ip link del vfio-tap1 diff --git a/src/main.rs b/src/main.rs index db1d6e0c5..9a71089aa 100755 --- a/src/main.rs +++ b/src/main.rs @@ -196,8 +196,10 @@ mod tests { disks: Vec, fw_path: String, guest_ip: String, + l2_guest_ip: String, host_ip: String, guest_mac: String, + l2_guest_mac: String, } fn prepare_virtiofsd(tmp_dir: &TempDir) -> (std::process::Child, String) { @@ -230,20 +232,18 @@ mod tests { } impl Guest { - fn new() -> Self { + fn new_from_ip_range(class: &str, id: u8) -> Self { let tmp_dir = TempDir::new("ch").unwrap(); - let mut guard = NEXT_VM_ID.lock().unwrap(); - let id = *guard; - *guard = id + 1; - let mut guest = Guest { tmp_dir, disks: Vec::new(), fw_path: String::new(), - guest_ip: format!("192.168.{}.2", id), - host_ip: format!("192.168.{}.1", id), + guest_ip: format!("{}.{}.2", class, id), + l2_guest_ip: format!("{}.{}.3", class, id), + host_ip: format!("{}.{}.1", class, id), guest_mac: format!("12:34:56:78:90:{:02x}", id), + l2_guest_mac: format!("de:ad:be:ef:12:{:02x}", id), }; guest.prepare_files(); @@ -251,6 +251,14 @@ mod tests { guest } + fn new() -> Self { + let mut guard = NEXT_VM_ID.lock().unwrap(); + let id = *guard; + *guard = id + 1; + + Self::new_from_ip_range("192.168", id) + } + fn prepare_cloudinit(&self) -> String { let cloudinit_file_path = String::from(self.tmp_dir.path().join("cloudinit").to_str().unwrap()); @@ -282,7 +290,9 @@ mod tests { user_data_string = user_data_string.replace("192.168.2.1", &self.host_ip); user_data_string = user_data_string.replace("192.168.2.2", &self.guest_ip); + user_data_string = user_data_string.replace("192.168.2.3", &self.l2_guest_ip); user_data_string = user_data_string.replace("12:34:56:78:90:ab", &self.guest_mac); + user_data_string = user_data_string.replace("de:ad:be:ef:12:34", &self.l2_guest_mac); fs::File::create(cloud_init_directory.join("latest").join("user_data")) .unwrap() @@ -343,7 +353,7 @@ mod tests { ) } - fn ssh_command(&self, command: &str) -> String { + fn ssh_command_ip(&self, command: &str, ip: &str) -> String { let mut s = String::new(); #[derive(Debug)] enum Error { @@ -355,8 +365,8 @@ mod tests { let mut counter = 0; loop { match (|| -> Result<(), Error> { - let tcp = TcpStream::connect(format!("{}:22", self.guest_ip)) - .map_err(|_| Error::Connection)?; + let tcp = + TcpStream::connect(format!("{}:22", ip)).map_err(|_| Error::Connection)?; let mut sess = Session::new().unwrap(); sess.handshake(&tcp).map_err(|_| Error::Connection)?; @@ -387,6 +397,18 @@ mod tests { s } + fn ssh_command(&self, command: &str) -> String { + self.ssh_command_ip(command, &self.guest_ip) + } + + fn ssh_command_l1(&self, command: &str) -> String { + self.ssh_command_ip(command, &self.guest_ip) + } + + fn ssh_command_l2(&self, command: &str) -> String { + self.ssh_command_ip(command, &self.l2_guest_ip) + } + fn get_cpu_count(&self) -> u32 { self.ssh_command("grep -c processor /proc/cpuinfo") .trim() @@ -1095,4 +1117,123 @@ mod tests { }); } + #[test] + // The VFIO integration test starts a qemu guest and then direct assigns + // one of the virtio-PCI device to a cloud-hypervisor nested guest. The + // test assigns one of the 2 virtio-pci networking interface, and thus + // the cloud-hypervisor guest will get a networking interface through that + // direct assignment. + // The test starts the QEMU guest with 2 TAP backed networking interfaces, + // bound through a simple bridge on the host. So if the nested + // cloud-hypervisor succeeds in getting a directly assigned interface from + // its QEMU host, we should be able to ssh into it, and verify that it's + // running with the right kernel command line (We tag the cloud-hypervisor + // command line for that puspose). + fn test_vfio() { + test_block!(tb, "", { + let guest = Guest::new_from_ip_range("172.16", 0); + + let home = dirs::home_dir().unwrap(); + let mut cloud_init_vfio_base_path = home.clone(); + cloud_init_vfio_base_path.push("workloads"); + cloud_init_vfio_base_path.push("vfio"); + cloud_init_vfio_base_path.push("cloudinit.img"); + + // We copy our cloudinit into the vfio mount point, for the nested + // cloud-hypervisor guest to use. + fs::copy(&guest.disks[1], &cloud_init_vfio_base_path) + .expect("copying of cloud-init disk failed"); + + let vfio_9p_path = format!( + "local,id=shared,path={}/workloads/vfio/,security_model=none", + home.to_str().unwrap() + ); + + let ovmf_path = format!("{}/workloads/OVMF.fd", home.to_str().unwrap()); + let os_disk = format!("file={},format=qcow2", guest.disks[0].as_str()); + let cloud_init_disk = format!("file={},format=raw", guest.disks[1].as_str()); + + let vfio_tap0 = "vfio-tap0"; + let vfio_tap1 = "vfio-tap1"; + + let ssh_net = "ssh-net"; + let vfio_net = "vfio-net"; + + let netdev_ssh = format!( + "tap,ifname={},id={},script=no,downscript=no", + vfio_tap0, ssh_net + ); + let netdev_ssh_device = format!( + "virtio-net-pci,netdev={},disable-legacy=on,iommu_platform=on,ats=on,mac={}", + ssh_net, guest.guest_mac + ); + + let netdev_vfio = format!( + "tap,ifname={},id={},script=no,downscript=no", + vfio_tap1, vfio_net + ); + let netdev_vfio_device = format!( + "virtio-net-pci,netdev={},disable-legacy=on,iommu_platform=on,ats=on,mac={}", + vfio_net, guest.l2_guest_mac + ); + + let mut qemu_child = Command::new("qemu-system-x86_64") + .args(&["-machine", "q35,accel=kvm,kernel_irqchip=split"]) + .args(&["-bios", &ovmf_path]) + .args(&["-smp", "sockets=1,cpus=4,cores=2"]) + .args(&["-cpu", "host"]) + .args(&["-m", "1024"]) + .args(&["-vga", "none"]) + .args(&["-nographic"]) + .args(&["-drive", &os_disk]) + .args(&["-drive", &cloud_init_disk]) + .args(&["-device", "virtio-rng-pci"]) + .args(&["-netdev", &netdev_ssh]) + .args(&["-device", &netdev_ssh_device]) + .args(&["-netdev", &netdev_vfio]) + .args(&["-device", &netdev_vfio_device]) + .args(&[ + "-device", + "intel-iommu,intremap=on,caching-mode=on,device-iotlb=on", + ]) + .args(&["-fsdev", &vfio_9p_path]) + .args(&[ + "-device", + "virtio-9p-pci,fsdev=shared,mount_tag=cloud_hypervisor", + ]) + .spawn() + .unwrap(); + + thread::sleep(std::time::Duration::new(30, 0)); + + guest.ssh_command_l1("sudo systemctl start vfio"); + thread::sleep(std::time::Duration::new(30, 0)); + + // We booted our cloud hypervisor L2 guest with a "VFIOTAG" tag + // added to its kernel command line. + // Let's ssh into it and verify that it's there. If it is it means + // we're in the right guest (The L2 one) because the QEMU L1 guest + // does not have this command line tag. + aver_eq!( + tb, + guest + .ssh_command_l2("cat /proc/cmdline | grep -c 'VFIOTAG'") + .trim() + .parse::() + .unwrap(), + 1 + ); + + guest.ssh_command_l2("sudo reboot"); + thread::sleep(std::time::Duration::new(10, 0)); + + guest.ssh_command_l1("sudo shutdown -h now"); + thread::sleep(std::time::Duration::new(10, 0)); + + let _ = qemu_child.kill(); + let _ = qemu_child.wait(); + + Ok(()) + }); + } } diff --git a/test_data/cloud-init/openstack/latest/user_data b/test_data/cloud-init/openstack/latest/user_data index 7f1d11b22..b74105fac 100644 --- a/test_data/cloud-init/openstack/latest/user_data +++ b/test_data/cloud-init/openstack/latest/user_data @@ -6,7 +6,7 @@ users: - ALL=(ALL) NOPASSWD:ALL write_files: - - path: /etc/systemd/network/00-static.network + path: /etc/systemd/network/00-static-l1.network permissions: 0644 content: | [Match] @@ -15,3 +15,42 @@ write_files: [Network] Address=192.168.2.2/24 Gateway=192.168.2.1 + + - + path: /etc/systemd/network/00-static-l2.network + permissions: 0644 + content: | + [Match] + MACAddress=de:ad:be:ef:12:34 + + [Network] + Address=192.168.2.3/24 + Gateway=192.168.2.1 + + - + path: /etc/systemd/system/vfio.service + permissions: 0644 + content: | + [Unit] + Description=VFIO test systemd service + + [Service] + Type=simple + ExecStart=/bin/bash /usr/bin/cloud-hypervisor-vfio.sh + + [Install] + WantedBy=multi-user.target + + - + path: /usr/bin/cloud-hypervisor-vfio.sh + permissions: 0755 + content: | + #!/bin/bash + + mount -t 9p -o trans=virtio cloud_hypervisor /mnt -oversion=9p2000.L,posixacl,cache=loose + modprobe vfio_iommu_type1 allow_unsafe_interrupts + modprobe vfio_pci + bash -c "echo 0000:00:03.0 > /sys/bus/pci/devices/0000\:00\:03.0/driver/unbind" + bash -c "echo 1af4 1041 > /sys/bus/pci/drivers/vfio-pci/new_id" + + /mnt/cloud-hypervisor --console off --serial tty --kernel /mnt/vmlinux --cmdline "console=ttyS0 reboot=k panic=1 nomodules i8042.noaux i8042.nomux i8042.nopnp i8042.dumbkbd root=/dev/vda2 VFIOTAG" --disk /mnt/clear-29810-cloud.img /mnt/cloudinit.img --cpus 1 --memory size=512M --rng --device /sys/bus/pci/devices/0000:00:03.0/