From 1365388f0b3263ef317938e04009703a7de0ad87 Mon Sep 17 00:00:00 2001 From: Bo Chen Date: Mon, 12 Jul 2021 19:10:04 -0700 Subject: [PATCH] tests: Extend 'test_vfio' with block device passthrough Fixes: #2822 Signed-off-by: Bo Chen --- test_data/cloud-init/ubuntu/user-data | 9 ++++--- tests/integration.rs | 37 +++++++++++++++++++-------- 2 files changed, 33 insertions(+), 13 deletions(-) diff --git a/test_data/cloud-init/ubuntu/user-data b/test_data/cloud-init/ubuntu/user-data index 5366316e0..feacfe70e 100644 --- a/test_data/cloud-init/ubuntu/user-data +++ b/test_data/cloud-init/ubuntu/user-data @@ -36,15 +36,18 @@ write_files: #!/bin/bash mount /dev/vdc /mnt + bash -c "echo 1af4 1042 > /sys/bus/pci/drivers/vfio-pci/new_id" + bash -c "echo 0000:00:05.0 > /sys/bus/pci/devices/0000\:00\:05.0/driver/unbind" + bash -c "echo 0000:00:05.0 > /sys/bus/pci/drivers/vfio-pci/bind" bash -c "echo 1af4 1041 > /sys/bus/pci/drivers/vfio-pci/new_id" - bash -c "echo 0000:00:06.0 > /sys/bus/pci/devices/0000\:00\:06.0/driver/unbind" - bash -c "echo 0000:00:06.0 > /sys/bus/pci/drivers/vfio-pci/bind" bash -c "echo 0000:00:07.0 > /sys/bus/pci/devices/0000\:00\:07.0/driver/unbind" bash -c "echo 0000:00:07.0 > /sys/bus/pci/drivers/vfio-pci/bind" + bash -c "echo 0000:00:08.0 > /sys/bus/pci/devices/0000\:00\:08.0/driver/unbind" + bash -c "echo 0000:00:08.0 > /sys/bus/pci/drivers/vfio-pci/bind" # 1G ram requires 512 pages echo 512 | sudo tee /proc/sys/vm/nr_hugepages sudo chmod a+rwX /dev/hugepages - /mnt/cloud-hypervisor --kernel /mnt/vmlinux --cmdline "console=hvc0 reboot=k panic=1 nomodules root=/dev/vda1 VFIOTAG" --disk path=/mnt/focal-server-cloudimg-amd64-custom-20210609-0.qcow2 path=/mnt/cloudinit.img --cpus boot=1 --memory size=512M,hotplug_size=1G,hugepages=on --device path=/sys/bus/pci/devices/0000:00:06.0/ path=/sys/bus/pci/devices/0000:00:07.0/ --api-socket /tmp/ch_api.sock + /mnt/cloud-hypervisor --kernel /mnt/vmlinux --cmdline "console=hvc0 reboot=k panic=1 nomodules root=/dev/vda1 VFIOTAG" --disk path=/mnt/focal-server-cloudimg-amd64-custom-20210609-0.qcow2 path=/mnt/cloudinit.img --cpus boot=1 --memory size=512M,hotplug_size=1G,hugepages=on --device path=/sys/bus/pci/devices/0000:00:05.0/ path=/sys/bus/pci/devices/0000:00:07.0/ path=/sys/bus/pci/devices/0000:00:08.0/ --api-socket /tmp/ch_api.sock - path: /etc/systemd/system/notify-booted.service diff --git a/tests/integration.rs b/tests/integration.rs index eee2134cf..af59abaad 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -3238,6 +3238,8 @@ mod tests { // line (We tag the command line from cloud-hypervisor for that purpose). // The third device is added to validate that hotplug works correctly since // it is being added to the L2 VM through hotplugging mechanism. + // Also, we pass-through a vitio-blk device to the L2 VM to test the 32-bit + // vfio device support fn test_vfio() { setup_vfio_network_interfaces(); @@ -3263,7 +3265,7 @@ mod tests { ) .expect("copying of cloud-init disk failed"); - let mut vfio_disk_path = workload_path; + let mut vfio_disk_path = workload_path.clone(); vfio_disk_path.push("vfio.img"); // Create the vfio disk image @@ -3279,6 +3281,9 @@ mod tests { panic!("mkfs.ext4 command generated an error"); } + let mut blk_file_path = workload_path; + blk_file_path.push("blk.img"); + let vfio_tap0 = "vfio-tap0"; let vfio_tap1 = "vfio-tap1"; let vfio_tap2 = "vfio-tap2"; @@ -3301,6 +3306,7 @@ mod tests { ) .as_str(), format!("path={}", vfio_disk_path.to_str().unwrap()).as_str(), + format!("path={},iommu=on", blk_file_path.to_str().unwrap()).as_str(), ]) .args(&[ "--cmdline", @@ -3374,27 +3380,38 @@ mod tests { .trim() .parse::() .unwrap_or_default(), - 7, + 8, + ); + + // Check both if /dev/vdc exists and if the block size is 16M in L2 VM + assert_eq!( + guest + .ssh_command_l2_1("lsblk | grep vdc | grep -c 16M") + .unwrap() + .trim() + .parse::() + .unwrap_or_default(), + 1 ); // Hotplug an extra virtio-net device through L2 VM. guest.ssh_command_l1( - "echo 0000:00:08.0 | sudo tee /sys/bus/pci/devices/0000:00:08.0/driver/unbind", + "echo 0000:00:09.0 | sudo tee /sys/bus/pci/devices/0000:00:09.0/driver/unbind", ).unwrap(); guest .ssh_command_l1( - "echo 0000:00:08.0 | sudo tee /sys/bus/pci/drivers/vfio-pci/bind", + "echo 0000:00:09.0 | sudo tee /sys/bus/pci/drivers/vfio-pci/bind", ) .unwrap(); let vfio_hotplug_output = guest .ssh_command_l1( "sudo /mnt/ch-remote \ --api-socket=/tmp/ch_api.sock \ - add-device path=/sys/bus/pci/devices/0000:00:08.0,id=vfio123", + add-device path=/sys/bus/pci/devices/0000:00:09.0,id=vfio123", ) .unwrap(); assert!( - vfio_hotplug_output.contains("{\"id\":\"vfio123\",\"bdf\":\"0000:00:07.0\"}") + vfio_hotplug_output.contains("{\"id\":\"vfio123\",\"bdf\":\"0000:00:08.0\"}") ); thread::sleep(std::time::Duration::new(10, 0)); @@ -3414,7 +3431,7 @@ mod tests { // Check the amount of PCI devices appearing in L2 VM. // There should be one more device than before, raising the count - // up to 8 PCI devices. + // up to 9 PCI devices. assert_eq!( guest .ssh_command_l2_1("ls /sys/bus/pci/devices | wc -l") @@ -3422,7 +3439,7 @@ mod tests { .trim() .parse::() .unwrap_or_default(), - 8, + 9, ); // Let's now verify that we can correctly remove the virtio-net @@ -3438,7 +3455,7 @@ mod tests { thread::sleep(std::time::Duration::new(10, 0)); // Check the amount of PCI devices appearing in L2 VM is back down - // to 7 devices. + // to 8 devices. assert_eq!( guest .ssh_command_l2_1("ls /sys/bus/pci/devices | wc -l") @@ -3446,7 +3463,7 @@ mod tests { .trim() .parse::() .unwrap_or_default(), - 7, + 8, ); // Perform memory hotplug in L2 and validate the memory is showing