mirror of
https://github.com/cloud-hypervisor/cloud-hypervisor.git
synced 2024-10-02 19:45:46 +00:00
ci: Add VFIO hotplug integration test
This commit extends the existing test_vfio by hotplugging an extra virtio-net device to the L2 VM. The test for validating the hotplug succeeded is the same as the one to verify the non-hotplugged devices. Signed-off-by: Sebastien Boeuf <sebastien.boeuf@intel.com>
This commit is contained in:
parent
d47f733e51
commit
1152b1a147
@ -157,6 +157,10 @@ sudo ip tuntap add vfio-tap2 mode tap
|
|||||||
sudo ip link set vfio-tap2 master vfio-br0
|
sudo ip link set vfio-tap2 master vfio-br0
|
||||||
sudo ip link set vfio-tap2 up
|
sudo ip link set vfio-tap2 up
|
||||||
|
|
||||||
|
sudo ip tuntap add vfio-tap3 mode tap
|
||||||
|
sudo ip link set vfio-tap3 master vfio-br0
|
||||||
|
sudo ip link set vfio-tap3 up
|
||||||
|
|
||||||
cargo build --release
|
cargo build --release
|
||||||
sudo setcap cap_net_admin+ep target/release/cloud-hypervisor
|
sudo setcap cap_net_admin+ep target/release/cloud-hypervisor
|
||||||
sudo setcap cap_net_admin+ep target/release/vhost_user_net
|
sudo setcap cap_net_admin+ep target/release/vhost_user_net
|
||||||
@ -203,5 +207,6 @@ sudo ip link del vfio-br0
|
|||||||
sudo ip link del vfio-tap0
|
sudo ip link del vfio-tap0
|
||||||
sudo ip link del vfio-tap1
|
sudo ip link del vfio-tap1
|
||||||
sudo ip link del vfio-tap2
|
sudo ip link del vfio-tap2
|
||||||
|
sudo ip link del vfio-tap3
|
||||||
|
|
||||||
exit $RES
|
exit $RES
|
||||||
|
@ -38,6 +38,17 @@ write_files:
|
|||||||
Address=192.168.2.4/24
|
Address=192.168.2.4/24
|
||||||
Gateway=192.168.2.1
|
Gateway=192.168.2.1
|
||||||
|
|
||||||
|
-
|
||||||
|
path: /etc/systemd/network/00-static-l2-3.network
|
||||||
|
permissions: 0644
|
||||||
|
content: |
|
||||||
|
[Match]
|
||||||
|
MACAddress=de:ad:be:ef:56:78
|
||||||
|
|
||||||
|
[Network]
|
||||||
|
Address=192.168.2.5/24
|
||||||
|
Gateway=192.168.2.1
|
||||||
|
|
||||||
-
|
-
|
||||||
path: /etc/systemd/system/vfio.service
|
path: /etc/systemd/system/vfio.service
|
||||||
permissions: 0644
|
permissions: 0644
|
||||||
@ -66,4 +77,4 @@ write_files:
|
|||||||
# 512M ram requires 256 pages
|
# 512M ram requires 256 pages
|
||||||
echo 256 | sudo tee /proc/sys/vm/nr_hugepages
|
echo 256 | sudo tee /proc/sys/vm/nr_hugepages
|
||||||
sudo chmod a+rwX /dev/hugepages
|
sudo chmod a+rwX /dev/hugepages
|
||||||
/mnt/cloud-hypervisor --kernel /mnt/vmlinux --cmdline "console=hvc0 reboot=k panic=1 nomodules i8042.noaux i8042.nomux i8042.nopnp i8042.dumbkbd root=PARTUUID=6fb4d1a8-6c8c-4dd7-9f7c-1fe0b9f2574c VFIOTAG" --disk path=/mnt/clear-31311-cloudguest.img path=/mnt/cloudinit.img --cpus boot=1 --memory size=512M,file=/dev/hugepages --device path=/sys/bus/pci/devices/0000:00:05.0/ path=/sys/bus/pci/devices/0000:00:06.0/
|
/mnt/cloud-hypervisor --kernel /mnt/vmlinux --cmdline "console=hvc0 reboot=k panic=1 nomodules i8042.noaux i8042.nomux i8042.nopnp i8042.dumbkbd root=PARTUUID=6fb4d1a8-6c8c-4dd7-9f7c-1fe0b9f2574c VFIOTAG" --disk path=/mnt/clear-31311-cloudguest.img path=/mnt/cloudinit.img --cpus boot=1 --memory size=512M,file=/dev/hugepages --device path=/sys/bus/pci/devices/0000:00:05.0/ path=/sys/bus/pci/devices/0000:00:06.0/ --api-socket /tmp/ch_api.sock
|
||||||
|
@ -39,10 +39,12 @@ mod tests {
|
|||||||
guest_ip: String,
|
guest_ip: String,
|
||||||
l2_guest_ip1: String,
|
l2_guest_ip1: String,
|
||||||
l2_guest_ip2: String,
|
l2_guest_ip2: String,
|
||||||
|
l2_guest_ip3: String,
|
||||||
host_ip: String,
|
host_ip: String,
|
||||||
guest_mac: String,
|
guest_mac: String,
|
||||||
l2_guest_mac1: String,
|
l2_guest_mac1: String,
|
||||||
l2_guest_mac2: String,
|
l2_guest_mac2: String,
|
||||||
|
l2_guest_mac3: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
struct Guest<'a> {
|
struct Guest<'a> {
|
||||||
@ -160,11 +162,14 @@ mod tests {
|
|||||||
user_data_string = user_data_string.replace("192.168.2.2", &network.guest_ip);
|
user_data_string = user_data_string.replace("192.168.2.2", &network.guest_ip);
|
||||||
user_data_string = user_data_string.replace("192.168.2.3", &network.l2_guest_ip1);
|
user_data_string = user_data_string.replace("192.168.2.3", &network.l2_guest_ip1);
|
||||||
user_data_string = user_data_string.replace("192.168.2.4", &network.l2_guest_ip2);
|
user_data_string = user_data_string.replace("192.168.2.4", &network.l2_guest_ip2);
|
||||||
|
user_data_string = user_data_string.replace("192.168.2.5", &network.l2_guest_ip3);
|
||||||
user_data_string = user_data_string.replace("12:34:56:78:90:ab", &network.guest_mac);
|
user_data_string = user_data_string.replace("12:34:56:78:90:ab", &network.guest_mac);
|
||||||
user_data_string =
|
user_data_string =
|
||||||
user_data_string.replace("de:ad:be:ef:12:34", &network.l2_guest_mac1);
|
user_data_string.replace("de:ad:be:ef:12:34", &network.l2_guest_mac1);
|
||||||
user_data_string =
|
user_data_string =
|
||||||
user_data_string.replace("de:ad:be:ef:34:56", &network.l2_guest_mac2);
|
user_data_string.replace("de:ad:be:ef:34:56", &network.l2_guest_mac2);
|
||||||
|
user_data_string =
|
||||||
|
user_data_string.replace("de:ad:be:ef:56:78", &network.l2_guest_mac3);
|
||||||
|
|
||||||
fs::File::create(cloud_init_directory.join("latest").join("user_data"))
|
fs::File::create(cloud_init_directory.join("latest").join("user_data"))
|
||||||
.unwrap()
|
.unwrap()
|
||||||
@ -498,10 +503,12 @@ mod tests {
|
|||||||
guest_ip: format!("{}.{}.2", class, id),
|
guest_ip: format!("{}.{}.2", class, id),
|
||||||
l2_guest_ip1: format!("{}.{}.3", class, id),
|
l2_guest_ip1: format!("{}.{}.3", class, id),
|
||||||
l2_guest_ip2: format!("{}.{}.4", class, id),
|
l2_guest_ip2: format!("{}.{}.4", class, id),
|
||||||
|
l2_guest_ip3: format!("{}.{}.5", class, id),
|
||||||
host_ip: format!("{}.{}.1", class, id),
|
host_ip: format!("{}.{}.1", class, id),
|
||||||
guest_mac: format!("12:34:56:78:90:{:02x}", id),
|
guest_mac: format!("12:34:56:78:90:{:02x}", id),
|
||||||
l2_guest_mac1: format!("de:ad:be:ef:12:{:02x}", id),
|
l2_guest_mac1: format!("de:ad:be:ef:12:{:02x}", id),
|
||||||
l2_guest_mac2: format!("de:ad:be:ef:34:{:02x}", id),
|
l2_guest_mac2: format!("de:ad:be:ef:34:{:02x}", id),
|
||||||
|
l2_guest_mac3: format!("de:ad:be:ef:56:{:02x}", id),
|
||||||
};
|
};
|
||||||
|
|
||||||
disk_config.prepare_files(&tmp_dir, &network);
|
disk_config.prepare_files(&tmp_dir, &network);
|
||||||
@ -572,6 +579,15 @@ mod tests {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn ssh_command_l2_3(&self, command: &str) -> Result<String, Error> {
|
||||||
|
ssh_command_ip(
|
||||||
|
command,
|
||||||
|
&self.network.l2_guest_ip3,
|
||||||
|
DEFAULT_SSH_RETRIES,
|
||||||
|
DEFAULT_SSH_TIMEOUT,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
fn api_create_body(&self, cpu_count: u8) -> String {
|
fn api_create_body(&self, cpu_count: u8) -> String {
|
||||||
format! {"{{\"cpus\":{{\"boot_vcpus\":{},\"max_vcpus\":{}}},\"kernel\":{{\"path\":\"{}\"}},\"cmdline\":{{\"args\": \"\"}},\"net\":[{{\"ip\":\"{}\", \"mask\":\"255.255.255.0\", \"mac\":\"{}\"}}], \"disks\":[{{\"path\":\"{}\"}}, {{\"path\":\"{}\"}}]}}",
|
format! {"{{\"cpus\":{{\"boot_vcpus\":{},\"max_vcpus\":{}}},\"kernel\":{{\"path\":\"{}\"}},\"cmdline\":{{\"args\": \"\"}},\"net\":[{{\"ip\":\"{}\", \"mask\":\"255.255.255.0\", \"mac\":\"{}\"}}], \"disks\":[{{\"path\":\"{}\"}}, {{\"path\":\"{}\"}}]}}",
|
||||||
cpu_count,
|
cpu_count,
|
||||||
@ -2178,17 +2194,14 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[cfg_attr(not(feature = "mmio"), test)]
|
#[cfg_attr(not(feature = "mmio"), test)]
|
||||||
// The VFIO integration test starts a cloud-hypervisor guest and then
|
// The VFIO integration test starts cloud-hypervisor guest with 3 TAP
|
||||||
// direct assigns one of the virtio-pci device to a cloud-hypervisor
|
// backed networking interfaces, bound through a simple bridge on the host.
|
||||||
// nested guest. The test assigns one of the 2 virtio-pci networking
|
// So if the nested cloud-hypervisor succeeds in getting a directly
|
||||||
// interface, and thus the cloud-hypervisor guest will get a networking
|
// assigned interface from its cloud-hypervisor host, we should be able to
|
||||||
// interface through that direct assignment.
|
// ssh into it, and verify that it's running with the right kernel command
|
||||||
// The test starts cloud-hypervisor guest with 2 TAP backed networking
|
// line (We tag the command line from cloud-hypervisor for that purpose).
|
||||||
// interfaces, bound through a simple bridge on the host. So if the nested
|
// The third device is added to validate that hotplug works correctly since
|
||||||
// cloud-hypervisor succeeds in getting a directly assigned interface from
|
// it is being added to the L2 VM through hotplugging mechanism.
|
||||||
// its cloud-hypervisor host, we should be able to ssh into it, and verify
|
|
||||||
// that it's running with the right kernel command line (We tag the command
|
|
||||||
// line from cloud-hypervisor for that purpose).
|
|
||||||
fn test_vfio() {
|
fn test_vfio() {
|
||||||
test_block!(tb, "", {
|
test_block!(tb, "", {
|
||||||
let mut clear = ClearDiskConfig::new();
|
let mut clear = ClearDiskConfig::new();
|
||||||
@ -2217,6 +2230,7 @@ mod tests {
|
|||||||
let vfio_tap0 = "vfio-tap0";
|
let vfio_tap0 = "vfio-tap0";
|
||||||
let vfio_tap1 = "vfio-tap1";
|
let vfio_tap1 = "vfio-tap1";
|
||||||
let vfio_tap2 = "vfio-tap2";
|
let vfio_tap2 = "vfio-tap2";
|
||||||
|
let vfio_tap3 = "vfio-tap3";
|
||||||
|
|
||||||
let (mut daemon_child, virtiofsd_socket_path) =
|
let (mut daemon_child, virtiofsd_socket_path) =
|
||||||
prepare_virtiofsd(&guest.tmp_dir, vfio_path.to_str().unwrap(), "none");
|
prepare_virtiofsd(&guest.tmp_dir, vfio_path.to_str().unwrap(), "none");
|
||||||
@ -2241,6 +2255,10 @@ mod tests {
|
|||||||
"tap={},mac={},iommu=on", vfio_tap2, guest.network.l2_guest_mac2
|
"tap={},mac={},iommu=on", vfio_tap2, guest.network.l2_guest_mac2
|
||||||
)
|
)
|
||||||
.as_str(),
|
.as_str(),
|
||||||
|
format!(
|
||||||
|
"tap={},mac={},iommu=on", vfio_tap3, guest.network.l2_guest_mac3
|
||||||
|
)
|
||||||
|
.as_str(),
|
||||||
])
|
])
|
||||||
.args(&[
|
.args(&[
|
||||||
"--fs",
|
"--fs",
|
||||||
@ -2287,6 +2305,37 @@ mod tests {
|
|||||||
1
|
1
|
||||||
);
|
);
|
||||||
|
|
||||||
|
// Hotplug an extra virtio-net device through L2 VM.
|
||||||
|
guest.ssh_command_l1(
|
||||||
|
"sudo bash -c 'echo 0000:00:07.0 > /sys/bus/pci/devices/0000:00:07.0/driver/unbind'",
|
||||||
|
)?;
|
||||||
|
guest.ssh_command_l1(
|
||||||
|
"sudo bash -c 'echo 1af4 1041 > /sys/bus/pci/drivers/vfio-pci/new_id'",
|
||||||
|
)?;
|
||||||
|
guest.ssh_command_l1(
|
||||||
|
"sudo curl \
|
||||||
|
--unix-socket /tmp/ch_api.sock \
|
||||||
|
-i \
|
||||||
|
-X PUT http://localhost/api/v1/vm.add-device \
|
||||||
|
-H 'Accept: application/json' -H 'Content-Type: application/json' \
|
||||||
|
-d '{\"path\":\"/sys/bus/pci/devices/0000:00:07.0\"}'",
|
||||||
|
)?;
|
||||||
|
thread::sleep(std::time::Duration::new(10, 0));
|
||||||
|
|
||||||
|
// Let's also verify from the third virtio-net device passed to
|
||||||
|
// the L2 VM. This third device has been hotplugged through the L2
|
||||||
|
// VM, so this is our way to validate hotplug works for VFIO PCI.
|
||||||
|
aver_eq!(
|
||||||
|
tb,
|
||||||
|
guest
|
||||||
|
.ssh_command_l2_3("grep -c VFIOTAG /proc/cmdline")
|
||||||
|
.unwrap_or_default()
|
||||||
|
.trim()
|
||||||
|
.parse::<u32>()
|
||||||
|
.unwrap_or_default(),
|
||||||
|
1
|
||||||
|
);
|
||||||
|
|
||||||
let _ = child.kill();
|
let _ = child.kill();
|
||||||
let _ = daemon_child.kill();
|
let _ = daemon_child.kill();
|
||||||
let _ = child.wait();
|
let _ = child.wait();
|
||||||
|
Loading…
Reference in New Issue
Block a user