From 944d09208e6bdd07612ad19c0d7a7686b8a8d8a3 Mon Sep 17 00:00:00 2001 From: Anatol Belski Date: Wed, 3 Aug 2022 23:52:53 +0200 Subject: [PATCH] ci: Initial Windows image integration This enables the Windows test module. One basic test is enabled, while all others are disabled yet for aarch64. Jenkins file is extended with the corresponding step for aarch64. installAzureCli() is parametrized. It seems that transferring a 30GB image would take >= 15 minutes. An optimization here is having a gzip'ed image to 10GB which would unpack in 3 minutes. Expect to be quicker than transferring an uncompressed image while on another network. Signed-off-by: Anatol Belski --- Jenkinsfile | 45 +++++++++++++- scripts/common-aarch64.sh | 49 +++++++++++++++ scripts/dev_cli.sh | 2 +- scripts/run_integration_tests_aarch64.sh | 47 +-------------- .../run_integration_tests_windows_aarch64.sh | 60 +++++++++++++++++++ ...> run_integration_tests_windows_x86_64.sh} | 0 ...windows-11-iot-enterprise-aarch64.raw.sha1 | 1 + tests/integration.rs | 21 ++++--- 8 files changed, 166 insertions(+), 59 deletions(-) create mode 100644 scripts/common-aarch64.sh create mode 100755 scripts/run_integration_tests_windows_aarch64.sh rename scripts/{run_integration_tests_windows.sh => run_integration_tests_windows_x86_64.sh} (100%) create mode 100644 scripts/windows-11-iot-enterprise-aarch64.raw.sha1 diff --git a/Jenkinsfile b/Jenkinsfile index ce5d19b86..3ae9ff95a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -103,6 +103,9 @@ pipeline{ return runWorkers } } + environment { + AZURE_CONNECTION_STRING = credentials('46b4e7d6-315f-4cc1-8333-b58780863b9b') + } stages { stage ('Checkout') { steps { @@ -123,6 +126,42 @@ pipeline{ sh "scripts/dev_cli.sh tests --integration --libc musl" } } + stage ('Install azure-cli') { + steps { + installAzureCli("bionic", "arm64") + } + } + stage ('Download Windows image') { + steps { + sh '''#!/bin/bash -x + IMG_BASENAME=windows-11-iot-enterprise-aarch64.raw + IMG_PATH=$HOME/workloads/$IMG_BASENAME + IMG_GZ_PATH=$HOME/workloads/$IMG_BASENAME.gz + IMG_GZ_BLOB_NAME=windows-11-iot-enterprise-aarch64-9-min.raw.gz + cp "scripts/$IMG_BASENAME.sha1" "$HOME/workloads/" + pushd "$HOME/workloads" + if sha1sum "$IMG_BASENAME.sha1" --check; then + exit + fi + popd + mkdir -p "$HOME/workloads" + az storage blob download \ + --container-name private-images \ + --file "$IMG_GZ_PATH" \ + --name "$IMG_GZ_BLOB_NAME" \ + --connection-string "$AZURE_CONNECTION_STRING" + gzip -d $IMG_GZ_PATH + ''' + } + } + stage ('Run Windows guest integration tests') { + options { + timeout(time: 1, unit: 'HOURS') + } + steps { + sh "scripts/dev_cli.sh tests --integration-windows --libc musl" + } + } } post { always { @@ -267,7 +306,7 @@ pipeline{ } stage ('Install azure-cli') { steps { - installAzureCli() + installAzureCli("jammy", "amd64") } } stage ('Download assets') { @@ -397,10 +436,10 @@ def cancelPreviousBuilds() { } } -def installAzureCli() { +def installAzureCli(distro, arch) { sh "sudo apt install -y ca-certificates curl apt-transport-https lsb-release gnupg" sh "curl -sL https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/microsoft.gpg > /dev/null" - sh "echo \"deb [arch=amd64] https://packages.microsoft.com/repos/azure-cli/ jammy main\" | sudo tee /etc/apt/sources.list.d/azure-cli.list" + sh "echo \"deb [arch=${arch}] https://packages.microsoft.com/repos/azure-cli/ ${distro} main\" | sudo tee /etc/apt/sources.list.d/azure-cli.list" sh "sudo apt update" sh "sudo apt install -y azure-cli" } diff --git a/scripts/common-aarch64.sh b/scripts/common-aarch64.sh new file mode 100644 index 000000000..e7a4ab1ca --- /dev/null +++ b/scripts/common-aarch64.sh @@ -0,0 +1,49 @@ +#!/bin/bash + +export BUILD_TARGET=${BUILD_TARGET-aarch64-unknown-linux-gnu} + +WORKLOADS_DIR="$HOME/workloads" + +mkdir -p "$WORKLOADS_DIR" + +build_edk2() { + EDK2_BUILD_DIR="$WORKLOADS_DIR/edk2_build" + EDK2_REPO="https://github.com/tianocore/edk2.git" + EDK2_DIR="$EDK2_BUILD_DIR/edk2" + EDK2_PLAT_REPO="https://github.com/tianocore/edk2-platforms.git" + EDK2_PLAT_DIR="$EDK2_BUILD_DIR/edk2-platforms" + ACPICA_REPO="https://github.com/acpica/acpica.git" + ACPICA_DIR="$EDK2_BUILD_DIR/acpica" + export WORKSPACE="$EDK2_BUILD_DIR" + export PACKAGES_PATH="$EDK2_DIR:$EDK2_PLAT_DIR" + export IASL_PREFIX="$ACPICA_DIR/generate/unix/bin/" + + if [ ! -d "$EDK2_BUILD_DIR" ]; then + mkdir -p "$EDK2_BUILD_DIR" + fi + + # Prepare source code + checkout_repo "$EDK2_DIR" "$EDK2_REPO" master "46b4606ba23498d3d0e66b53e498eb3d5d592586" + pushd "$EDK2_DIR" + git submodule update --init + popd + checkout_repo "$EDK2_PLAT_DIR" "$EDK2_PLAT_REPO" master "8227e9e9f6a8aefbd772b40138f835121ccb2307" + checkout_repo "$ACPICA_DIR" "$ACPICA_REPO" master "b9c69f81a05c45611c91ea9cbce8756078d76233" + + if [[ ! -f "$EDK2_DIR/.built" || \ + ! -f "$EDK2_PLAT_DIR/.built" || \ + ! -f "$ACPICA_DIR/.built" ]]; then + pushd "$EDK2_BUILD_DIR" + # Build + make -C acpica -j `nproc` + source edk2/edksetup.sh + make -C edk2/BaseTools -j `nproc` + build -a AARCH64 -t GCC5 -p ArmVirtPkg/ArmVirtCloudHv.dsc -b RELEASE -n 0 + cp Build/ArmVirtCloudHv-AARCH64/RELEASE_GCC5/FV/CLOUDHV_EFI.fd "$WORKLOADS_DIR" + touch "$EDK2_DIR"/.built + touch "$EDK2_PLAT_DIR"/.built + touch "$ACPICA_DIR"/.built + popd + fi +} + diff --git a/scripts/dev_cli.sh b/scripts/dev_cli.sh index eda5cddcb..8aa798bab 100755 --- a/scripts/dev_cli.sh +++ b/scripts/dev_cli.sh @@ -471,7 +471,7 @@ cmd_tests() { --env USER="root" \ --env CH_LIBC="${libc}" \ "$CTR_IMAGE" \ - ./scripts/run_integration_tests_windows.sh "$@" || fix_dir_perms $? || exit $? + ./scripts/run_integration_tests_windows_"$(uname -m)".sh "$@" || fix_dir_perms $? || exit $? fi if [ "$integration_live_migration" = true ]; then diff --git a/scripts/run_integration_tests_aarch64.sh b/scripts/run_integration_tests_aarch64.sh index ce6d52b77..cf4d0b573 100755 --- a/scripts/run_integration_tests_aarch64.sh +++ b/scripts/run_integration_tests_aarch64.sh @@ -3,55 +3,10 @@ set -x source $HOME/.cargo/env source $(dirname "$0")/test-util.sh +source $(dirname "$0")/common-aarch64.sh -export BUILD_TARGET=${BUILD_TARGET-aarch64-unknown-linux-gnu} - -WORKLOADS_DIR="$HOME/workloads" WORKLOADS_LOCK="$WORKLOADS_DIR/integration_test.lock" -mkdir -p "$WORKLOADS_DIR" - -build_edk2() { - EDK2_BUILD_DIR="$WORKLOADS_DIR/edk2_build" - EDK2_REPO="https://github.com/tianocore/edk2.git" - EDK2_DIR="$EDK2_BUILD_DIR/edk2" - EDK2_PLAT_REPO="https://github.com/tianocore/edk2-platforms.git" - EDK2_PLAT_DIR="$EDK2_BUILD_DIR/edk2-platforms" - ACPICA_REPO="https://github.com/acpica/acpica.git" - ACPICA_DIR="$EDK2_BUILD_DIR/acpica" - export WORKSPACE="$EDK2_BUILD_DIR" - export PACKAGES_PATH="$EDK2_DIR:$EDK2_PLAT_DIR" - export IASL_PREFIX="$ACPICA_DIR/generate/unix/bin/" - - if [ ! -d "$EDK2_BUILD_DIR" ]; then - mkdir -p "$EDK2_BUILD_DIR" - fi - - # Prepare source code - checkout_repo "$EDK2_DIR" "$EDK2_REPO" master "46b4606ba23498d3d0e66b53e498eb3d5d592586" - pushd "$EDK2_DIR" - git submodule update --init - popd - checkout_repo "$EDK2_PLAT_DIR" "$EDK2_PLAT_REPO" master "8227e9e9f6a8aefbd772b40138f835121ccb2307" - checkout_repo "$ACPICA_DIR" "$ACPICA_REPO" master "b9c69f81a05c45611c91ea9cbce8756078d76233" - - if [[ ! -f "$EDK2_DIR/.built" || \ - ! -f "$EDK2_PLAT_DIR/.built" || \ - ! -f "$ACPICA_DIR/.built" ]]; then - pushd "$EDK2_BUILD_DIR" - # Build - make -C acpica -j `nproc` - source edk2/edksetup.sh - make -C edk2/BaseTools -j `nproc` - build -a AARCH64 -t GCC5 -p ArmVirtPkg/ArmVirtCloudHv.dsc -b RELEASE -n 0 - cp Build/ArmVirtCloudHv-AARCH64/RELEASE_GCC5/FV/CLOUDHV_EFI.fd "$WORKLOADS_DIR" - touch "$EDK2_DIR"/.built - touch "$EDK2_PLAT_DIR"/.built - touch "$ACPICA_DIR"/.built - popd - fi -} - build_spdk_nvme() { SPDK_DIR="$WORKLOADS_DIR/spdk" SPDK_REPO="https://github.com/spdk/spdk.git" diff --git a/scripts/run_integration_tests_windows_aarch64.sh b/scripts/run_integration_tests_windows_aarch64.sh new file mode 100755 index 000000000..91d5819f7 --- /dev/null +++ b/scripts/run_integration_tests_windows_aarch64.sh @@ -0,0 +1,60 @@ +#!/bin/bash +set -x + +source $HOME/.cargo/env +source $(dirname "$0")/test-util.sh +source $(dirname "$0")/common-aarch64.sh + +process_common_args "$@" +# For now these values are default for kvm +features="" + +# aarch64 not supported for MSHV +if [[ "$hypervisor" = "mshv" ]]; then + echo "AArch64 is not supported in Microsoft Hypervisor" + exit 1 +fi + +WIN_IMAGE_BASENAME="windows-11-iot-enterprise-aarch64.raw" +WIN_IMAGE_FILE="$WORKLOADS_DIR/$WIN_IMAGE_BASENAME" + +# Checkout and build EDK2 +OVMF_FW="$WORKLOADS_DIR/CLOUDHV_EFI.fd" +build_edk2 + +BUILD_TARGET="$(uname -m)-unknown-linux-${CH_LIBC}" +CFLAGS="" +TARGET_CC="" +if [[ "${BUILD_TARGET}" == "aarch64-unknown-linux-musl" ]]; then +export TARGET_CC="musl-gcc" +export RUSTFLAGS="-C link-arg=-lgcc -C link_arg=-specs -C link_arg=/usr/lib/aarch64-linux-musl/musl-gcc.specs" +fi + +# Check if the images are present +if [[ ! -f ${WIN_IMAGE_FILE} || ! -f ${OVMF_FW} ]]; then + echo "Windows image/firmware not present in the host" + exit 1 +fi + +# Use device mapper to create a snapshot of the Windows image +img_blk_size=$(du -b -B 512 ${WIN_IMAGE_FILE} | awk '{print $1;}') +loop_device=$(losetup --find --show --read-only ${WIN_IMAGE_FILE}) +dmsetup create windows-base --table "0 $img_blk_size linear $loop_device 0" +dmsetup mknodes +dmsetup create windows-snapshot-base --table "0 $img_blk_size snapshot-origin /dev/mapper/windows-base" +dmsetup mknodes + +export RUST_BACKTRACE=1 + +cargo build --all --release $features --target $BUILD_TARGET +strip target/$BUILD_TARGET/release/cloud-hypervisor + +# Only run with 1 thread to avoid tests interfering with one another because +# Windows has a static IP configured +time cargo test $features "windows::$test_filter" --target $BUILD_TARGET -- ${test_binary_args[*]} +RES=$? + +dmsetup remove_all -f +losetup -D + +exit $RES diff --git a/scripts/run_integration_tests_windows.sh b/scripts/run_integration_tests_windows_x86_64.sh similarity index 100% rename from scripts/run_integration_tests_windows.sh rename to scripts/run_integration_tests_windows_x86_64.sh diff --git a/scripts/windows-11-iot-enterprise-aarch64.raw.sha1 b/scripts/windows-11-iot-enterprise-aarch64.raw.sha1 new file mode 100644 index 000000000..b7c23bbf1 --- /dev/null +++ b/scripts/windows-11-iot-enterprise-aarch64.raw.sha1 @@ -0,0 +1 @@ +707e939654b341717d3f54b41c57eea4432978e5 windows-11-iot-enterprise-aarch64.raw diff --git a/tests/integration.rs b/tests/integration.rs index 205127690..3e895f13b 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -24,7 +24,6 @@ use std::process::{Child, Command, Stdio}; use std::string::String; use std::sync::mpsc; use std::sync::mpsc::Receiver; -#[cfg(target_arch = "x86_64")] use std::sync::Mutex; use std::thread; use test_infra::*; @@ -59,6 +58,8 @@ mod aarch64 { pub const FOCAL_IMAGE_NAME_VHD: &str = "focal-server-cloudimg-arm64-custom-20210929-0.vhd"; pub const FOCAL_IMAGE_NAME_VHDX: &str = "focal-server-cloudimg-arm64-custom-20210929-0.vhdx"; pub const JAMMY_IMAGE_NAME: &str = "jammy-server-cloudimg-arm64-custom-20220329-0.raw"; + pub const WINDOWS_IMAGE_NAME: &str = "windows-11-iot-enterprise-aarch64.raw"; + pub const OVMF_NAME: &str = "CLOUDHV_EFI.fd"; pub const GREP_SERIAL_IRQ_CMD: &str = "grep -c 'GICv3.*uart-pl011' /proc/interrupts || true"; pub const GREP_PMU_IRQ_CMD: &str = "grep -c 'GICv3.*arm-pmu' /proc/interrupts || true"; } @@ -175,12 +176,11 @@ fn direct_kernel_boot_path() -> PathBuf { kernel_path } -#[cfg(target_arch = "aarch64")] fn edk2_path() -> PathBuf { let mut workload_path = dirs::home_dir().unwrap(); workload_path.push("workloads"); let mut edk2_path = workload_path; - edk2_path.push("CLOUDHV_EFI.fd"); + edk2_path.push(OVMF_NAME); edk2_path } @@ -6795,7 +6795,6 @@ mod sequential { } } -#[cfg(target_arch = "x86_64")] mod windows { use crate::*; use once_cell::sync::Lazy; @@ -7096,14 +7095,10 @@ mod windows { fn test_windows_guest() { let windows_guest = WindowsGuest::new(); - let mut ovmf_path = dirs::home_dir().unwrap(); - ovmf_path.push("workloads"); - ovmf_path.push(OVMF_NAME); - let mut child = GuestCommand::new(windows_guest.guest()) .args(&["--cpus", "boot=2,kvm_hyperv=on"]) .args(&["--memory", "size=4G"]) - .args(&["--kernel", ovmf_path.to_str().unwrap()]) + .args(&["--kernel", edk2_path().to_str().unwrap()]) .args(&["--serial", "tty"]) .args(&["--console", "off"]) .default_disks() @@ -7139,6 +7134,7 @@ mod windows { } #[test] + #[cfg(not(target_arch = "aarch64"))] fn test_windows_guest_multiple_queues() { let windows_guest = WindowsGuest::new(); @@ -7205,6 +7201,7 @@ mod windows { #[test] #[cfg(not(feature = "mshv"))] + #[cfg(not(target_arch = "aarch64"))] #[ignore = "See #4327"] fn test_windows_guest_snapshot_restore() { let windows_guest = WindowsGuest::new(); @@ -7294,6 +7291,7 @@ mod windows { #[test] #[cfg(not(feature = "mshv"))] + #[cfg(not(target_arch = "aarch64"))] fn test_windows_guest_cpu_hotplug() { let windows_guest = WindowsGuest::new(); @@ -7368,6 +7366,7 @@ mod windows { #[test] #[cfg(not(feature = "mshv"))] + #[cfg(not(target_arch = "aarch64"))] fn test_windows_guest_ram_hotplug() { let windows_guest = WindowsGuest::new(); @@ -7442,6 +7441,7 @@ mod windows { #[test] #[cfg(not(feature = "mshv"))] + #[cfg(not(target_arch = "aarch64"))] fn test_windows_guest_netdev_hotplug() { let windows_guest = WindowsGuest::new(); @@ -7514,6 +7514,7 @@ mod windows { #[test] #[cfg(not(feature = "mshv"))] + #[cfg(not(target_arch = "aarch64"))] fn test_windows_guest_disk_hotplug() { let windows_guest = WindowsGuest::new(); @@ -7608,6 +7609,7 @@ mod windows { #[test] #[cfg(not(feature = "mshv"))] + #[cfg(not(target_arch = "aarch64"))] fn test_windows_guest_disk_hotplug_multi() { let windows_guest = WindowsGuest::new(); @@ -7737,6 +7739,7 @@ mod windows { #[test] #[cfg(not(feature = "mshv"))] + #[cfg(not(target_arch = "aarch64"))] fn test_windows_guest_netdev_multi() { let windows_guest = WindowsGuest::new();