From 7e3cbf04dec6ed634dad813aff676fa763d41941 Mon Sep 17 00:00:00 2001 From: Michael Zhao Date: Tue, 28 Jul 2020 11:57:01 +0800 Subject: [PATCH] scripts: Improve AArch64 CI for parallel executors Lock "work_loads" folder when one job is syncing files. If another job arrives, wait until the lock is released. Signed-off-by: Michael Zhao --- scripts/run_integration_tests_aarch64.sh | 299 ++++++++++++----------- 1 file changed, 158 insertions(+), 141 deletions(-) diff --git a/scripts/run_integration_tests_aarch64.sh b/scripts/run_integration_tests_aarch64.sh index 8cfce2971..f97b04873 100755 --- a/scripts/run_integration_tests_aarch64.sh +++ b/scripts/run_integration_tests_aarch64.sh @@ -6,162 +6,175 @@ source $HOME/.cargo/env export BUILD_TARGET=${BUILD_TARGET-aarch64-unknown-linux-gnu} WORKLOADS_DIR="$HOME/workloads" +WORKLOADS_LOCK="$WORKLOADS_DIR/integration_test.lock" + mkdir -p "$WORKLOADS_DIR" -cp scripts/sha1sums-aarch64 $WORKLOADS_DIR +update_workloads() { + cp scripts/sha1sums-aarch64 $WORKLOADS_DIR + + BIONIC_OS_IMAGE_DOWNLOAD_NAME="bionic-server-cloudimg-arm64.img" + BIONIC_OS_IMAGE_DOWNLOAD_URL="https://cloudhypervisorstorage.blob.core.windows.net/images/$BIONIC_OS_IMAGE_DOWNLOAD_NAME" + BIONIC_OS_DOWNLOAD_IMAGE="$WORKLOADS_DIR/$BIONIC_OS_IMAGE_DOWNLOAD_NAME" + if [ ! -f "$BIONIC_OS_DOWNLOAD_IMAGE" ]; then + pushd $WORKLOADS_DIR + time wget --quiet $BIONIC_OS_IMAGE_DOWNLOAD_URL || exit 1 + popd + fi + + BIONIC_OS_RAW_IMAGE_NAME="bionic-server-cloudimg-arm64.raw" + BIONIC_OS_RAW_IMAGE="$WORKLOADS_DIR/$BIONIC_OS_RAW_IMAGE_NAME" + if [ ! -f "$BIONIC_OS_RAW_IMAGE" ]; then + pushd $WORKLOADS_DIR + time qemu-img convert -p -f qcow2 -O raw $BIONIC_OS_IMAGE_DOWNLOAD_NAME $BIONIC_OS_RAW_IMAGE_NAME || exit 1 + popd + fi + + # Convert the raw image to qcow2 image to remove compressed blocks from the disk. Therefore letting the + # qcow2 format image can be directly used in the integration test. + BIONIC_OS_QCOW2_IMAGE_UNCOMPRESSED_NAME="bionic-server-cloudimg-arm64.qcow2" + BIONIC_OS_QCOW2_UNCOMPRESSED_IMAGE="$WORKLOADS_DIR/$BIONIC_OS_QCOW2_IMAGE_UNCOMPRESSED_NAME" + if [ ! -f "$BIONIC_OS_QCOW2_UNCOMPRESSED_IMAGE" ]; then + pushd $WORKLOADS_DIR + time qemu-img convert -p -f raw -O qcow2 $BIONIC_OS_RAW_IMAGE_NAME $BIONIC_OS_QCOW2_UNCOMPRESSED_IMAGE || exit 1 + popd + fi + + FOCAL_OS_IMAGE_DOWNLOAD_NAME="focal-server-cloudimg-arm64.img" + FOCAL_OS_IMAGE_DOWNLOAD_URL="https://cloudhypervisorstorage.blob.core.windows.net/images/$FOCAL_OS_IMAGE_DOWNLOAD_NAME" + FOCAL_OS_DOWNLOAD_IMAGE="$WORKLOADS_DIR/$FOCAL_OS_IMAGE_DOWNLOAD_NAME" + if [ ! -f "$FOCAL_OS_DOWNLOAD_IMAGE" ]; then + pushd $WORKLOADS_DIR + time wget --quiet $FOCAL_OS_IMAGE_DOWNLOAD_URL || exit 1 + popd + fi + + FOCAL_OS_RAW_IMAGE_NAME="focal-server-cloudimg-arm64.raw" + FOCAL_OS_RAW_IMAGE="$WORKLOADS_DIR/$FOCAL_OS_RAW_IMAGE_NAME" + if [ ! -f "$FOCAL_OS_RAW_IMAGE" ]; then + pushd $WORKLOADS_DIR + time qemu-img convert -p -f qcow2 -O raw $FOCAL_OS_IMAGE_DOWNLOAD_NAME $FOCAL_OS_RAW_IMAGE_NAME || exit 1 + popd + fi + + # Convert the raw image to qcow2 image to remove compressed blocks from the disk. Therefore letting the + # qcow2 format image can be directly used in the integration test. + FOCAL_OS_QCOW2_IMAGE_UNCOMPRESSED_NAME="focal-server-cloudimg-arm64.qcow2" + FOCAL_OS_QCOW2_UNCOMPRESSED_IMAGE="$WORKLOADS_DIR/$FOCAL_OS_QCOW2_IMAGE_UNCOMPRESSED_NAME" + if [ ! -f "$FOCAL_OS_QCOW2_UNCOMPRESSED_IMAGE" ]; then + pushd $WORKLOADS_DIR + time qemu-img convert -p -f raw -O qcow2 $FOCAL_OS_RAW_IMAGE_NAME $FOCAL_OS_QCOW2_UNCOMPRESSED_IMAGE || exit 1 + popd + fi + + ALPINE_MINIROOTFS_URL="http://dl-cdn.alpinelinux.org/alpine/v3.11/releases/aarch64/alpine-minirootfs-3.11.3-aarch64.tar.gz" + ALPINE_MINIROOTFS_TARBALL="$WORKLOADS_DIR/alpine-minirootfs-aarch64.tar.gz" + if [ ! -f "$ALPINE_MINIROOTFS_TARBALL" ]; then + pushd $WORKLOADS_DIR + time wget --quiet $ALPINE_MINIROOTFS_URL -O $ALPINE_MINIROOTFS_TARBALL || exit 1 + popd + fi + + ALPINE_INITRAMFS_IMAGE="$WORKLOADS_DIR/alpine_initramfs.img" + if [ ! -f "$ALPINE_INITRAMFS_IMAGE" ]; then + pushd $WORKLOADS_DIR + mkdir alpine-minirootfs + tar xf "$ALPINE_MINIROOTFS_TARBALL" -C alpine-minirootfs + cat > alpine-minirootfs/init <<-EOF + #! /bin/sh + mount -t devtmpfs dev /dev + echo \$TEST_STRING > /dev/console + poweroff -f + EOF + chmod +x alpine-minirootfs/init + cd alpine-minirootfs + find . -print0 | + cpio --null --create --verbose --owner root:root --format=newc > "$ALPINE_INITRAMFS_IMAGE" + popd + fi -BIONIC_OS_IMAGE_DOWNLOAD_NAME="bionic-server-cloudimg-arm64.img" \ -BIONIC_OS_IMAGE_DOWNLOAD_URL="https://cloudhypervisorstorage.blob.core.windows.net/images/$BIONIC_OS_IMAGE_DOWNLOAD_NAME" -BIONIC_OS_DOWNLOAD_IMAGE="$WORKLOADS_DIR/$BIONIC_OS_IMAGE_DOWNLOAD_NAME" -if [ ! -f "$BIONIC_OS_DOWNLOAD_IMAGE" ]; then pushd $WORKLOADS_DIR - time wget --quiet $BIONIC_OS_IMAGE_DOWNLOAD_URL || exit 1 + sha1sum sha1sums-aarch64 --check + if [ $? -ne 0 ]; then + echo "sha1sum validation of images failed, remove invalid images to fix the issue." + exit 1 + fi popd -fi -BIONIC_OS_RAW_IMAGE_NAME="bionic-server-cloudimg-arm64.raw" \ -BIONIC_OS_RAW_IMAGE="$WORKLOADS_DIR/$BIONIC_OS_RAW_IMAGE_NAME" -if [ ! -f "$BIONIC_OS_RAW_IMAGE" ]; then - pushd $WORKLOADS_DIR - time qemu-img convert -p -f qcow2 -O raw $BIONIC_OS_IMAGE_DOWNLOAD_NAME $BIONIC_OS_RAW_IMAGE_NAME || exit 1 - popd -fi + # Build custom kernel based on virtio-pmem and virtio-fs upstream patches + PE_IMAGE="$WORKLOADS_DIR/Image" + LINUX_CUSTOM_DIR="$WORKLOADS_DIR/linux-custom" -# Convert the raw image to qcow2 image to remove compressed blocks from the disk. Therefore letting the -# qcow2 format image can be directly used in the integration test. -BIONIC_OS_QCOW2_IMAGE_UNCOMPRESSED_NAME="bionic-server-cloudimg-arm64.qcow2" \ -BIONIC_OS_QCOW2_UNCOMPRESSED_IMAGE="$WORKLOADS_DIR/$BIONIC_OS_QCOW2_IMAGE_UNCOMPRESSED_NAME" -if [ ! -f "$BIONIC_OS_QCOW2_UNCOMPRESSED_IMAGE" ]; then - pushd $WORKLOADS_DIR - time qemu-img convert -p -f raw -O qcow2 $BIONIC_OS_RAW_IMAGE_NAME $BIONIC_OS_QCOW2_UNCOMPRESSED_IMAGE || exit 1 - popd -fi + build_custom_linux_kernel() { + pushd $LINUX_CUSTOM_DIR + time make -j `nproc` + cp arch/arm64/boot/Image $WORKLOADS_DIR/Image || exit 1 + popd + } -FOCAL_OS_IMAGE_DOWNLOAD_NAME="focal-server-cloudimg-arm64.img" \ -FOCAL_OS_IMAGE_DOWNLOAD_URL="https://cloudhypervisorstorage.blob.core.windows.net/images/$FOCAL_OS_IMAGE_DOWNLOAD_NAME" -FOCAL_OS_DOWNLOAD_IMAGE="$WORKLOADS_DIR/$FOCAL_OS_IMAGE_DOWNLOAD_NAME" -if [ ! -f "$FOCAL_OS_DOWNLOAD_IMAGE" ]; then - pushd $WORKLOADS_DIR - time wget --quiet $FOCAL_OS_IMAGE_DOWNLOAD_URL || exit 1 - popd -fi + if [ ! -d "$LINUX_CUSTOM_DIR" ]; then + pushd $WORKLOADS_DIR + time git clone --depth 1 "https://github.com/cloud-hypervisor/linux.git" -b "virtio-fs-virtio-iommu-virtio-mem-5.6-rc4" $LINUX_CUSTOM_DIR + popd + else + pushd $LINUX_CUSTOM_DIR + git fetch + git checkout -f + popd + fi -FOCAL_OS_RAW_IMAGE_NAME="focal-server-cloudimg-arm64.raw" \ -FOCAL_OS_RAW_IMAGE="$WORKLOADS_DIR/$FOCAL_OS_RAW_IMAGE_NAME" -if [ ! -f "$FOCAL_OS_RAW_IMAGE" ]; then - pushd $WORKLOADS_DIR - time qemu-img convert -p -f qcow2 -O raw $FOCAL_OS_IMAGE_DOWNLOAD_NAME $FOCAL_OS_RAW_IMAGE_NAME || exit 1 - popd -fi + SRCDIR=$PWD + cp $SRCDIR/resources/linux-config-aarch64 $LINUX_CUSTOM_DIR/.config + build_custom_linux_kernel -# Convert the raw image to qcow2 image to remove compressed blocks from the disk. Therefore letting the -# qcow2 format image can be directly used in the integration test. -FOCAL_OS_QCOW2_IMAGE_UNCOMPRESSED_NAME="focal-server-cloudimg-arm64.qcow2" \ -FOCAL_OS_QCOW2_UNCOMPRESSED_IMAGE="$WORKLOADS_DIR/$FOCAL_OS_QCOW2_IMAGE_UNCOMPRESSED_NAME" -if [ ! -f "$FOCAL_OS_QCOW2_UNCOMPRESSED_IMAGE" ]; then - pushd $WORKLOADS_DIR - time qemu-img convert -p -f raw -O qcow2 $FOCAL_OS_RAW_IMAGE_NAME $FOCAL_OS_QCOW2_UNCOMPRESSED_IMAGE || exit 1 - popd -fi + VIRTIOFSD="$WORKLOADS_DIR/virtiofsd" + QEMU_DIR="qemu_build" -ALPINE_MINIROOTFS_URL="http://dl-cdn.alpinelinux.org/alpine/v3.11/releases/aarch64/alpine-minirootfs-3.11.3-aarch64.tar.gz" -ALPINE_MINIROOTFS_TARBALL="$WORKLOADS_DIR/alpine-minirootfs-aarch64.tar.gz" -if [ ! -f "$ALPINE_MINIROOTFS_TARBALL" ]; then - pushd $WORKLOADS_DIR - time wget --quiet $ALPINE_MINIROOTFS_URL -O $ALPINE_MINIROOTFS_TARBALL || exit 1 - popd -fi + if [ ! -f "$VIRTIOFSD" ]; then + pushd $WORKLOADS_DIR + git clone --depth 1 "https://gitlab.com/virtio-fs/qemu.git" -b "virtio-fs-dev" $QEMU_DIR + pushd $QEMU_DIR + time ./configure --prefix=$PWD --target-list=aarch64-softmmu + time make virtiofsd -j `nproc` + cp virtiofsd $VIRTIOFSD || exit 1 + popd + rm -rf $QEMU_DIR + sudo setcap cap_chown,cap_dac_override,cap_dac_read_search,cap_fowner,cap_fsetid,cap_setgid,cap_setuid,cap_mknod,cap_setfcap,cap_sys_admin+epi "virtiofsd" || exit 1 + popd + fi -ALPINE_INITRAMFS_IMAGE="$WORKLOADS_DIR/alpine_initramfs.img" -if [ ! -f "$ALPINE_INITRAMFS_IMAGE" ]; then - pushd $WORKLOADS_DIR - mkdir alpine-minirootfs - tar xf "$ALPINE_MINIROOTFS_TARBALL" -C alpine-minirootfs - cat > alpine-minirootfs/init <<-EOF - #! /bin/sh - mount -t devtmpfs dev /dev - echo \$TEST_STRING > /dev/console - poweroff -f - EOF - chmod +x alpine-minirootfs/init - cd alpine-minirootfs - find . -print0 | - cpio --null --create --verbose --owner root:root --format=newc > "$ALPINE_INITRAMFS_IMAGE" - popd -fi + BLK_IMAGE="$WORKLOADS_DIR/blk.img" + MNT_DIR="mount_image" + if [ ! -f "$BLK_IMAGE" ]; then + pushd $WORKLOADS_DIR + fallocate -l 16M $BLK_IMAGE + mkfs.ext4 -j $BLK_IMAGE + mkdir $MNT_DIR + sudo mount -t ext4 $BLK_IMAGE $MNT_DIR + sudo bash -c "echo bar > $MNT_DIR/foo" || exit 1 + sudo umount $BLK_IMAGE + rm -r $MNT_DIR + popd + fi -pushd $WORKLOADS_DIR -sha1sum sha1sums-aarch64 --check -if [ $? -ne 0 ]; then - echo "sha1sum validation of images failed, remove invalid images to fix the issue." - exit 1 -fi -popd - -# Build custom kernel based on virtio-pmem and virtio-fs upstream patches -PE_IMAGE="$WORKLOADS_DIR/Image" - -LINUX_CUSTOM_DIR="$WORKLOADS_DIR/linux-custom" - -build_custom_linux_kernel() { - pushd $LINUX_CUSTOM_DIR - time make -j `nproc` - cp arch/arm64/boot/Image $WORKLOADS_DIR/Image || exit 1 - popd + SHARED_DIR="$WORKLOADS_DIR/shared_dir" + if [ ! -d "$SHARED_DIR" ]; then + mkdir -p $SHARED_DIR + echo "foo" > "$SHARED_DIR/file1" + echo "bar" > "$SHARED_DIR/file3" || exit 1 + fi } -if [ ! -d "$LINUX_CUSTOM_DIR" ]; then - pushd $WORKLOADS_DIR - time git clone --depth 1 "https://github.com/cloud-hypervisor/linux.git" -b "virtio-fs-virtio-iommu-virtio-mem-5.6-rc4" $LINUX_CUSTOM_DIR - popd +# lock the workloads folder to avoid parallel updating by different containers +( + echo "try to lock $WORKLOADS_DIR folder and update" + flock -x 12 && update_workloads +) 12>$WORKLOADS_LOCK -else - pushd $LINUX_CUSTOM_DIR - git fetch - git checkout -f - popd -fi - -SRCDIR=$PWD -cp $SRCDIR/resources/linux-config-aarch64 $LINUX_CUSTOM_DIR/.config -build_custom_linux_kernel - -VIRTIOFSD="$WORKLOADS_DIR/virtiofsd" -QEMU_DIR="qemu_build" - -if [ ! -f "$VIRTIOFSD" ]; then - pushd $WORKLOADS_DIR - git clone --depth 1 "https://gitlab.com/virtio-fs/qemu.git" -b "virtio-fs-dev" $QEMU_DIR - pushd $QEMU_DIR - time ./configure --prefix=$PWD --target-list=aarch64-softmmu - time make virtiofsd -j `nproc` - cp virtiofsd $VIRTIOFSD || exit 1 - popd - rm -rf $QEMU_DIR - sudo setcap cap_chown,cap_dac_override,cap_dac_read_search,cap_fowner,cap_fsetid,cap_setgid,cap_setuid,cap_mknod,cap_setfcap,cap_sys_admin+epi "virtiofsd" || exit 1 - popd -fi - -BLK_IMAGE="$WORKLOADS_DIR/blk.img" -MNT_DIR="mount_image" -if [ ! -f "$BLK_IMAGE" ]; then - pushd $WORKLOADS_DIR - fallocate -l 16M $BLK_IMAGE - mkfs.ext4 -j $BLK_IMAGE - mkdir $MNT_DIR - sudo mount -t ext4 $BLK_IMAGE $MNT_DIR - sudo bash -c "echo bar > $MNT_DIR/foo" || exit 1 - sudo umount $BLK_IMAGE - rm -r $MNT_DIR - popd -fi - -SHARED_DIR="$WORKLOADS_DIR/shared_dir" -if [ ! -d "$SHARED_DIR" ]; then - mkdir -p $SHARED_DIR - echo "foo" > "$SHARED_DIR/file1" - echo "bar" > "$SHARED_DIR/file3" || exit 1 -fi +# Create tap interface without multipe queues support for vhost_user_net test. +sudo ip tuntap add name vunet-tap0 mode tap +# Create tap interface with multipe queues support for vhost_user_net test. +sudo ip tuntap add name vunet-tap1 mode tap multi_queue BUILD_TARGET="aarch64-unknown-linux-${CH_LIBC}" CFLAGS="" @@ -176,7 +189,7 @@ sed -i 's/"with-serde",\ //g' hypervisor/Cargo.toml cargo_args=("$@") cargo_args+=("--no-default-features") cargo_args+=("--features mmio,kvm") -cargo build --release --target $BUILD_TARGET ${cargo_args[@]} +cargo build --all --release --target $BUILD_TARGET ${cargo_args[@]} strip target/$BUILD_TARGET/release/cloud-hypervisor strip target/$BUILD_TARGET/release/vhost_user_net strip target/$BUILD_TARGET/release/ch-remote @@ -221,4 +234,8 @@ EOF RES=$? fi +# Tear vhost_user_net test network down +sudo ip link del vunet-tap0 +sudo ip link del vunet-tap1 + exit $RES