cloud-hypervisor/scripts/dev_cli.sh

712 lines
22 KiB
Bash
Raw Normal View History

#!/usr/bin/env bash
# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
# Copyright © 2020 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
CLI_NAME="Cloud Hypervisor"
CTR_IMAGE_TAG="ghcr.io/cloud-hypervisor/cloud-hypervisor"
# Needs to match explicit version in docker-image.yaml workflow
CTR_IMAGE_VERSION="20240507-0"
: "${CTR_IMAGE:=${CTR_IMAGE_TAG}:${CTR_IMAGE_VERSION}}"
DOCKER_RUNTIME="docker"
# Host paths
CLH_SCRIPTS_DIR=$(cd "$(dirname "$0")" && pwd)
CLH_ROOT_DIR=$(cd "${CLH_SCRIPTS_DIR}/.." && pwd)
CLH_BUILD_DIR="${CLH_ROOT_DIR}/build"
CLH_CARGO_TARGET="${CLH_BUILD_DIR}/cargo_target"
CLH_DOCKERFILE="${CLH_SCRIPTS_DIR}/../resources/Dockerfile"
CLH_CTR_BUILD_DIR="/tmp/cloud-hypervisor/ctr-build"
CLH_INTEGRATION_WORKLOADS="${HOME}/workloads"
# Container paths
CTR_CLH_ROOT_DIR="/cloud-hypervisor"
CTR_CLH_CARGO_BUILT_DIR="${CTR_CLH_ROOT_DIR}/build"
CTR_CLH_CARGO_TARGET="${CTR_CLH_CARGO_BUILT_DIR}/cargo_target"
CTR_CLH_INTEGRATION_WORKLOADS="/root/workloads"
# Container networking option
CTR_CLH_NET="bridge"
# Cargo paths
# Full path to the cargo registry dir on the host. This appears on the host
# because we want to persist the cargo registry across container invocations.
# Otherwise, any rust crates from crates.io would be downloaded again each time
# we build or test.
CARGO_REGISTRY_DIR="${CLH_BUILD_DIR}/cargo_registry"
# Full path to the cargo git registry on the host. This serves the same purpose
# as CARGO_REGISTRY_DIR, for crates downloaded from GitHub repos instead of
# crates.io.
CARGO_GIT_REGISTRY_DIR="${CLH_BUILD_DIR}/cargo_git_registry"
# Full path to the cargo target dir on the host.
CARGO_TARGET_DIR="${CLH_BUILD_DIR}/cargo_target"
# Send a decorated message to stdout, followed by a new line
#
say() {
[ -t 1 ] && [ -n "$TERM" ] &&
echo "$(tput setaf 2)[$CLI_NAME]$(tput sgr0) $*" ||
echo "[$CLI_NAME] $*"
}
# Send a decorated message to stdout, without a trailing new line
#
say_noln() {
[ -t 1 ] && [ -n "$TERM" ] &&
echo -n "$(tput setaf 2)[$CLI_NAME]$(tput sgr0) $*" ||
echo "[$CLI_NAME] $*"
}
# Send a text message to stderr
#
say_err() {
[ -t 2 ] && [ -n "$TERM" ] &&
echo "$(tput setaf 1)[$CLI_NAME] $*$(tput sgr0)" 1>&2 ||
echo "[$CLI_NAME] $*" 1>&2
}
# Send a warning-highlighted text to stdout
say_warn() {
[ -t 1 ] && [ -n "$TERM" ] &&
echo "$(tput setaf 3)[$CLI_NAME] $*$(tput sgr0)" ||
echo "[$CLI_NAME] $*"
}
# Exit with an error message and (optional) code
# Usage: die [-c <error code>] <error message>
#
die() {
code=1
[[ "$1" = "-c" ]] && {
code="$2"
shift 2
}
say_err "$@"
exit "$code"
}
# Exit with an error message if the last exit code is not 0
#
ok_or_die() {
code=$?
[[ $code -eq 0 ]] || die -c $code "$@"
}
# Make sure the build/ dirs are available. Exit if we can't create them.
# Upon returning from this call, the caller can be certain the build/ dirs exist.
#
ensure_build_dir() {
for dir in "$CLH_BUILD_DIR" \
"$CLH_INTEGRATION_WORKLOADS" \
"$CLH_CTR_BUILD_DIR" \
"$CARGO_TARGET_DIR" \
"$CARGO_REGISTRY_DIR" \
"$CARGO_GIT_REGISTRY_DIR"; do
mkdir -p "$dir" || die "Error: cannot create dir $dir"
[ -x "$dir" ] && [ -w "$dir" ] ||
{
say "Wrong permissions for $dir. Attempting to fix them ..."
chmod +x+w "$dir"
} ||
die "Error: wrong permissions for $dir. Should be +x+w"
done
}
# Make sure we're using the latest dev container, by just pulling it.
ensure_latest_ctr() {
if [ "$CTR_IMAGE_VERSION" = "local" ]; then
build_container
else
if ! $DOCKER_RUNTIME pull "$CTR_IMAGE"; then
build_container
fi
ok_or_die "Error pulling/building container image. Aborting."
fi
}
# Fix main directory permissions after a container ran as root.
# Since the container ran as root, any files it creates will be owned by root.
# This fixes that by recursively changing the ownership of /cloud-hypervisor to the
# current user.
#
fix_dir_perms() {
# Yes, running Docker to get elevated privileges, just to chown some files
# is a dirty hack.
$DOCKER_RUNTIME run \
--workdir "$CTR_CLH_ROOT_DIR" \
--rm \
--volume /dev:/dev \
--volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" \
${exported_volumes:+"$exported_volumes"} \
"$CTR_IMAGE" \
chown -R "$(id -u):$(id -g)" "$CTR_CLH_ROOT_DIR"
return "$1"
}
# Process exported volumes argument, separate the volumes and make docker compatible
# Sample input: --volumes /a:/a#/b:/b
# Sample output: --volume /a:/a --volume /b:/b
#
process_volumes_args() {
if [ -z "$arg_vols" ]; then
return
fi
exported_volumes=""
arr_vols=("${arg_vols//#/ }")
for var in "${arr_vols[@]}"; do
parts=("${var//:/ }")
if [[ ! -e "${parts[0]}" ]]; then
echo "The volume ${parts[0]} does not exist."
exit 1
fi
exported_volumes="$exported_volumes --volume $var"
done
}
cmd_help() {
echo ""
echo "Cloud Hypervisor $(basename "$0")"
echo "Usage: $(basename "$0") [flags] <command> [<command args>]"
echo ""
echo "Available flags":
echo ""
echo " --local Set the container image version being used to \"local\"."
echo ""
echo "Available commands:"
echo ""
echo " build [--debug|--release] [--libc musl|gnu] [-- [<cargo args>]]"
echo " Build the Cloud Hypervisor binaries."
echo " --debug Build the debug binaries. This is the default."
echo " --release Build the release binaries."
echo " --libc Select the C library Cloud Hypervisor will be built against. Default is gnu"
echo " --volumes Hash separated volumes to be exported. Example --volumes /mnt:/mnt#/myvol:/myvol"
echo " --hypervisor Underlying hypervisor. Options kvm, mshv"
echo ""
echo " tests [<test type (see below)>] [--libc musl|gnu] [-- [<test scripts args>] [-- [<test binary args>]]] "
echo " Run the Cloud Hypervisor tests."
echo " --unit Run the unit tests."
echo " --integration Run the integration tests."
echo " --integration-sgx Run the SGX integration tests."
echo " --integration-vfio Run the VFIO integration tests."
echo " --integration-windows Run the Windows guest integration tests."
echo " --integration-live-migration Run the live-migration integration tests."
echo " --integration-rate-limiter Run the rate-limiter integration tests."
echo " --libc Select the C library Cloud Hypervisor will be built against. Default is gnu"
echo " --metrics Generate performance metrics"
echo " --volumes Hash separated volumes to be exported. Example --volumes /mnt:/mnt#/myvol:/myvol"
echo " --hypervisor Underlying hypervisor. Options kvm, mshv"
echo " --all Run all tests."
echo ""
echo " build-container [--type]"
echo " Build the Cloud Hypervisor container."
echo ""
echo " clean [<cargo args>]]"
echo " Remove the Cloud Hypervisor artifacts."
echo ""
echo " shell"
echo " Run the development container into an interactive, privileged BASH shell."
echo " --volumes Hash separated volumes to be exported. Example --volumes /mnt:/mnt#/myvol:/myvol"
echo ""
echo " help"
echo " Display this help message."
echo ""
}
cmd_build() {
build="debug"
libc="gnu"
hypervisor="kvm"
features_build=()
exported_device="/dev/kvm"
while [ $# -gt 0 ]; do
case "$1" in
"-h" | "--help") {
cmd_help
exit 1
} ;;
"--debug") { build="debug"; } ;;
"--release") { build="release"; } ;;
"--runtime")
shift
DOCKER_RUNTIME="$1"
export DOCKER_RUNTIME
;;
"--libc")
shift
[[ "$1" =~ ^(musl|gnu)$ ]] ||
die "Invalid libc: $1. Valid options are \"musl\" and \"gnu\"."
libc="$1"
;;
"--volumes")
shift
arg_vols="$1"
;;
"--hypervisor")
shift
hypervisor="$1"
;;
"--features")
shift
features_build=(--features "$1")
;;
"--") {
shift
break
} ;;
*)
die "Unknown build argument: $1. Please use --help for help."
;;
esac
shift
done
ensure_build_dir
ensure_latest_ctr
process_volumes_args
if [[ ! ("$hypervisor" = "kvm" || "$hypervisor" = "mshv") ]]; then
die "Hypervisor value must be kvm or mshv"
fi
if [[ "$hypervisor" = "mshv" ]]; then
exported_device="/dev/mshv"
fi
target="$(uname -m)-unknown-linux-${libc}"
cargo_args=("$@")
[ $build = "release" ] && cargo_args+=("--release")
cargo_args+=(--target "$target")
# shellcheck disable=SC2153
rustflags="$RUSTFLAGS"
target_cc=""
if [ "$(uname -m)" = "aarch64" ] && [ "$libc" = "musl" ]; then
rustflags="$rustflags -C link-args=-Wl,-Bstatic -C link-args=-lc"
fi
$DOCKER_RUNTIME run \
--user "$(id -u):$(id -g)" \
--workdir "$CTR_CLH_ROOT_DIR" \
--rm \
--volume $exported_device \
--volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" \
${exported_volumes:+"$exported_volumes"} \
--env RUSTFLAGS="$rustflags" \
--env TARGET_CC="$target_cc" \
"$CTR_IMAGE" \
cargo build --all "${features_build[@]}" \
--target-dir "$CTR_CLH_CARGO_TARGET" \
"${cargo_args[@]}" && say "Binaries placed under $CLH_CARGO_TARGET/$target/$build"
}
cmd_clean() {
cargo_args=("$@")
ensure_build_dir
ensure_latest_ctr
$DOCKER_RUNTIME run \
--user "$(id -u):$(id -g)" \
--workdir "$CTR_CLH_ROOT_DIR" \
--rm \
--volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" \
${exported_volumes:+"$exported_volumes"} \
"$CTR_IMAGE" \
cargo clean \
--target-dir "$CTR_CLH_CARGO_TARGET" \
"${cargo_args[@]}"
}
cmd_tests() {
unit=false
integration=false
integration_sgx=false
integration_vfio=false
integration_windows=false
integration_live_migration=false
integration_rate_limiter=false
metrics=false
libc="gnu"
arg_vols=""
hypervisor="kvm"
exported_device="/dev/kvm"
while [ $# -gt 0 ]; do
case "$1" in
"-h" | "--help") {
cmd_help
exit 1
} ;;
"--unit") { unit=true; } ;;
"--integration") { integration=true; } ;;
"--integration-sgx") { integration_sgx=true; } ;;
"--integration-vfio") { integration_vfio=true; } ;;
"--integration-windows") { integration_windows=true; } ;;
"--integration-live-migration") { integration_live_migration=true; } ;;
"--integration-rate-limiter") { integration_rate_limiter=true; } ;;
"--metrics") { metrics=true; } ;;
"--libc")
shift
[[ "$1" =~ ^(musl|gnu)$ ]] ||
die "Invalid libc: $1. Valid options are \"musl\" and \"gnu\"."
libc="$1"
;;
"--volumes")
shift
arg_vols="$1"
;;
"--hypervisor")
shift
hypervisor="$1"
;;
"--all") {
unit=true
integration=true
} ;;
"--") {
shift
break
} ;;
*)
die "Unknown tests argument: $1. Please use --help for help."
;;
esac
shift
done
if [[ ! ("$hypervisor" = "kvm" || "$hypervisor" = "mshv") ]]; then
die "Hypervisor value must be kvm or mshv"
fi
if [[ "$hypervisor" = "mshv" ]]; then
exported_device="/dev/mshv"
fi
if [ ! -e "${exported_device}" ]; then
die "${exported_device} does not exist on the system"
fi
set -- '--hypervisor' "$hypervisor" "$@"
ensure_build_dir
ensure_latest_ctr
process_volumes_args
target="$(uname -m)-unknown-linux-${libc}"
rustflags="$RUSTFLAGS"
target_cc=""
if [ "$(uname -m)" = "aarch64" ] && [ "$libc" = "musl" ]; then
rustflags="$rustflags -C link-args=-Wl,-Bstatic -C link-args=-lc"
fi
if [[ "$unit" = true ]]; then
say "Running unit tests for $target..."
$DOCKER_RUNTIME run \
--workdir "$CTR_CLH_ROOT_DIR" \
--rm \
--device $exported_device \
--device /dev/net/tun \
--cap-add net_admin \
--volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" \
${exported_volumes:+"$exported_volumes"} \
--env BUILD_TARGET="$target" \
--env RUSTFLAGS="$rustflags" \
--env TARGET_CC="$target_cc" \
"$CTR_IMAGE" \
./scripts/run_unit_tests.sh "$@" || fix_dir_perms $? || exit $?
fi
if [ "$integration" = true ]; then
say "Running integration tests for $target..."
$DOCKER_RUNTIME run \
--workdir "$CTR_CLH_ROOT_DIR" \
--rm \
--privileged \
--security-opt seccomp=unconfined \
--ipc=host \
--net="$CTR_CLH_NET" \
--mount type=tmpfs,destination=/tmp \
--volume /dev:/dev \
--volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" \
${exported_volumes:+"$exported_volumes"} \
--volume "$CLH_INTEGRATION_WORKLOADS:$CTR_CLH_INTEGRATION_WORKLOADS" \
--env USER="root" \
--env BUILD_TARGET="$target" \
--env RUSTFLAGS="$rustflags" \
--env TARGET_CC="$target_cc" \
--env AUTH_DOWNLOAD_TOKEN="$AUTH_DOWNLOAD_TOKEN" \
"$CTR_IMAGE" \
dbus-run-session ./scripts/run_integration_tests_"$(uname -m)".sh "$@" || fix_dir_perms $? || exit $?
fi
if [ "$integration_sgx" = true ]; then
say "Running SGX integration tests for $target..."
$DOCKER_RUNTIME run \
--workdir "$CTR_CLH_ROOT_DIR" \
--rm \
--privileged \
--security-opt seccomp=unconfined \
--ipc=host \
--net="$CTR_CLH_NET" \
--mount type=tmpfs,destination=/tmp \
--volume /dev:/dev \
--volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" \
${exported_volumes:+"$exported_volumes"} \
--volume "$CLH_INTEGRATION_WORKLOADS:$CTR_CLH_INTEGRATION_WORKLOADS" \
--env USER="root" \
--env BUILD_TARGET="$target" \
--env RUSTFLAGS="$rustflags" \
--env TARGET_CC="$target_cc" \
--env AUTH_DOWNLOAD_TOKEN="$AUTH_DOWNLOAD_TOKEN" \
"$CTR_IMAGE" \
./scripts/run_integration_tests_sgx.sh "$@" || fix_dir_perms $? || exit $?
fi
if [ "$integration_vfio" = true ]; then
say "Running VFIO integration tests for $target..."
$DOCKER_RUNTIME run \
--workdir "$CTR_CLH_ROOT_DIR" \
--rm \
--privileged \
--security-opt seccomp=unconfined \
--ipc=host \
--net="$CTR_CLH_NET" \
--mount type=tmpfs,destination=/tmp \
--volume /dev:/dev \
--volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" \
${exported_volumes:+"$exported_volumes"} \
--volume "$CLH_INTEGRATION_WORKLOADS:$CTR_CLH_INTEGRATION_WORKLOADS" \
--env USER="root" \
--env BUILD_TARGET="$target" \
--env RUSTFLAGS="$rustflags" \
--env TARGET_CC="$target_cc" \
--env AUTH_DOWNLOAD_TOKEN="$AUTH_DOWNLOAD_TOKEN" \
"$CTR_IMAGE" \
./scripts/run_integration_tests_vfio.sh "$@" || fix_dir_perms $? || exit $?
fi
if [ "$integration_windows" = true ]; then
say "Running Windows integration tests for $target..."
$DOCKER_RUNTIME run \
--workdir "$CTR_CLH_ROOT_DIR" \
--rm \
--privileged \
--security-opt seccomp=unconfined \
--ipc=host \
--net="$CTR_CLH_NET" \
--mount type=tmpfs,destination=/tmp \
--volume /dev:/dev \
--volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" \
${exported_volumes:+"$exported_volumes"} \
--volume "$CLH_INTEGRATION_WORKLOADS:$CTR_CLH_INTEGRATION_WORKLOADS" \
--env USER="root" \
--env BUILD_TARGET="$target" \
--env RUSTFLAGS="$rustflags" \
--env TARGET_CC="$target_cc" \
--env AUTH_DOWNLOAD_TOKEN="$AUTH_DOWNLOAD_TOKEN" \
"$CTR_IMAGE" \
./scripts/run_integration_tests_windows_"$(uname -m)".sh "$@" || fix_dir_perms $? || exit $?
fi
if [ "$integration_live_migration" = true ]; then
say "Running 'live migration' integration tests for $target..."
$DOCKER_RUNTIME run \
--workdir "$CTR_CLH_ROOT_DIR" \
--rm \
--privileged \
--security-opt seccomp=unconfined \
--ipc=host \
--net="$CTR_CLH_NET" \
--mount type=tmpfs,destination=/tmp \
--volume /dev:/dev \
--volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" \
${exported_volumes:+"$exported_volumes"} \
--volume "$CLH_INTEGRATION_WORKLOADS:$CTR_CLH_INTEGRATION_WORKLOADS" \
--env USER="root" \
--env BUILD_TARGET="$target" \
--env RUSTFLAGS="$rustflags" \
--env TARGET_CC="$target_cc" \
--env AUTH_DOWNLOAD_TOKEN="$AUTH_DOWNLOAD_TOKEN" \
"$CTR_IMAGE" \
./scripts/run_integration_tests_live_migration.sh "$@" || fix_dir_perms $? || exit $?
fi
if [ "$integration_rate_limiter" = true ]; then
say "Running 'rate limiter' integration tests for $target..."
$DOCKER_RUNTIME run \
--workdir "$CTR_CLH_ROOT_DIR" \
--rm \
--privileged \
--security-opt seccomp=unconfined \
--ipc=host \
--net="$CTR_CLH_NET" \
--mount type=tmpfs,destination=/tmp \
--volume /dev:/dev \
--volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" \
${exported_volumes:+"$exported_volumes"} \
--volume "$CLH_INTEGRATION_WORKLOADS:$CTR_CLH_INTEGRATION_WORKLOADS" \
--env USER="root" \
--env BUILD_TARGET="$target" \
--env RUSTFLAGS="$rustflags" \
--env TARGET_CC="$target_cc" \
--env AUTH_DOWNLOAD_TOKEN="$AUTH_DOWNLOAD_TOKEN" \
"$CTR_IMAGE" \
./scripts/run_integration_tests_rate_limiter.sh "$@" || fix_dir_perms $? || exit $?
fi
if [ "$metrics" = true ]; then
say "Generating performance metrics for $target..."
$DOCKER_RUNTIME run \
--workdir "$CTR_CLH_ROOT_DIR" \
--rm \
--privileged \
--security-opt seccomp=unconfined \
--ipc=host \
--net="$CTR_CLH_NET" \
--mount type=tmpfs,destination=/tmp \
--volume /dev:/dev \
--volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" \
${exported_volumes:+"$exported_volumes"} \
--volume "$CLH_INTEGRATION_WORKLOADS:$CTR_CLH_INTEGRATION_WORKLOADS" \
--env USER="root" \
--env BUILD_TARGET="$target" \
--env RUSTFLAGS="$rustflags" \
--env TARGET_CC="$target_cc" \
--env RUST_BACKTRACE="${RUST_BACKTRACE}" \
--env AUTH_DOWNLOAD_TOKEN="$AUTH_DOWNLOAD_TOKEN" \
"$CTR_IMAGE" \
./scripts/run_metrics.sh "$@" || fix_dir_perms $? || exit $?
fi
fix_dir_perms $?
}
build_container() {
ensure_build_dir
BUILD_DIR=/tmp/cloud-hypervisor/container/
mkdir -p $BUILD_DIR
cp "$CLH_DOCKERFILE" $BUILD_DIR
[ "$(uname -m)" = "aarch64" ] && TARGETARCH="arm64"
[ "$(uname -m)" = "x86_64" ] && TARGETARCH="amd64"
$DOCKER_RUNTIME build \
--target dev \
-t "$CTR_IMAGE" \
-f $BUILD_DIR/Dockerfile \
--build-arg TARGETARCH="$TARGETARCH" \
$BUILD_DIR
}
cmd_build-container() {
while [ $# -gt 0 ]; do
case "$1" in
"-h" | "--help") {
cmd_help
exit 1
} ;;
"--") {
shift
break
} ;;
*)
die "Unknown build-container argument: $1. Please use --help for help."
;;
esac
shift
done
build_container
}
cmd_shell() {
while [ $# -gt 0 ]; do
case "$1" in
"-h" | "--help") {
cmd_help
exit 1
} ;;
"--volumes")
shift
arg_vols="$1"
;;
"--") {
shift
break
} ;;
*) ;;
esac
shift
done
ensure_build_dir
ensure_latest_ctr
process_volumes_args
say_warn "Starting a privileged shell prompt as root ..."
say_warn "WARNING: Your $CLH_ROOT_DIR folder will be bind-mounted in the container under $CTR_CLH_ROOT_DIR"
$DOCKER_RUNTIME run \
-ti \
--workdir "$CTR_CLH_ROOT_DIR" \
--rm \
--privileged \
--security-opt seccomp=unconfined \
--ipc=host \
--net="$CTR_CLH_NET" \
--tmpfs /tmp:exec \
--volume /dev:/dev \
--volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" \
${exported_volumes:+"$exported_volumes"} \
--volume "$CLH_INTEGRATION_WORKLOADS:$CTR_CLH_INTEGRATION_WORKLOADS" \
--env USER="root" \
--entrypoint bash \
"$CTR_IMAGE"
fix_dir_perms $?
}
if [ $# = 0 ]; then
cmd_help
say_err "Please specify command to run!"
exit 1
fi
# Parse main command line args.
#
while [ $# -gt 0 ]; do
case "$1" in
-h | --help) {
cmd_help
exit 1
} ;;
--local) {
CTR_IMAGE_VERSION="local"
CTR_IMAGE="${CTR_IMAGE_TAG}:${CTR_IMAGE_VERSION}"
} ;;
-*)
die "Unknown arg: $1. Please use \`$0 help\` for help."
;;
*)
break
;;
esac
shift
done
# $1 is now a command name. Check if it is a valid command and, if so,
# run it.
#
declare -f "cmd_$1" >/dev/null
ok_or_die "Unknown command: $1. Please use \`$0 help\` for help."
cmd=cmd_$1
shift
$cmd "$@"