#!/usr/bin/env bash # Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. # Copyright © 2020 Intel Corporation # SPDX-License-Identifier: Apache-2.0 CLI_NAME="Cloud Hypervisor" CTR_IMAGE_TAG="ghcr.io/cloud-hypervisor/cloud-hypervisor" # Needs to match explicit version in docker-image.yaml workflow CTR_IMAGE_VERSION="20241022-0" : "${CTR_IMAGE:=${CTR_IMAGE_TAG}:${CTR_IMAGE_VERSION}}" DOCKER_RUNTIME="docker" # Host paths CLH_SCRIPTS_DIR=$(cd "$(dirname "$0")" && pwd) CLH_ROOT_DIR=$(cd "${CLH_SCRIPTS_DIR}/.." && pwd) CLH_BUILD_DIR="${CLH_ROOT_DIR}/build" CLH_CARGO_TARGET="${CLH_BUILD_DIR}/cargo_target" CLH_DOCKERFILE="${CLH_SCRIPTS_DIR}/../resources/Dockerfile" CLH_CTR_BUILD_DIR="/tmp/cloud-hypervisor/ctr-build" CLH_INTEGRATION_WORKLOADS="${HOME}/workloads" # Container paths CTR_CLH_ROOT_DIR="/cloud-hypervisor" CTR_CLH_CARGO_BUILT_DIR="${CTR_CLH_ROOT_DIR}/build" CTR_CLH_CARGO_TARGET="${CTR_CLH_CARGO_BUILT_DIR}/cargo_target" CTR_CLH_INTEGRATION_WORKLOADS="/root/workloads" # Container networking option CTR_CLH_NET="bridge" # Cargo paths # Full path to the cargo registry dir on the host. This appears on the host # because we want to persist the cargo registry across container invocations. # Otherwise, any rust crates from crates.io would be downloaded again each time # we build or test. CARGO_REGISTRY_DIR="${CLH_BUILD_DIR}/cargo_registry" # Full path to the cargo git registry on the host. This serves the same purpose # as CARGO_REGISTRY_DIR, for crates downloaded from GitHub repos instead of # crates.io. CARGO_GIT_REGISTRY_DIR="${CLH_BUILD_DIR}/cargo_git_registry" # Full path to the cargo target dir on the host. CARGO_TARGET_DIR="${CLH_BUILD_DIR}/cargo_target" # Send a decorated message to stdout, followed by a new line # say() { [ -t 1 ] && [ -n "$TERM" ] && echo "$(tput setaf 2)[$CLI_NAME]$(tput sgr0) $*" || echo "[$CLI_NAME] $*" } # Send a decorated message to stdout, without a trailing new line # say_noln() { [ -t 1 ] && [ -n "$TERM" ] && echo -n "$(tput setaf 2)[$CLI_NAME]$(tput sgr0) $*" || echo "[$CLI_NAME] $*" } # Send a text message to stderr # say_err() { [ -t 2 ] && [ -n "$TERM" ] && echo "$(tput setaf 1)[$CLI_NAME] $*$(tput sgr0)" 1>&2 || echo "[$CLI_NAME] $*" 1>&2 } # Send a warning-highlighted text to stdout say_warn() { [ -t 1 ] && [ -n "$TERM" ] && echo "$(tput setaf 3)[$CLI_NAME] $*$(tput sgr0)" || echo "[$CLI_NAME] $*" } # Exit with an error message and (optional) code # Usage: die [-c ] # die() { code=1 [[ "$1" = "-c" ]] && { code="$2" shift 2 } say_err "$@" exit "$code" } # Exit with an error message if the last exit code is not 0 # ok_or_die() { code=$? [[ $code -eq 0 ]] || die -c $code "$@" } # Make sure the build/ dirs are available. Exit if we can't create them. # Upon returning from this call, the caller can be certain the build/ dirs exist. # ensure_build_dir() { for dir in "$CLH_BUILD_DIR" \ "$CLH_INTEGRATION_WORKLOADS" \ "$CLH_CTR_BUILD_DIR" \ "$CARGO_TARGET_DIR" \ "$CARGO_REGISTRY_DIR" \ "$CARGO_GIT_REGISTRY_DIR"; do mkdir -p "$dir" || die "Error: cannot create dir $dir" [ -x "$dir" ] && [ -w "$dir" ] || { say "Wrong permissions for $dir. Attempting to fix them ..." chmod +x+w "$dir" } || die "Error: wrong permissions for $dir. Should be +x+w" done } # Make sure we're using the latest dev container, by just pulling it. ensure_latest_ctr() { if [ "$CTR_IMAGE_VERSION" = "local" ]; then build_container else if ! $DOCKER_RUNTIME pull "$CTR_IMAGE"; then build_container fi ok_or_die "Error pulling/building container image. Aborting." fi } # Fix main directory permissions after a container ran as root. # Since the container ran as root, any files it creates will be owned by root. # This fixes that by recursively changing the ownership of /cloud-hypervisor to the # current user. # fix_dir_perms() { # Yes, running Docker to get elevated privileges, just to chown some files # is a dirty hack. $DOCKER_RUNTIME run \ --workdir "$CTR_CLH_ROOT_DIR" \ --rm \ --volume /dev:/dev \ --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" \ ${exported_volumes:+"$exported_volumes"} \ "$CTR_IMAGE" \ chown -R "$(id -u):$(id -g)" "$CTR_CLH_ROOT_DIR" return "$1" } # Process exported volumes argument, separate the volumes and make docker compatible # Sample input: --volumes /a:/a#/b:/b # Sample output: --volume /a:/a --volume /b:/b # process_volumes_args() { if [ -z "$arg_vols" ]; then return fi exported_volumes="" arr_vols=("${arg_vols//#/ }") for var in "${arr_vols[@]}"; do parts=("${var//:/ }") if [[ ! -e "${parts[0]}" ]]; then echo "The volume ${parts[0]} does not exist." exit 1 fi exported_volumes="$exported_volumes --volume $var" done } cmd_help() { echo "" echo "Cloud Hypervisor $(basename "$0")" echo "Usage: $(basename "$0") [flags] []" echo "" echo "Available flags": echo "" echo " --local Set the container image version being used to \"local\"." echo "" echo "Available commands:" echo "" echo " build [--debug|--release] [--libc musl|gnu] [-- []]" echo " Build the Cloud Hypervisor binaries." echo " --debug Build the debug binaries. This is the default." echo " --release Build the release binaries." echo " --libc Select the C library Cloud Hypervisor will be built against. Default is gnu" echo " --volumes Hash separated volumes to be exported. Example --volumes /mnt:/mnt#/myvol:/myvol" echo " --hypervisor Underlying hypervisor. Options kvm, mshv" echo "" echo " tests [] [--libc musl|gnu] [-- [] [-- []]] " echo " Run the Cloud Hypervisor tests." echo " --unit Run the unit tests." echo " --integration Run the integration tests." echo " --integration-sgx Run the SGX integration tests." echo " --integration-vfio Run the VFIO integration tests." echo " --integration-windows Run the Windows guest integration tests." echo " --integration-live-migration Run the live-migration integration tests." echo " --integration-rate-limiter Run the rate-limiter integration tests." echo " --libc Select the C library Cloud Hypervisor will be built against. Default is gnu" echo " --metrics Generate performance metrics" echo " --coverage Generate code coverage information" echo " --volumes Hash separated volumes to be exported. Example --volumes /mnt:/mnt#/myvol:/myvol" echo " --hypervisor Underlying hypervisor. Options kvm, mshv" echo " --all Run all tests." echo "" echo " build-container [--type]" echo " Build the Cloud Hypervisor container." echo "" echo " clean []]" echo " Remove the Cloud Hypervisor artifacts." echo "" echo " shell" echo " Run the development container into an interactive, privileged BASH shell." echo " --volumes Hash separated volumes to be exported. Example --volumes /mnt:/mnt#/myvol:/myvol" echo "" echo " help" echo " Display this help message." echo "" } cmd_build() { build="debug" libc="gnu" hypervisor="kvm" features_build=() exported_device="/dev/kvm" while [ $# -gt 0 ]; do case "$1" in "-h" | "--help") { cmd_help exit 1 } ;; "--debug") { build="debug"; } ;; "--release") { build="release"; } ;; "--runtime") shift DOCKER_RUNTIME="$1" export DOCKER_RUNTIME ;; "--libc") shift [[ "$1" =~ ^(musl|gnu)$ ]] || die "Invalid libc: $1. Valid options are \"musl\" and \"gnu\"." libc="$1" ;; "--volumes") shift arg_vols="$1" ;; "--hypervisor") shift hypervisor="$1" ;; "--features") shift features_build=(--features "$1") ;; "--") { shift break } ;; *) die "Unknown build argument: $1. Please use --help for help." ;; esac shift done ensure_build_dir ensure_latest_ctr process_volumes_args if [[ ! ("$hypervisor" = "kvm" || "$hypervisor" = "mshv") ]]; then die "Hypervisor value must be kvm or mshv" fi if [[ "$hypervisor" = "mshv" ]]; then exported_device="/dev/mshv" fi target="$(uname -m)-unknown-linux-${libc}" cargo_args=("$@") [ $build = "release" ] && cargo_args+=("--release") cargo_args+=(--target "$target") # shellcheck disable=SC2153 rustflags="$RUSTFLAGS" target_cc="" if [ "$(uname -m)" = "aarch64" ] && [ "$libc" = "musl" ]; then rustflags="$rustflags -C link-args=-Wl,-Bstatic -C link-args=-lc" fi $DOCKER_RUNTIME run \ --user "$(id -u):$(id -g)" \ --workdir "$CTR_CLH_ROOT_DIR" \ --rm \ --volume $exported_device \ --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" \ ${exported_volumes:+"$exported_volumes"} \ --env RUSTFLAGS="$rustflags" \ --env TARGET_CC="$target_cc" \ "$CTR_IMAGE" \ cargo build --all "${features_build[@]}" \ --target-dir "$CTR_CLH_CARGO_TARGET" \ "${cargo_args[@]}" && say "Binaries placed under $CLH_CARGO_TARGET/$target/$build" } cmd_clean() { cargo_args=("$@") ensure_build_dir ensure_latest_ctr $DOCKER_RUNTIME run \ --user "$(id -u):$(id -g)" \ --workdir "$CTR_CLH_ROOT_DIR" \ --rm \ --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" \ ${exported_volumes:+"$exported_volumes"} \ "$CTR_IMAGE" \ cargo clean \ --target-dir "$CTR_CLH_CARGO_TARGET" \ "${cargo_args[@]}" } cmd_tests() { unit=false integration=false integration_sgx=false integration_vfio=false integration_windows=false integration_live_migration=false integration_rate_limiter=false metrics=false coverage=false libc="gnu" arg_vols="" hypervisor="kvm" exported_device="/dev/kvm" while [ $# -gt 0 ]; do case "$1" in "-h" | "--help") { cmd_help exit 1 } ;; "--unit") { unit=true; } ;; "--integration") { integration=true; } ;; "--integration-sgx") { integration_sgx=true; } ;; "--integration-vfio") { integration_vfio=true; } ;; "--integration-windows") { integration_windows=true; } ;; "--integration-live-migration") { integration_live_migration=true; } ;; "--integration-rate-limiter") { integration_rate_limiter=true; } ;; "--metrics") { metrics=true; } ;; "--coverage") { coverage=true; } ;; "--libc") shift [[ "$1" =~ ^(musl|gnu)$ ]] || die "Invalid libc: $1. Valid options are \"musl\" and \"gnu\"." libc="$1" ;; "--volumes") shift arg_vols="$1" ;; "--hypervisor") shift hypervisor="$1" ;; "--all") { unit=true integration=true } ;; "--") { shift break } ;; *) die "Unknown tests argument: $1. Please use --help for help." ;; esac shift done if [[ ! ("$hypervisor" = "kvm" || "$hypervisor" = "mshv") ]]; then die "Hypervisor value must be kvm or mshv" fi if [[ "$hypervisor" = "mshv" ]]; then exported_device="/dev/mshv" fi if [ ! -e "${exported_device}" ]; then die "${exported_device} does not exist on the system" fi set -- '--hypervisor' "$hypervisor" "$@" ensure_build_dir ensure_latest_ctr process_volumes_args target="$(uname -m)-unknown-linux-${libc}" rustflags="$RUSTFLAGS" target_cc="" if [ "$(uname -m)" = "aarch64" ] && [ "$libc" = "musl" ]; then rustflags="$rustflags -C link-args=-Wl,-Bstatic -C link-args=-lc" fi if [[ "$unit" = true ]]; then say "Running unit tests for $target..." $DOCKER_RUNTIME run \ --workdir "$CTR_CLH_ROOT_DIR" \ --rm \ --device $exported_device \ --device /dev/net/tun \ --cap-add net_admin \ --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" \ ${exported_volumes:+"$exported_volumes"} \ --env BUILD_TARGET="$target" \ --env RUSTFLAGS="$rustflags" \ --env TARGET_CC="$target_cc" \ --env LLVM_PROFILE_FILE="$LLVM_PROFILE_FILE" \ "$CTR_IMAGE" \ ./scripts/run_unit_tests.sh "$@" || fix_dir_perms $? || exit $? fi if [ "$integration" = true ]; then say "Running integration tests for $target..." $DOCKER_RUNTIME run \ --workdir "$CTR_CLH_ROOT_DIR" \ --rm \ --privileged \ --security-opt seccomp=unconfined \ --ipc=host \ --net="$CTR_CLH_NET" \ --mount type=tmpfs,destination=/tmp \ --volume /dev:/dev \ --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" \ ${exported_volumes:+"$exported_volumes"} \ --volume "$CLH_INTEGRATION_WORKLOADS:$CTR_CLH_INTEGRATION_WORKLOADS" \ --env USER="root" \ --env BUILD_TARGET="$target" \ --env RUSTFLAGS="$rustflags" \ --env TARGET_CC="$target_cc" \ --env AUTH_DOWNLOAD_TOKEN="$AUTH_DOWNLOAD_TOKEN" \ --env LLVM_PROFILE_FILE="$LLVM_PROFILE_FILE" \ "$CTR_IMAGE" \ dbus-run-session ./scripts/run_integration_tests_"$(uname -m)".sh "$@" || fix_dir_perms $? || exit $? fi if [ "$integration_sgx" = true ]; then say "Running SGX integration tests for $target..." $DOCKER_RUNTIME run \ --workdir "$CTR_CLH_ROOT_DIR" \ --rm \ --privileged \ --security-opt seccomp=unconfined \ --ipc=host \ --net="$CTR_CLH_NET" \ --mount type=tmpfs,destination=/tmp \ --volume /dev:/dev \ --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" \ ${exported_volumes:+"$exported_volumes"} \ --volume "$CLH_INTEGRATION_WORKLOADS:$CTR_CLH_INTEGRATION_WORKLOADS" \ --env USER="root" \ --env BUILD_TARGET="$target" \ --env RUSTFLAGS="$rustflags" \ --env TARGET_CC="$target_cc" \ --env AUTH_DOWNLOAD_TOKEN="$AUTH_DOWNLOAD_TOKEN" \ "$CTR_IMAGE" \ ./scripts/run_integration_tests_sgx.sh "$@" || fix_dir_perms $? || exit $? fi if [ "$integration_vfio" = true ]; then say "Running VFIO integration tests for $target..." $DOCKER_RUNTIME run \ --workdir "$CTR_CLH_ROOT_DIR" \ --rm \ --privileged \ --security-opt seccomp=unconfined \ --ipc=host \ --net="$CTR_CLH_NET" \ --mount type=tmpfs,destination=/tmp \ --volume /dev:/dev \ --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" \ ${exported_volumes:+"$exported_volumes"} \ --volume "$CLH_INTEGRATION_WORKLOADS:$CTR_CLH_INTEGRATION_WORKLOADS" \ --env USER="root" \ --env BUILD_TARGET="$target" \ --env RUSTFLAGS="$rustflags" \ --env TARGET_CC="$target_cc" \ --env AUTH_DOWNLOAD_TOKEN="$AUTH_DOWNLOAD_TOKEN" \ "$CTR_IMAGE" \ ./scripts/run_integration_tests_vfio.sh "$@" || fix_dir_perms $? || exit $? fi if [ "$integration_windows" = true ]; then say "Running Windows integration tests for $target..." $DOCKER_RUNTIME run \ --workdir "$CTR_CLH_ROOT_DIR" \ --rm \ --privileged \ --security-opt seccomp=unconfined \ --ipc=host \ --net="$CTR_CLH_NET" \ --mount type=tmpfs,destination=/tmp \ --volume /dev:/dev \ --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" \ ${exported_volumes:+"$exported_volumes"} \ --volume "$CLH_INTEGRATION_WORKLOADS:$CTR_CLH_INTEGRATION_WORKLOADS" \ --env USER="root" \ --env BUILD_TARGET="$target" \ --env RUSTFLAGS="$rustflags" \ --env TARGET_CC="$target_cc" \ --env AUTH_DOWNLOAD_TOKEN="$AUTH_DOWNLOAD_TOKEN" \ "$CTR_IMAGE" \ ./scripts/run_integration_tests_windows_"$(uname -m)".sh "$@" || fix_dir_perms $? || exit $? fi if [ "$integration_live_migration" = true ]; then say "Running 'live migration' integration tests for $target..." $DOCKER_RUNTIME run \ --workdir "$CTR_CLH_ROOT_DIR" \ --rm \ --privileged \ --security-opt seccomp=unconfined \ --ipc=host \ --net="$CTR_CLH_NET" \ --mount type=tmpfs,destination=/tmp \ --volume /dev:/dev \ --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" \ ${exported_volumes:+"$exported_volumes"} \ --volume "$CLH_INTEGRATION_WORKLOADS:$CTR_CLH_INTEGRATION_WORKLOADS" \ --env USER="root" \ --env BUILD_TARGET="$target" \ --env RUSTFLAGS="$rustflags" \ --env TARGET_CC="$target_cc" \ --env AUTH_DOWNLOAD_TOKEN="$AUTH_DOWNLOAD_TOKEN" \ --env LLVM_PROFILE_FILE="$LLVM_PROFILE_FILE" \ "$CTR_IMAGE" \ ./scripts/run_integration_tests_live_migration.sh "$@" || fix_dir_perms $? || exit $? fi if [ "$integration_rate_limiter" = true ]; then say "Running 'rate limiter' integration tests for $target..." $DOCKER_RUNTIME run \ --workdir "$CTR_CLH_ROOT_DIR" \ --rm \ --privileged \ --security-opt seccomp=unconfined \ --ipc=host \ --net="$CTR_CLH_NET" \ --mount type=tmpfs,destination=/tmp \ --volume /dev:/dev \ --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" \ ${exported_volumes:+"$exported_volumes"} \ --volume "$CLH_INTEGRATION_WORKLOADS:$CTR_CLH_INTEGRATION_WORKLOADS" \ --env USER="root" \ --env BUILD_TARGET="$target" \ --env RUSTFLAGS="$rustflags" \ --env TARGET_CC="$target_cc" \ --env AUTH_DOWNLOAD_TOKEN="$AUTH_DOWNLOAD_TOKEN" \ "$CTR_IMAGE" \ ./scripts/run_integration_tests_rate_limiter.sh "$@" || fix_dir_perms $? || exit $? fi if [ "$metrics" = true ]; then say "Generating performance metrics for $target..." $DOCKER_RUNTIME run \ --workdir "$CTR_CLH_ROOT_DIR" \ --rm \ --privileged \ --security-opt seccomp=unconfined \ --ipc=host \ --net="$CTR_CLH_NET" \ --mount type=tmpfs,destination=/tmp \ --volume /dev:/dev \ --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" \ ${exported_volumes:+"$exported_volumes"} \ --volume "$CLH_INTEGRATION_WORKLOADS:$CTR_CLH_INTEGRATION_WORKLOADS" \ --env USER="root" \ --env BUILD_TARGET="$target" \ --env RUSTFLAGS="$rustflags" \ --env TARGET_CC="$target_cc" \ --env RUST_BACKTRACE="${RUST_BACKTRACE}" \ --env AUTH_DOWNLOAD_TOKEN="$AUTH_DOWNLOAD_TOKEN" \ "$CTR_IMAGE" \ ./scripts/run_metrics.sh "$@" || fix_dir_perms $? || exit $? fi if [ "$coverage" = true ]; then say "Generating code coverage information for $target..." $DOCKER_RUNTIME run \ --workdir "$CTR_CLH_ROOT_DIR" \ --rm \ --privileged \ --security-opt seccomp=unconfined \ --ipc=host \ --net="$CTR_CLH_NET" \ --mount type=tmpfs,destination=/tmp \ --volume /dev:/dev \ --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" \ ${exported_volumes:+"$exported_volumes"} \ --volume "$CLH_INTEGRATION_WORKLOADS:$CTR_CLH_INTEGRATION_WORKLOADS" \ --env USER="root" \ --env BUILD_TARGET="$target" \ --env RUSTFLAGS="$rustflags" \ --env TARGET_CC="$target_cc" \ --env AUTH_DOWNLOAD_TOKEN="$AUTH_DOWNLOAD_TOKEN" \ "$CTR_IMAGE" \ dbus-run-session ./scripts/run_coverage.sh "$@" || fix_dir_perms $? || exit $? fi fix_dir_perms $? } build_container() { ensure_build_dir BUILD_DIR=/tmp/cloud-hypervisor/container/ mkdir -p $BUILD_DIR cp "$CLH_DOCKERFILE" $BUILD_DIR [ "$(uname -m)" = "aarch64" ] && TARGETARCH="arm64" [ "$(uname -m)" = "x86_64" ] && TARGETARCH="amd64" $DOCKER_RUNTIME build \ --target dev \ -t "$CTR_IMAGE" \ -f $BUILD_DIR/Dockerfile \ --build-arg TARGETARCH="$TARGETARCH" \ $BUILD_DIR } cmd_build-container() { while [ $# -gt 0 ]; do case "$1" in "-h" | "--help") { cmd_help exit 1 } ;; "--") { shift break } ;; *) die "Unknown build-container argument: $1. Please use --help for help." ;; esac shift done build_container } cmd_shell() { while [ $# -gt 0 ]; do case "$1" in "-h" | "--help") { cmd_help exit 1 } ;; "--volumes") shift arg_vols="$1" ;; "--") { shift break } ;; *) ;; esac shift done ensure_build_dir ensure_latest_ctr process_volumes_args say_warn "Starting a privileged shell prompt as root ..." say_warn "WARNING: Your $CLH_ROOT_DIR folder will be bind-mounted in the container under $CTR_CLH_ROOT_DIR" $DOCKER_RUNTIME run \ -ti \ --workdir "$CTR_CLH_ROOT_DIR" \ --rm \ --privileged \ --security-opt seccomp=unconfined \ --ipc=host \ --net="$CTR_CLH_NET" \ --tmpfs /tmp:exec \ --volume /dev:/dev \ --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" \ ${exported_volumes:+"$exported_volumes"} \ --volume "$CLH_INTEGRATION_WORKLOADS:$CTR_CLH_INTEGRATION_WORKLOADS" \ --env USER="root" \ --entrypoint bash \ "$CTR_IMAGE" fix_dir_perms $? } if [ $# = 0 ]; then cmd_help say_err "Please specify command to run!" exit 1 fi # Parse main command line args. # while [ $# -gt 0 ]; do case "$1" in -h | --help) { cmd_help exit 1 } ;; --local) { CTR_IMAGE_VERSION="local" CTR_IMAGE="${CTR_IMAGE_TAG}:${CTR_IMAGE_VERSION}" } ;; -*) die "Unknown arg: $1. Please use \`$0 help\` for help." ;; *) break ;; esac shift done # $1 is now a command name. Check if it is a valid command and, if so, # run it. # declare -f "cmd_$1" >/dev/null ok_or_die "Unknown command: $1. Please use \`$0 help\` for help." cmd=cmd_$1 shift $cmd "$@"