#!/usr/bin/env bash
# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
# SPDX-License-Identifier: Apache-2.0
# Firecracker devtool
#
# Use this script to build and test Firecracker.
#
# TL;DR
# Make sure you have Docker installed and properly configured
# (http://docker.com). Then,
# building: `./devtool build`
# Then find the binaries under build/debug/
# testing: `./devtool test`
# Will run the entire test battery; will take several minutes to complete.
# deep-dive: `./devtool shell`
# Open a shell prompt inside the container. Then build or test (or do
# anything, really) manually.
#
# Still TL;DR: have Docker; ./devtool build; ./devtool test; ./devtool help.
#
#
# Both building and testing are done inside a Docker container. Please make sure
# you have Docker up and running on your system (see http:/docker.com) and your
# user has permission to run Docker containers.
#
# The Firecracker sources dir will be bind-mounted inside the development
# container (under /firecracker) and any files generated by the build process
# will show up under the build/ dir. This includes the final binaries, as well
# as any intermediate or cache files.
#
# By default, all devtool commands run the container transparently, removing
# it after the command completes. Any persisting files will be stored under
# build/.
# If, for any reason, you want to access the container directly, please use
# `devtool shell`. This will perform the initial setup (bind-mounting the
# sources dir, setting privileges) and will then drop into a BASH shell inside
# the container.
#
# Building:
# Run `./devtool build`.
# By default, the debug binaries are built and placed under build/debug/.
# To build the release version, run `./devtool build --release` instead.
# You can then find the binaries under build/release/.
#
# Testing:
# Run `./devtool test`.
# This will run the entire integration test battery. The testing system is
# based on pytest (http://pytest.org).
#
# Opening a shell prompt inside the development container:
# Run `./devtool shell`.
#
# Additional information:
# Run `./devtool help`.
#
#
# TODO:
# - Cache test binaries, preserving them across `./devtool test` invocations.
# At the moment, Firecracker is rebuilt everytime a test is run.
# - List tests by parsing the `pytest --collect-only` output.
# - Implement test filtering with `./devtool test --filter <filter>`
# - Find an easier way to run individual tests on existing binaries.
# - Add a `./devtool run` command to set up and run Firecracker.
# - Add a `./devtool diag` command to help with troubleshooting, by checking
# the most common failure conditions.
# - Look into caching the Cargo registry within the container and if that
# would help with reproducible builds (in addition to pinning Cargo.lock)
# Development container image (without tag)
DEVCTR_IMAGE_NO_TAG="public.ecr.aws/firecracker/fcuvm"
# Development container tag
DEVCTR_IMAGE_TAG="v30"
# Development container image (name:tag)
# This should be updated whenever we upgrade the development container.
# (Yet another step on our way to reproducible builds.)
DEVCTR_IMAGE="${DEVCTR_IMAGE_NO_TAG}:${DEVCTR_IMAGE_TAG}"
# Naming things is hard
MY_NAME="Firecracker $(basename "$0")"
# Full path to the Firecracker tools dir on the host.
FC_TOOLS_DIR=$(cd "$(dirname "$0")" && pwd)
# Full path to the Firecracker sources dir on the host.
FC_ROOT_DIR=$(cd "${FC_TOOLS_DIR}/.." && pwd)
# Full path to the build dir on the host.
FC_BUILD_DIR="${FC_ROOT_DIR}/build"
# Full path to devctr dir on the host.
FC_DEVCTR_DIR="${FC_ROOT_DIR}/tools/devctr"
# Path to the linux kernel directory on the host.
KERNEL_DIR="${FC_ROOT_DIR}/.kernel"
# Full path to the cargo registry dir on the host. This appears on the host
# because we want to persist the cargo registry across container invocations.
# Otherwise, any rust crates from crates.io would be downloaded again each time
# we build or test.
CARGO_REGISTRY_DIR="${FC_BUILD_DIR}/cargo_registry"
# Full path to the cargo git registry on the host. This serves the same purpose
# as CARGO_REGISTRY_DIR, for crates downloaded from GitHub repos instead of
# crates.io.
CARGO_GIT_REGISTRY_DIR="${FC_BUILD_DIR}/cargo_git_registry"
# Full path to the cargo target dir on the host.
CARGO_TARGET_DIR="${FC_BUILD_DIR}/cargo_target"
# Full path to the seccompiler cargo target dir on the host.
CARGO_SECCOMPILER_TARGET_DIR="${FC_BUILD_DIR}/seccompiler"
# Full path to the Firecracker sources dir, as bind-mounted in the container.
CTR_FC_ROOT_DIR="/firecracker"
# Full path to the build dir, as bind-mounted in the container.
CTR_FC_BUILD_DIR="${CTR_FC_ROOT_DIR}/build"
# Full path to the cargo target dir, as bind-mounted in the container.
CTR_CARGO_TARGET_DIR="$CTR_FC_BUILD_DIR/cargo_target"
# Full path to the seccompiler cargo target dir, as bind-mounted in the container.
CTR_CARGO_SECCOMPILER_TARGET_DIR="$CTR_FC_BUILD_DIR/seccompiler"
# Full path to the microVM images cache dir
CTR_MICROVM_IMAGES_DIR="$CTR_FC_BUILD_DIR/img"
# Full path to the public key mapping on the guest
PUB_KEY_PATH=/root/.ssh/id_rsa.pub
# Full path to the private key mapping on the guest
PRIV_KEY_PATH=/root/.ssh/id_rsa
# Path to the linux kernel directory, as bind-mounted in the container.
CTR_KERNEL_DIR="${CTR_FC_ROOT_DIR}/.kernel"
# Global options received by $0
# These options are not command-specific, so we store them as global vars
OPT_UNATTENDED=false
# Get the target prefix to avoid repeated calls to uname -m
TARGET_PREFIX="$(uname -m)-unknown-linux-"
DEFAULT_TEST_SESSION_ROOT_PATH=/srv
DEFAULT_RAMDISK_PATH=/mnt/devtool-ramdisk
# Send a decorated message to stdout, followed by a new line
#
say() {
[ -t 1 ] && [ -n "$TERM" ] \
&& echo "$(tput setaf 2)[$MY_NAME]$(tput sgr0) $*" \
|| echo "[$MY_NAME] $*"
}
# Send a decorated message to stdout, without a trailing new line
#
say_noln() {
[ -t 1 ] && [ -n "$TERM" ] \
&& echo -n "$(tput setaf 2)[$MY_NAME]$(tput sgr0) $*" \
|| echo "[$MY_NAME] $*"
}
# Send a text message to stderr
#
say_err() {
[ -t 2 ] && [ -n "$TERM" ] \
&& echo -e "$(tput setaf 1)[$MY_NAME] $*$(tput sgr0)" 1>&2 \
|| echo -e "[$MY_NAME] $*" 1>&2
}
# Send a warning-highlighted text to stdout
say_warn() {
[ -t 1 ] && [ -n "$TERM" ] \
&& echo "$(tput setaf 3)[$MY_NAME] $*$(tput sgr0)" \
|| echo "[$MY_NAME] $*"
}
# Exit with an error message and (optional) code
# Usage: die [-c <error code>] <error message>
#
die() {
code=1
[[ "$1" = "-c" ]] && {
code="$2"
shift 2
}
say_err "$@"
exit $code
}
# Exit with an error message if the last exit code is not 0
#
ok_or_die() {
code=$?
[[ $code -eq 0 ]] || die -c $code "$@"
}
# Check if Docker is available and exit if it's not.
# Upon returning from this call, the caller can be certain Docker is available.
#
ensure_docker() {
NEWLINE=$'\n'
output=$(which docker 2>&1)
ok_or_die "Docker not found. Aborting." \
"Please make sure you have Docker (http://docker.com) installed" \
"and properly configured.${NEWLINE}" \
"Error: $?, command output: ${output}"
output=$(docker ps 2>&1)
ok_or_die "Error accessing Docker. Please make sure the Docker daemon" \
"is running and that you are part of the docker group.${NEWLINE}" \
"Error: $?, command output: ${output}${NEWLINE}" \
"For more information, see" \
"https://docs.docker.com/install/linux/linux-postinstall/"
}
# Run a command and retry multiple times if it fails. Once it stops
# failing return to normal execution. If there are "retry count"
# failures, set the last error code.
# $1 - command
# $2 - retry count
# $3 - sleep interval between retries
retry_cmd() {
command=$1
retry_cnt=$2
sleep_int=$3
{
$command
} || {
# Command failed, substract one from retry_cnt
retry_cnt=$((retry_cnt - 1))
# If retry_cnt is larger than 0, sleep and call again
if [ "$retry_cnt" -gt 0 ]; then
echo "$command failed, retrying..."
sleep "$sleep_int"
retry_cmd "$command" "$retry_cnt" "$sleep_int"
fi
}
}
# Attempt to download our Docker image. Exit if that fails.
# Upon returning from this call, the caller can be certain our Docker image is
# available on this system.
#
ensure_devctr() {
# We depend on having Docker present.
ensure_docker
# Check if we have the container image available locally. Attempt to
# download it, if we don't.
[[ $(docker images -q "$DEVCTR_IMAGE" | wc -l) -gt 0 ]] || {
say "About to pull docker image $DEVCTR_IMAGE"
get_user_confirmation || die "Aborted."
# Run docker pull 5 times in case it fails - sleep 3 seconds
# between attempts
retry_cmd "docker pull $DEVCTR_IMAGE" 5 3
ok_or_die "Error pulling docker image. Aborting."
}
}
# Check if /dev/kvm exists. Exit if it doesn't.
# Upon returning from this call, the caller can be certain /dev/kvm is
# available.
#
ensure_kvm() {
[[ -c /dev/kvm ]] || die "/dev/kvm not found. Aborting."
}
# Make sure the build/ dirs are available. Exit if we can't create them.
# Upon returning from this call, the caller can be certain the build/ dirs exist.
#
ensure_build_dir() {
for dir in "$FC_BUILD_DIR" "$CARGO_TARGET_DIR" \
"$CARGO_REGISTRY_DIR" "$CARGO_GIT_REGISTRY_DIR"; do
mkdir -p "$dir" || die "Error: cannot create dir $dir"
[ -x "$dir" ] && [ -w "$dir" ] || \
{
say "Wrong permissions for $dir. Attempting to fix them ..."
chmod +x+w "$dir"
} || \
die "Error: wrong permissions for $dir. Should be +x+w"
done
}
# Makes sure that Firecracker release binaries were built.
# This is relevant in the context of stripping the already built release binaries.
ensure_release_binaries_exist() {
target=$1
profile=$2
firecracker_bin_path="$CARGO_TARGET_DIR/$target/$profile/firecracker"
jailer_bin_path="$CARGO_TARGET_DIR/$target/$profile/jailer"
seccompiler_bin_path="$CARGO_SECCOMPILER_TARGET_DIR/$target/$profile/seccompiler-bin"
# Both binaries must exist for the stripping to be successful.
[ -f "$firecracker_bin_path" ] && [ -f "$jailer_bin_path" ] && [ -f "$seccompiler_bin_path" ] || \
die "Missing release binaries. Needed files:\n" \
"* $firecracker_bin_path\n" \
"* $jailer_bin_path\n" \
"* $seccompiler_bin_path\n" \
"To build the binaries, run:\n\t$0 build --$profile"
}
# Fix build/ dir permissions after a privileged container run.
# Since the privileged container runs as root, any files it creates will be
# owned by root. This fixes that by recursively changing the ownership of build/
# to the current user.
#
cmd_fix_perms() {
# Yes, running Docker to get elevated privileges, just to chown some files
# is a dirty hack.
run_devctr \
-- \
chown -R "$(id -u):$(id -g)" "$CTR_FC_BUILD_DIR"
}
# Builds the development container from its Dockerfile.
#
cmd_build_devctr() {
arch=$(uname -m)
docker_file_name="Dockerfile.$arch"
should_update_python_packages=true
build_args=""
while [ $# -gt 0 ]; do
case "$1" in
"-h"|"--help") { cmd_help; exit 1; } ;;
"-n"|"--no-python-package-update")
shift
should_update_python_packages=false
build_args="--build-arg POETRY_LOCK_PATH=tools/devctr/poetry.lock"
;;
"--") { shift; break; } ;;
*)
die "Unknown argument: $1. Please use --help for help."
;;
esac
shift
done
docker build -t "$DEVCTR_IMAGE_NO_TAG" -f "$FC_DEVCTR_DIR/$docker_file_name" $build_args .
if [ "$should_update_python_packages" = true ]; then
update_python_package_version
fi
}
# Prompt the user for confirmation before proceeding.
# Args:
# $1 prompt text.
# Default: Continue? (y/n)
# $2 confirmation input.
# Default: y
# Return:
# exit code 0 for successful confirmation
# exit code != 0 if the user declined
#
get_user_confirmation() {
# Pass if running unattended
[[ "$OPT_UNATTENDED" = true ]] && return 0
# Fail if STDIN is not a terminal (there's no user to confirm anything)
[[ -t 0 ]] || return 1
# Otherwise, ask the user
#
msg=$([ -n "$1" ] && echo -n "$1" || echo -n "Continue? (y/n) ")
yes=$([ -n "$2" ] && echo -n "$2" || echo -n "y")
say_noln "$msg"
read c && [ "$c" = "$yes" ] && return 0
return 1
}
# Validate the user supplied version number.
# It must be composed of 3 groups of integers separated by dot.
#
validate_version() {
declare version_regex="^([0-9]+.){2}[0-9]+$"
version="$1"
if [ -z "$version" ]; then
die "Version cannot be empty."
elif [[ ! "$version" =~ $version_regex ]]; then
die "Invalid version number: $version (expected: \$Major.\$Minor.\$Build)."
fi
}
# Validate the user supplied kernel version number.
# It must be composed of 2 groups of integers separated by dot, with an optional third group.
validate_kernel_version() {
local version_regex="^([0-9]+.)[0-9]+(.[0-9]+)?$"
version="$1"
if [ -z "$version" ]; then
die "Version cannot be empty."
elif [[ ! "$version" =~ $version_regex ]]; then
die "Invalid version number: $version (expected: \$Major.\$Minor.\$Patch(optional))."
fi
}
# Compose the text for a new release tag using the information in the changelog,
# between the two specified releases.
# The following transformations are applied:
# * `-` is replaced with `*` for unnumbered lists.
# * section headers (`###`) are removed.
#
# Args:
# $1 previous version.
# $2 new version.
#
compose_tag_text() {
declare prev_ver="$1"
declare curr_ver="$2"
declare changelog="$FC_ROOT_DIR/CHANGELOG.md"
validate_version "$prev_ver"
validate_version "$curr_ver"
# Patterns for the sections in the changelog corresponding to the versions.
pat_prev="^##\s\[$prev_ver\]"
pat_curr="^##\s\[$curr_ver\]"
# Extract the section enclosed between the 2 headers and strip off the first
# 2 and last 2 lines (one is blank and one contains the header `## [A.B.C]`).
# Then, replace `-` with `*` and remove section headers.
sed "/$pat_curr/,/$pat_prev/!d" "$changelog" \
| sed '1,2d;$d' \
| sed "s/^-/*/g" \
| sed "s/^###\s//g"
}
# Helper function to run the dev container.
# Usage: run_devctr <docker args> -- <container args>
# Example: run_devctr --privileged -- bash -c "echo 'hello world'"
run_devctr() {
docker_args=()
ctr_args=()
docker_args_done=false
while [[ $# -gt 0 ]]; do
[[ "$1" = "--" ]] && {
docker_args_done=true
shift
continue
}
[[ $docker_args_done = true ]] && ctr_args+=("$1") || docker_args+=("$1")
shift
done
# If we're running in a terminal, pass the terminal to Docker and run
# the container interactively
[[ -t 0 ]] && docker_args+=("-i")
[[ -t 1 ]] && docker_args+=("-t")
# Try to pass these environments from host into container for network proxies
proxies=(http_proxy HTTP_PROXY https_proxy HTTPS_PROXY no_proxy NO_PROXY)
for i in "${proxies[@]}"; do
if [[ ! -z ${!i} ]]; then
docker_args+=("--env") && docker_args+=("$i=${!i}")
fi
done
# Finally, run the dev container
# Use 'z' on the --volume parameter for docker to automatically relabel the
# content and allow sharing between containers.
docker run "${docker_args[@]}" \
--rm \
--volume /dev:/dev \
--volume "$FC_ROOT_DIR:$CTR_FC_ROOT_DIR:z" \
--env OPT_LOCAL_IMAGES_PATH="$(dirname "$CTR_MICROVM_IMAGES_DIR")" \
--env PYTHONDONTWRITEBYTECODE=1 \
"$DEVCTR_IMAGE" "${ctr_args[@]}"
}
# Helper function to test that the argument provided is a valid path to a SSH key.
#
test_key() {
ssh-keygen -lf "$1" &>/dev/null
ret=$?
[ $ret -ne 0 ] && die "$1 is not a valid key file."
}
# Tries to update any outdated python packages on the container,
# locks the new versions and copies back the ```poetry.lock```
# file to the host.
#
update_python_package_version() {
say "Updating python packages..."
# defined in Dockerfile
poetry_dir_on_container="/tmp/poetry"
lock_file_location_on_host="$FC_DEVCTR_DIR/poetry.lock"
image_id=$(docker images -q "$DEVCTR_IMAGE_NO_TAG" | head -n 1)
dummy_container_name=$(uuidgen)
dummy_container_id=$(docker create -ti --name \
"$dummy_container_name" \
"$image_id" \
bash)
docker start "$dummy_container_id"
cmd="cd "$poetry_dir_on_container"; poetry update"
docker exec -ti "$dummy_container_id" /bin/bash -c "${cmd}"
docker cp \
"$dummy_container_id":/tmp/poetry/poetry.lock \
"$lock_file_location_on_host"
docker commit "$dummy_container_id" "$DEVCTR_IMAGE_NO_TAG"
docker stop "$dummy_container_id"
docker rm -f "$dummy_container_name"
}
# `$0 help`
# Show the detailed devtool usage information.
#
cmd_help() {
echo ""
echo "Firecracker $(basename $0)"
echo "Usage: $(basename $0) [<args>] <command> [<command args>]"
echo ""
echo "Global arguments"
echo " -y, --unattended Run unattended. Assume the user would always"
echo " answer \"yes\" to any confirmation prompt."
echo ""
echo "Available commands:"
echo ""
echo " build [--debug|--release] [-l|--libc musl|gnu] [-- [<cargo args>]]"
echo " Build the Firecracker binaries."
echo " Firecracker is built using the Rust build system (cargo). All arguments after --"
echo " will be passed through to cargo."
echo " --debug Build the debug binaries. This is the default."
echo " --release Build the release binaries."
echo " -l, --libc musl|gnu Choose the libc flavor against which Firecracker will"
echo " be linked. Default is musl."
echo " --ssh-keys Provide the paths to the public and private SSH keys on the host"
echo " (in this particular order) required for the git authentication."
echo " It is mandatory that both keys are specified."
echo ""
echo " build_devctr [--no-python-package-update]"
echo " Builds the development container from its Dockerfile."
echo " -n, --no-python-package-update Do not update python packages."
echo ""
echo " checkenv"
echo " Performs prerequisites checks needed to execute firecracker."
echo ""
echo " ci"
echo " Run a continuous integration test run that executes the integration tests and"
echo " checks that the release process works."
echo ""
echo " distclean"
echo " Clean up the build tree and remove the docker container."
echo ""
echo " fix_perms"
echo " Fixes permissions when devtool dies in the middle of a privileged session."
echo ""
echo " fmt"
echo " Auto-format all Rust source files, to match the Firecracker requirements."
echo " This should be used as the last step in every commit, to ensure that the"
echo " Rust style tests pass."
echo ""
echo " generate_syscall_tables <version>"
echo " Generates the syscall tables for seccompiler, according to a given kernel version."
echo " Release candidate (rc) linux versions are not allowed."
echo " Outputs a rust file for each supported arch: src/seccompiler/src/syscall_table/{arch}.rs"
echo " Supported architectures: x86_64 and aarch64."
echo ""
echo " install [-p|--path] [--debug|--release]"
echo " Install firecracker, jailer and seccomp binaries to /usr/local/bin or a given path."
echo " Only the musl linked binaries are supported."
echo " --path Install binaries to a specified path."
echo " --debug Install the debug binaries."
echo " --release Install the release binaries. This is the default."
echo ""
echo " help"
echo " Display this help message."
echo ""
echo " prepare_release <version>"
echo " Prepare a new Firecracker release by updating the version number, crate "
echo " dependencies and credits."
echo ""
echo " shell [--privileged]"
echo " Launch the development container and open an interactive BASH shell."
echo " -p, --privileged Run the container as root, in privileged mode."
echo " Running Firecracker via the jailer requires elevated"
echo " privileges, though the build phase does not."
echo ""
echo " tag <version>"
echo " Create a git tag for the specified version. The tag message will contain "
echo " the contents of CHANGELOG.md enclosed between the header corresponding to "
echo " the specified version and the one corresponding to the previous version."
echo ""
echo " test [-- [<pytest args>]]"
echo " Run the Firecracker integration tests."
echo " The Firecracker testing system is based on pytest. All arguments after --"
echo " will be passed through to pytest."
echo " -c, --cpuset-cpus cpulist Set a dedicated cpulist to be used by the tests."
echo " -m, --cpuset-mems memlist Set a dedicated memlist to be used by the tests."
echo " -r, --ramdisk size[k|m|g] Use a ramdisk of `size` MB for
the entire test session (e.g
stored artifacts, Firecracker
binaries, logs/metrics FIFOs
and test created device files)."
echo ""
echo " strip"
echo " Strip debug symbols from the Firecracker release binaries."
echo ""
}
# `$0 build` - build Firecracker
# Please see `$0 help` for more information.
#
cmd_build() {
# By default, we'll build the debug binaries.
profile="debug"
libc="musl"
# Parse any command line args.
while [ $# -gt 0 ]; do
case "$1" in
"-h"|"--help") { cmd_help; exit 1; } ;;
"--debug") { profile="debug"; } ;;
"--release") { profile="release"; } ;;
"--ssh-keys")
shift
[[ -z "$1" ]] && \
die "Please provide the path to the public SSH key."
[[ ! -f "$1" ]] && die "The public key file does not exist: $1."
test_key "$1"
host_pub_key_path="$1"
shift
[[ -z "$1" ]] && \
die "Please provide the path to the private SSH key."
[[ ! -f "$1" ]] && die "The private key file does not exist: $1."
test_key "$1"
host_priv_key_path="$1"
;;
"-l"|"--libc")
shift
[[ "$1" =~ ^(musl|gnu)$ ]] || \
die "Invalid libc: $1. Valid options are \"musl\" and \"gnu\"."
libc="$1"
;;
"--") { shift; break; } ;;
*)
die "Unknown argument: $1. Please use --help for help."
;;
esac
shift
done
target="$TARGET_PREFIX${libc}"
# Check prerequisites
ensure_devctr
ensure_build_dir
say "Starting build ($profile, $libc) ..."
# Cargo uses the debug profile by default. If we're building the release
# binaries, we need to pass an extra argument to cargo.
cargo_args=("$@")
# Add the default target if we did not get that argument in the build command.
add_default_target=true
for flag in "${@}"; do
if [[ "$flag" == "--" ]]; then
break
elif [[ "$flag" == "--target" || "$flag" =~ --target=.* ]]; then
add_default_target=false
fi
done
if [ "$add_default_target" = true ]; then
cargo_args+=(--target "$target")
fi
[ $profile = "release" ] && cargo_args+=("--release")
# Map the public and private keys to the guest if they are specified.
[ ! -z "$host_pub_key_path" ] && [ ! -z "$host_priv_key_path" ] &&
extra_args="--volume $host_pub_key_path:$PUB_KEY_PATH:z \
--volume $host_priv_key_path:$PRIV_KEY_PATH:z"
# Artificially trigger a re-run of the build script,
# to make sure that `firecracker --version` reports the latest changes.
touch "$FC_ROOT_DIR/build.rs"
# Run the cargo build process inside the container.
# We don't need any special privileges for the build phase, so we run the
# container as the current user/group.
# Build seccompiler-bin.
run_devctr \
--user "$(id -u):$(id -g)" \
--workdir "$CTR_FC_ROOT_DIR" \
${extra_args} \
-- \
cargo build -p seccompiler --bin seccompiler-bin \
--target-dir "$CTR_CARGO_SECCOMPILER_TARGET_DIR" \
"${cargo_args[@]}"
ret=$?
[ $ret -ne 0 ] && return $ret
# Build Firecracker.
run_devctr \
--user "$(id -u):$(id -g)" \
--workdir "$CTR_FC_ROOT_DIR" \
${extra_args} \
-- \
cargo build \
--target-dir "$CTR_CARGO_TARGET_DIR" \
"${cargo_args[@]}"
ret=$?
[ $ret -ne 0 ] && return $ret
# Build jailer only in case of musl for compatibility reasons.
if [ "$libc" == "musl" ];then
run_devctr \
--user "$(id -u):$(id -g)" \
--workdir "$CTR_FC_ROOT_DIR" \
${extra_args} \
-- \
cargo build -p jailer \
--target-dir "$CTR_CARGO_TARGET_DIR" \
"${cargo_args[@]}"
fi
ret=$?
# If `cargo build` was successful, output a message.
[ $ret -eq 0 ] && {
cargo_bin_dir="$CARGO_TARGET_DIR/$target/$profile"
seccompiler_bin_dir="$CARGO_SECCOMPILER_TARGET_DIR/$target/$profile"
# Seccompiler has a different build folder, we need to output two
# messages.
say "Build successful."
say "Firecracker and Jailer binaries placed under $cargo_bin_dir"
say "Seccompiler-bin binary placed under $seccompiler_bin_dir"
}
return $ret
}
cmd_distclean() {
# List of folders to remove.
dirs=("build" "test_results")
for dir in "${dirs[@]}"; do
if [ -d "$dir" ]; then
say "Removing $dir"
rm -rf "$dir"
fi
done
# Remove devctr if it exists
if [ $(docker images -q "$DEVCTR_IMAGE" | wc -l) -eq "1" ]; then
say "Removing $DEVCTR_IMAGE"
docker rmi -f "$DEVCTR_IMAGE"
fi
}
cmd_strip() {
profile="release"
target="$TARGET_PREFIX""musl"
# Parse any command line args.
while [ $# -gt 0 ]; do
case "$1" in
"-h"|"--help") { cmd_help; exit 1; } ;;
*)
die "Unknown argument: $1. Please use --help for help."
;;
esac
shift
done
# Check prerequisites
ensure_devctr
ensure_build_dir
ensure_release_binaries_exist $target $profile
say "Starting stripping the debug symbols for $profile binaries built against $target target."
strip_flags="--strip-debug"
say "Strip flags: $strip_flags."
run_devctr \
--user "$(id -u):$(id -g)" \
-- \
strip $strip_flags\
"$CTR_CARGO_TARGET_DIR/$target/$profile/firecracker" \
"$CTR_CARGO_TARGET_DIR/$target/$profile/jailer" \
"$CTR_CARGO_SECCOMPILER_TARGET_DIR/$target/$profile/seccompiler-bin"
ret=$?
[ $ret -eq 0 ] && {
say "Stripping was successful."
say "Stripped Firecracker and Jailer binaries placed under $CARGO_TARGET_DIR/$target/$profile."
say "Stripped seccompiler-bin binary placed under $CARGO_SECCOMPILER_TARGET_DIR/$target/$profile."
}
return $ret
}
mount_ramdisk() {
say "Using ramdisk ..."
local ramdisk_size="$1"
umount_ramdisk
mkdir -p ${DEFAULT_RAMDISK_PATH} && \
mount -t tmpfs -o size=${ramdisk_size} tmpfs ${DEFAULT_RAMDISK_PATH}
ok_or_die "Failed to mount ramdisk to ${DEFAULT_RAMDISK_PATH}. Check the permission."
mkdir -p ${DEFAULT_RAMDISK_PATH}/srv
mkdir -p ${DEFAULT_RAMDISK_PATH}/tmp
}
umount_ramdisk() {
if [ ! -e "${DEFAULT_RAMDISK_PATH}" ]; then
return 0
fi
if [ ! -d "${DEFAULT_RAMDISK_PATH}" ]; then
die "${DEFAULT_RAMDISK_PATH} is not a directory."
fi
if [ ! -w "${DEFAULT_RAMDISK_PATH}" ]; then
die "Failed to unmount ${DEFAULT_RAMDISK_PATH}. Check the permission."
fi
umount ${DEFAULT_RAMDISK_PATH} &>/dev/null
rmdir ${DEFAULT_RAMDISK_PATH} &>/dev/null
}
# `$0 test` - run integration tests
# Please see `$0 help` for more information.
#
cmd_test() {
# Parse any command line args.
while [ $# -gt 0 ]; do
case "$1" in
"-h"|"--help") { cmd_help; exit 1; } ;;
"-c"|"--cpuset-cpus")
shift
local cpuset_cpus="$1"
;;
"-m"|"--cpuset-mems")
shift
local cpuset_mems="$1"
;;
"-r"|"--ramdisk")
shift
local ramdisk_size="$1"
local ramdisk=true
;;
"--") { shift; break; } ;;
*)
die "Unknown argument: $1. Please use --help for help."
;;
esac
shift
done
# Check prerequisites.
ensure_kvm
ensure_devctr
ensure_build_dir
# If we got to here, we've got all we need to continue.
say "$(date -u +'%F %H:%M:%S %Z')"
say "Starting test run ..."
if [[ $ramdisk = true ]]; then
mount_ramdisk ${ramdisk_size}
ramdisk_args="--volume ${DEFAULT_RAMDISK_PATH}:${DEFAULT_TEST_SESSION_ROOT_PATH}"
fi
# Testing (running Firecracker via the jailer) needs root access,
# in order to set-up the Firecracker jail (manipulating cgroups, net
# namespaces, etc).
# We need to run a privileged container to get that kind of access.
run_devctr \
--privileged \
--security-opt seccomp=unconfined \
--ulimit core=0 \
--ulimit nofile=4096:4096 \
--workdir "$CTR_FC_ROOT_DIR/tests" \
--cpuset-cpus="$cpuset_cpus" \
--cpuset-mems="$cpuset_mems" \
${ramdisk_args} \
-- \
pytest "$@"
ret=$?
if [[ $ramdisk = true ]]; then
umount_ramdisk
fi
# Running as root would have created some root-owned files under the build
# dir. Let's fix that.
cmd_fix_perms
return $ret
}
# `$0 shell` - drop to a shell prompt inside the dev container
# Please see `$0 help` for more information.
#
cmd_shell() {
# By default, we run the container as the current user.
privileged=false
# Parse any command line args.
while [ $# -gt 0 ]; do
case "$1" in
"-h"|"--help") { cmd_help; exit 1; } ;;
"-p"|"--privileged") { privileged=true; } ;;
"-r"|"--ramdisk")
shift
local ramdisk_size="$1"
local ramdisk=true
;;
"--") { shift; break; } ;;
*)
die "Unknown argument: $1. Please use --help for help."
;;
esac
shift
done
# Make sure we have what we need to continue.
ensure_devctr
ensure_build_dir
if [[ $ramdisk = true ]]; then
mount_ramdisk ${ramdisk_size}
ramdisk_args="--volume ${DEFAULT_RAMDISK_PATH}:${DEFAULT_TEST_SESSION_ROOT_PATH}"
fi
if [[ $privileged = true ]]; then
# If requested, spin up a privileged container.
#
say "Dropping to a privileged shell prompt ..."
say "Note: $FC_ROOT_DIR is bind-mounted under $CTR_FC_ROOT_DIR"
say_warn "You are running as root; any files that get created under" \
"$CTR_FC_ROOT_DIR will be owned by root."
run_devctr \
--privileged \
--ulimit nofile=4096:4096 \
--security-opt seccomp=unconfined \
--workdir "$CTR_FC_ROOT_DIR" \
${ramdisk_args} \
-- \
bash
ret=$?
# Running as root may have created some root-owned files under the build
# dir. Let's fix that.
#
cmd_fix_perms
else
say "Dropping to shell prompt as user $(whoami) ..."
say "Note: $FC_ROOT_DIR is bind-mounted under $CTR_FC_ROOT_DIR"
say_warn "You won't be able to run Firecracker via the jailer," \
"but you can still build it."
say "You can use \`$0 shell --privileged\` to get a root shell."
[ -w /dev/kvm ] || \
say_warn "WARNING: user $(whoami) doesn't have permission to" \
"access /dev/kvm. You won't be able to run Firecracker."
run_devctr \
--user "$(id -u):$(id -g)" \
--ulimit nofile=4096:4096 \
--device=/dev/kvm:/dev/kvm \
--workdir "$CTR_FC_ROOT_DIR" \
--env PS1="$(whoami)@\h:\w\$ " \
-- \
bash --norc
ret=$?
fi
if [[ $ramdisk = true ]]; then
umount_ramdisk
fi
return $ret
}
# Auto-format all source code, to match the Firecracker requirements. For the
# moment, this is just a wrapper over `cargo fmt --all`
# Example: `devtool fmt`
#
cmd_fmt() {
# Parse any command line args.
while [ $# -gt 0 ]; do
case "$1" in
"-h"|"--help") { cmd_help; exit 1; } ;;
*)
die "Unknown argument: $1. Please use --help for help."
;;
esac
shift
done
ensure_devctr
say "Applying rustfmt ..."
run_devctr \
--user "$(id -u):$(id -g)" \
--workdir "$CTR_FC_ROOT_DIR" \
-- \
cargo fmt --all
}
# Prepare a Firecracker release by updating the version, changelog
# and credits.
# Example: `devtool release 0.42.0`
#
cmd_prepare_release() {
# Parse any command line args.
while [ $# -gt 0 ]; do
case "$1" in
"-h"|"--help") { cmd_help; exit 1; } ;;
*) { version="$1"; break; } ;;
esac
shift
done
validate_version "$version"
# We'll be needing the dev container later on.
ensure_devctr
# The cargo registry dir needs to be there for `cargo update`.
ensure_build_dir
# Get current version from the swagger spec.
swagger="$FC_ROOT_DIR/src/api_server/swagger/firecracker.yaml"
curr_ver=$(grep "version: " "$swagger" | awk -F : '{print $2}' | tr -d ' ')
say "Updating from $curr_ver to $version ..."
get_user_confirmation || die "Aborted."
# Update version in files.
files_to_change=("$swagger" \
"$FC_ROOT_DIR/src/firecracker/Cargo.toml" \
"$FC_ROOT_DIR/src/jailer/Cargo.toml" \
"$FC_ROOT_DIR/src/seccompiler/Cargo.toml")
say "Updating source files:"
for file in "${files_to_change[@]}"; do
say "- $file"
# Dirty hack to make this work on both macOS/BSD and Linux.
sed -i="" "s/$curr_ver/$version/g" "$file"
rm -f "${file}="
done
# Run `cargo check` to update firecracker and jailer versions in
# `Cargo.lock`.
say "Updating lockfile..."
run_devctr \
--user "$(id -u):$(id -g)" \
--workdir "$CTR_FC_ROOT_DIR" \
-- \
cargo check
ok_or_die "cargo check failed."
# Update credits.
say "Updating credits..."
"$FC_TOOLS_DIR/update-credits.sh"
# Update changelog.
say "Updating changelog..."
sed -i="" "s/\[Unreleased\]/\[$version\]/g" "$FC_ROOT_DIR/CHANGELOG.md"
rm -f "$FC_ROOT_DIR/CHANGELOG.md="
}
# Create a tag for the specified release.
# The tag text will be composed from the changelog contents enclosed between the
# specified release number and the previous one.
# Args:
# $1 release number.
# Example: `devtool tag 0.42.0`
#
cmd_tag() {
# Parse any command line args.
while [ $# -gt 0 ]; do
case "$1" in
"-h"|"--help") { cmd_help; exit 1; } ;;
*) { version="$1"; break; } ;;
esac
shift
done
validate_version "$version"
declare pat_release="^## \[([0-9]+\.){2}[0-9]+\]"
declare changelog="$FC_ROOT_DIR/CHANGELOG.md"
grep -q "\[$version\]" "$changelog"
ok_or_die "No changelog entry for release $version can be found."
# We work with the assumption that the changelog has already been updated
# and contains a header (and a corresponding section) for the new release.
# Step 1: Get all release numbers.
all_releases=($(grep -E "$pat_release" "$changelog"))
# Step 2: Trim out headers (`##`).
all_releases=(${all_releases[@]//##*})
# Step 3: Walk the array until we come across the desired release number,
# then pick up the next one. Since the latest releases are at the top of the
# changelog, the next one in line will be the previous one chronologically.
# The array now contains all the release numbers in the changelog, enclosed
# in square brackets.
found=
for release in "${all_releases[@]}"; do
if [ ! -z "$found" ]; then
# Trim out square brackets.
prev_version=$(echo "$release" | awk -F"[][]" "{print \$2}")
break
elif [ "$release" == "[$version]" ]; then
found=1
fi
done
# Create tag.
tag_text=$(compose_tag_text "$prev_version" "$version")
say "Preparing to create tag..."
say "Tag: v$version"
say "Tag text:"
echo "$tag_text"
say "Continue with tag creation?"
get_user_confirmation || die "Tag not created."
git tag -a "v$version" -m "$tag_text"
ok_or_die "Tag v$version not created."
say "Tag v$version created."
}
# Check if able to run firecracker.
# ../docs/getting-started.md#prerequisites
ensure_kvm_rw () {
[[ -c /dev/kvm && -w /dev/kvm && -r /dev/kvm ]] || \
say_err "FAILED: user $(whoami) doesn't have permission to" \
"access /dev/kvm."
}
check_kernver () {
KERN_MAJOR=4
KERN_MINOR=14
(uname -r | awk -v MAJOR=$KERN_MAJOR -v MINOR=$KERN_MINOR '{ split($0,kver,".");
if( (kver[1] + (kver[2] / 100) ) < MAJOR + (MINOR/100) )
{
exit 1;
} }') ||
say_err "FAILED: Kernel version must be >= $KERN_MAJOR.$KERN_MINOR"
}
# Check Production Host Setup
# ../docs/prod-host-setup.md
check_SMT () {
(grep -q "^forceoff$\|^notsupported$" \
/sys/devices/system/cpu/smt/control) ||
say_warn "WARNING: Hyperthreading ENABLED."
}
check_KPTI () {
(grep -q "^Mitigation: PTI$" \
/sys/devices/system/cpu/vulnerabilities/meltdown) || \
say_warn "WARNING: KPTI NOT SUPPORTED"
}
check_KSM () {
(grep -q "^0$" /sys/kernel/mm/ksm/run) || \
say_warn "WARNING: KSM ENABLED"
}
check_IBPB_IBRS () {
(grep -q "^Mitigation: Full generic retpoline, IBPB, IBRS_FW$"\
/sys/devices/system/cpu/vulnerabilities/spectre_v2) || \
say_warn "WARNING: retpoline, IBPB, IBRS: DISABLED."
}
check_L1TF () {
declare -a CONDITIONS=("Mitigation: PTE Inversion" "VMX: cache flushes")
for cond in "${CONDITIONS[@]}";
do (grep -q "$cond" /sys/devices/system/cpu/vulnerabilities/l1tf) ||
say_warn "WARNING: $cond: DISABLED";
done
}
check_swap () {
(grep -q "swap.img" /proc/swaps ) && \
say_warn "WARNING: SWAP enabled"
}
check_SSBD () {
arch=$(uname -m)
if [ "$arch" = "aarch64" ]; then
local param="ssbd=force-on"
elif [ "$arch" = "x86_64" ]; then
local param="spec_store_bypass_disable=on"
fi
ssbd_sysfs_file="/sys/devices/system/cpu/vulnerabilities/spec_store_bypass"
if [ -f "$ssbd_sysfs_file" ]; then
(grep -q "^Vulnerable" $ssbd_sysfs_file) && \
say_warn "WARNING: SSBD mitigation is either globally disabled or"\
"system does not support mitigation via prctl or seccomp. Try"\
"enabling it system-wide, using the \`${param}\` boot parameter."
else
say_warn "WARNING: SSBD mitigation not supported on this kernel."\
"View the prod-host-setup.md for more details."
fi
}
check_vm() {
if [ $(dmesg | grep -c -i "hypervisor detected") -gt 0 ]; then
say_warn "WARNING: you are running in a virtual machine." \
"Firecracker is not well tested under nested virtualization."
fi
}
cmd_checkenv() {
# Parse any command line args.
while [ $# -gt 0 ]; do
case "$1" in
"-h"|"--help") { cmd_help; exit 1; } ;;
*)
die "Unknown argument: $1. Please use --help for help."
;;
esac
shift
done
PROD_DOC="../docs/prod-host-setup.md"
QUICKSTART="../docs/getting-started.md#prerequisites"
say "Checking prerequisites for running Firecracker."
say "Please check $QUICKSTART in case of any error."
ensure_kvm_rw
check_kernver
check_vm
say "Checking Host Security Configuration."
say "Please check $PROD_DOC in case of any error."
check_KSM
check_IBPB_IBRS
check_L1TF
check_SMT
check_swap
check_SSBD
}
generate_syscall_table_x86_64() {
path_to_rust_file="$FC_ROOT_DIR/src/seccompiler/src/syscall_table/x86_64.rs"
echo "$header" > $path_to_rust_file
# the table for x86_64 is nicely formatted here: linux/arch/x86/entry/syscalls/syscall_64.tbl
cat linux/arch/x86/entry/syscalls/syscall_64.tbl | grep -v "^#" | grep -v -e '^$' |\
awk '{print $2,$3,$1}' | grep -v "^x32" |\
awk '{print " map.insert(\""$2"\".to_string(), "$3");"}' | sort >> $path_to_rust_file
echo "$footer" >> $path_to_rust_file
say "Generated at: $path_to_rust_file"
}
generate_syscall_table_aarch64() {
path_to_rust_file="$FC_ROOT_DIR/src/seccompiler/src/syscall_table/aarch64.rs"
# filter for substituting `#define`s that point to other macros;
# values taken from linux/include/uapi/asm-generic/unistd.h
replace+='s/__NR3264_fadvise64/223/;'
replace+='s/__NR3264_fcntl/25/;'
replace+='s/__NR3264_fstatat/79/;'
replace+='s/__NR3264_fstatfs/44/;'
replace+='s/__NR3264_fstat/80/;'
replace+='s/__NR3264_ftruncate/46/;'
replace+='s/__NR3264_lseek/62/;'
replace+='s/__NR3264_sendfile/71/;'
replace+='s/__NR3264_statfs/43/;'
replace+='s/__NR3264_truncate/45/;'
replace+='s/__NR3264_mmap/222/;'
echo "$header" > $path_to_rust_file
# run the gcc command in the Docker container (to make sure that we have gcc installed)
# the aarch64 syscall table is not located in a .tbl file, like x86; we run gcc's
# pre-processor to extract the numeric constants from header files.
run_devctr \
--user "$(id -u):$(id -g)" \
--workdir "$CTR_KERNEL_DIR" \
-- \
gcc -E -dM -D__ARCH_WANT_RENAMEAT -D__BITS_PER_LONG=64\
linux/arch/arm64/include/uapi/asm/unistd.h | grep "#define __NR_" |\
grep -v "__NR_syscalls" | grep -v "__NR_arch_specific_syscall" |\
awk -F '__NR_' '{print $2}' | sed $replace |\
awk '{ print " map.insert(\""$1"\".to_string(), "$2");" }' |\
sort >> $path_to_rust_file
ret=$?
[ $ret -ne 0 ] && return $ret
echo "$footer" >> $path_to_rust_file
say "Generated at: $path_to_rust_file"
}
cmd_generate_syscall_tables() {
# Parse any command line args.
while [ $# -gt 0 ]; do
case "$1" in
"-h"|"--help") { cmd_help; exit 1; } ;;
*) { kernel_version="$1"; break; } ;;
esac
shift
done
validate_kernel_version "$kernel_version"
kernel_major=v$(echo ${kernel_version} | cut -d . -f 1).x
kernel_baseurl=https://www.kernel.org/pub/linux/kernel/${kernel_major}
kernel_archive=linux-${kernel_version}.tar.xz
ensure_devctr
# Create the kernel clone directory
rm -rf "$KERNEL_DIR"
mkdir -p "$KERNEL_DIR" || die "Error: cannot create dir $dir"
[ -x "$KERNEL_DIR" ] && [ -w "$dir" ] || \
{
chmod +x+w "$KERNEL_DIR"
} || \
die "Error: wrong permissions for $KERNEL_DIR. Should be +x+w"
cd "$KERNEL_DIR"
say "Fetching linux kernel..."
# Get sha256 checksum.
curl -fsSLO ${kernel_baseurl}/sha256sums.asc && \
kernel_sha256=$(grep ${kernel_archive} sha256sums.asc | cut -d ' ' -f 1)
# Get kernel archive.
curl -fsSLO "$kernel_baseurl/$kernel_archive" && \
# Verify checksum.
echo "${kernel_sha256} ${kernel_archive}" | sha256sum -c - && \
# Decompress the kernel source.
xz -d "${kernel_archive}" && \
cat linux-${kernel_version}.tar | tar -x && mv linux-${kernel_version} linux
ret=$?
[ $ret -ne 0 ] && return $ret
# rust file header
read -r -d '' header << EOM
// Copyright $(date +"%Y") Amazon.com, Inc. or its affiliates. All Rights Reserved.
// SPDX-License-Identifier: Apache-2.0
// This file is auto-generated by \`tools/devtool generate_syscall_tables\`.
// Do NOT manually edit!
// Generated at: $(date)
// Kernel version: $kernel_version
use std::collections::HashMap;
pub(crate) fn make_syscall_table(map: &mut HashMap<String, i64>) {
EOM
# rust file footer
read -r -d '' footer << EOM
}
EOM
# generate syscall table for x86_64
say "Generating table for x86_64..."
generate_syscall_table_x86_64 $header $footer
# generate syscall table for aarch64
say "Generating table for aarch64..."
generate_syscall_table_aarch64 $header $footer
ret=$?
[ $ret -ne 0 ] && return $ret
}
cmd_install() {
# By default we install release/musl binaries.
profile="release"
target="$TARGET_PREFIX""musl"
install_path="/usr/local/bin"
# Parse any command line args.
while [ $# -gt 0 ]; do
case "$1" in
"-h"|"--help") { cmd_help; exit 1; } ;;
"-p"|"--path")
shift;
install_path=$1;
;;
"--debug") { profile="debug"; } ;;
"--release") { profile="release"; } ;;
*)
die "Unknown argument: $1. Please use --help for help."
;;
esac
shift
done
# Check that the binaries exist first
ensure_release_binaries_exist $target $profile
say "Installing firecracker in $install_path"
install -m 755 "$CARGO_TARGET_DIR/$target/$profile/firecracker" "$install_path"
say "Installing jailer in $install_path"
install -m 755 "$CARGO_TARGET_DIR/$target/$profile/jailer" "$install_path"
say "Installing seccomp in $install_path"
install -m 755 "$CARGO_SECCOMPILER_TARGET_DIR/$target/$profile/seccompiler-bin" "$install_path"
}
main() {
if [ $# = 0 ]; then
die "No command provided. Please use \`$0 help\` for help."
fi
# Parse main command line args.
#
while [ $# -gt 0 ]; do
case "$1" in
-h|--help) { cmd_help; exit 1; } ;;
-y|--unattended) { OPT_UNATTENDED=true; } ;;
-*)
die "Unknown arg: $1. Please use \`$0 help\` for help."
;;
*)
break
;;
esac
shift
done
# $1 is now a command name. Check if it is a valid command and, if so,
# run it.
#
declare -f "cmd_$1" > /dev/null
ok_or_die "Unknown command: $1. Please use \`$0 help\` for help."
cmd=cmd_$1
shift
# $@ is now a list of command-specific args
#
$cmd "$@"
}
main "$@"
