Light, fluffy, and always free - The AWS Local Emulator alternative
ci(compat): run the Terraform and OpenTofu bats files concurrently (#3282)
Hector Venturab3465ec94b77
diff --git a/compatibility-tests/README.md b/compatibility-tests/README.mdindex 7c40544155..5897d4c995 100644--- a/compatibility-tests/README.md+++ b/compatibility-tests/README.md@@ -114,8 +114,13 @@ Bats-based suites keep their normal console output and also write JUnit XML repo- `sdk-test-awscli/test-results/junit.xml`- `compat-cdk/test-results/junit.xml`-- `compat-terraform/test-results/junit.xml`-- `compat-opentofu/test-results/junit.xml`+- `compat-terraform/test-results/junit-<bats file>.xml`+- `compat-opentofu/test-results/junit-<bats file>.xml`++The Terraform and OpenTofu suites run each `test/*.bats` file in its own process+concurrently, so they write one report per file rather than a single `junit.xml`.+Every consumer globs the directory, so nothing downstream changes. Set+`BATS_PARALLEL_FILES=0` to run the files one at a time when isolating a failure.## Configurationdiff --git a/compatibility-tests/compat-opentofu/.dockerignore b/compatibility-tests/compat-opentofu/.dockerignorenew file mode 100644index 0000000000..2bea3ccfb5--- /dev/null+++ b/compatibility-tests/compat-opentofu/.dockerignore@@ -0,0 +1,11 @@+# Keep host-side Terraform working state out of the image. CI builds from a fresh+# checkout where none of this exists (.terraform/ and .terraform.lock.hcl are+# gitignored), so copying a local run's leftovers in would make the image behave+# differently from CI and quietly skew any timing comparison.+.terraform+.terraform.lock.hcl+*.tfstate+*.tfstate.*+errored.tfstate+test-results+large-object.bindiff --git a/compatibility-tests/compat-opentofu/main.tf b/compatibility-tests/compat-opentofu/main.tfindex 735b63d56f..53067497ca 100644--- a/compatibility-tests/compat-opentofu/main.tf+++ b/compatibility-tests/compat-opentofu/main.tf@@ -291,6 +291,9 @@ resource "aws_elasticache_replication_group" "compat" {# port it would hand out anyway, so pinning it asserts nothing. A non-default port is# what actually exercises CreateReplicationGroup's Port input, and a mismatch shows up# as permanent drift because Terraform treats the port as replacement-forcing.+ #+ # It must also stay above the block the cluster-mode fixture allocates: the bats files+ # run concurrently and that fixture takes one port per node from the base of the range.port = 6395}diff --git a/compatibility-tests/compat-opentofu/run-bats-in-container.sh b/compatibility-tests/compat-opentofu/run-bats-in-container.shold mode 100644new mode 100755index f89f6897ab..18a3c19dfd--- a/compatibility-tests/compat-opentofu/run-bats-in-container.sh+++ b/compatibility-tests/compat-opentofu/run-bats-in-container.sh@@ -1,16 +1,175 @@#!/usr/bin/env bash-set -euo pipefail+# Runs this suite's bats files, one process per file, concurrently by default.+#+# Each test/*.bats file is an independent root module: it creates its own state+# (local state for test/*-tf/, the S3 backend for the main suite), uses distinct+# resource identifiers, and cleans up in teardown_file. Nothing is shared, so the+# only thing serialising them was running `bats test/` as a single process.+#+# bats' own --jobs needs GNU parallel or rush, neither of which is in this image,+# so the files are forked here and reaped with wait.+#+# Environment:+# RESULTS_DIR where junit-<file>.xml is written (default /results)+# BATS_PARALLEL_FILES 0 to run the files sequentially (bisecting a flake)+# IAC_BIN terraform | tofu (default: whichever is on PATH)+set -uo pipefail-report_dir="$(mktemp -d /tmp/bats-junit-XXXXXX)"-trap 'rm -rf "$report_dir"' EXIT+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"+cd "$SCRIPT_DIR"-set +e-/opt/bats-core/bin/bats --report-formatter junit -o "$report_dir" test/-status=$?-set -e+RESULTS_DIR="${RESULTS_DIR:-/results}"+PARALLEL="${BATS_PARALLEL_FILES:-1}"-if [ -f "$report_dir/report.xml" ]; then- mv "$report_dir/report.xml" /results/junit.xml+if [ -n "${IAC_BIN:-}" ]; then+ :+elif command -v terraform >/dev/null 2>&1; then+ IAC_BIN=terraform+elif command -v tofu >/dev/null 2>&1; then+ IAC_BIN=tofu+else+ echo "Error: neither terraform nor tofu is on PATH" >&2+ exit 1fi+# Same resolution order as test/test_helper/common-setup.bash, so this script+# behaves identically in the image (BATS_LIB_PATH=/opt) and from a local checkout.+if [ -n "${BATS_LIB_PATH:-}" ] && [ -x "${BATS_LIB_PATH}/bats-core/bin/bats" ]; then+ BATS_BIN="${BATS_LIB_PATH}/bats-core/bin/bats"+elif [ -x "${SCRIPT_DIR}/../lib/bats-core/bin/bats" ]; then+ BATS_BIN="${SCRIPT_DIR}/../lib/bats-core/bin/bats"+else+ echo "Error: bats-core not found. Run 'just setup-bats' first." >&2+ exit 1+fi++FILES=()+for f in test/*.bats; do+ [ -e "$f" ] || continue+ FILES+=("$f")+done+if [ "${#FILES[@]}" -eq 0 ]; then+ echo "Error: no test/*.bats files found in $SCRIPT_DIR" >&2+ exit 1+fi++WORK_DIR="$(mktemp -d "${TMPDIR:-/tmp}/bats-run-XXXXXX")"+trap 'rm -rf "$WORK_DIR"' EXIT+mkdir -p "$RESULTS_DIR"+# Clear only the reports this run will write. A run that stops before rewriting every one+# would otherwise leave the survivors beside the new results, and every consumer globs the+# directory. Naming the files rather than globbing junit*.xml is deliberate:+# docker/run-docker-tests.sh mounts one shared test-results directory for all eight suites, so+# a glob here would delete compat-terraform's reports when compat-opentofu starts, and both+# would delete the junit.xml that sdk-test-awscli and compat-cdk write.+for report in "${FILES[@]}"; do+ rm -f "${RESULTS_DIR}/junit-$(basename "$report" .bats).xml"+done++# One shared provider download instead of one per file. Every fixture deletes its+# lock file in setup_file and re-resolves, so without this the concurrent inits+# would each fetch the provider at the same time. Warmed sequentially here because+# the plugin cache is only safe for concurrent readers, not concurrent writers.+warm_provider_cache() {+ local constraint warm_dir+ constraint="$(sed -n 's/.*version[[:space:]]*=[[:space:]]*"\([^"]*\)".*/\1/p' provider.tf | head -1)"+ if [ -z "$constraint" ]; then+ echo "warn: no provider version constraint found in provider.tf, skipping cache warm-up" >&2+ return 0+ fi++ export TF_PLUGIN_CACHE_DIR="${TF_PLUGIN_CACHE_DIR:-${WORK_DIR}/plugin-cache}"+ export TF_PLUGIN_CACHE_MAY_BREAK_DEPENDENCY_LOCK_FILE=true+ mkdir -p "$TF_PLUGIN_CACHE_DIR"++ warm_dir="${WORK_DIR}/warm"+ mkdir -p "$warm_dir"+ cat > "${warm_dir}/versions.tf" <<EOF+terraform {+ required_providers {+ aws = {+ source = "hashicorp/aws"+ version = "${constraint}"+ }+ }+}+EOF+ echo "# --- warming provider cache (aws ${constraint}) into ${TF_PLUGIN_CACHE_DIR} ---"+ if ! (cd "$warm_dir" && "$IAC_BIN" init -backend=false -input=false -no-color >/dev/null 2>&1); then+ # Not fatal: every fixture still resolves the provider itself, just without+ # the shared cache. Losing the warm-up costs time, never correctness.+ echo "warn: provider cache warm-up failed; files will resolve the provider individually" >&2+ unset TF_PLUGIN_CACHE_DIR TF_PLUGIN_CACHE_MAY_BREAK_DEPENDENCY_LOCK_FILE+ fi+}++run_one() {+ local file="$1" name="$2"+ "$BATS_BIN" --report-formatter junit -o "${WORK_DIR}/${name}" "$file" \+ > "${WORK_DIR}/${name}.log" 2>&1+ echo "$?" > "${WORK_DIR}/${name}.status"+}++warm_provider_cache++NAMES=()+for f in "${FILES[@]}"; do+ name="$(basename "$f" .bats)"+ NAMES+=("$name")+ mkdir -p "${WORK_DIR}/${name}"+done++start=$(date +%s)+if [ "$PARALLEL" = "0" ]; then+ echo "# --- running ${#FILES[@]} bats files sequentially ---"+ for i in "${!FILES[@]}"; do+ run_one "${FILES[$i]}" "${NAMES[$i]}"+ done+else+ echo "# --- running ${#FILES[@]} bats files concurrently ---"+ pids=()+ for i in "${!FILES[@]}"; do+ run_one "${FILES[$i]}" "${NAMES[$i]}" &+ pids+=("$!")+ done+ for pid in "${pids[@]}"; do+ wait "$pid"+ done+fi+elapsed=$(( $(date +%s) - start ))++# Output is collected per file and replayed in a fixed order: six interleaved bats+# streams are unreadable, and grouped blocks keep a failure next to its own TAP plan.+status=0+for name in "${NAMES[@]}"; do+ echo+ echo "=============== ${name}.bats ==============="+ cat "${WORK_DIR}/${name}.log" 2>/dev/null || true++ file_status=1+ if [ -f "${WORK_DIR}/${name}.status" ]; then+ file_status="$(cat "${WORK_DIR}/${name}.status")"+ fi+ [ "$file_status" -eq 0 ] || status=1++ if [ -f "${WORK_DIR}/${name}/report.xml" ]; then+ mv "${WORK_DIR}/${name}/report.xml" "${RESULTS_DIR}/junit-${name}.xml"+ else+ echo "Error: no JUnit report produced for ${name}.bats" >&2+ status=1+ fi+done++echo+echo "=============== summary ==============="+for name in "${NAMES[@]}"; do+ file_status="$(cat "${WORK_DIR}/${name}.status" 2>/dev/null || echo 1)"+ if [ "$file_status" -eq 0 ]; then+ echo " PASS ${name}"+ else+ echo " FAIL ${name} (exit ${file_status})"+ fi+done+echo "total ${elapsed}s"+exit "$status"diff --git a/compatibility-tests/compat-opentofu/run.sh b/compatibility-tests/compat-opentofu/run.shindex 2678aa3207..f68d1646b3 100755--- a/compatibility-tests/compat-opentofu/run.sh+++ b/compatibility-tests/compat-opentofu/run.sh@@ -1,7 +1,10 @@#!/bin/bashset -euo pipefail-# Environment setup for Docker container+# Runs the OpenTofu compatibility suite against a Floci already listening on+# FLOCI_ENDPOINT. Delegates to run-bats-in-container.sh, the same entrypoint the+# Docker image uses, so a local run and a CI run execute identical logic.+export AWS_REGION=us-east-1export AWS_DEFAULT_REGION=us-east-1export AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID:-test}"@@ -12,13 +15,13 @@ export AWS_ENDPOINT_URL="$FLOCI_ENDPOINT"SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"-# Ensure bats is availableif [ ! -d "$REPO_ROOT/lib/bats-core" ]; thenecho "Error: bats-core not found. Run 'just setup-bats' first."exit 1fi-# Run bats tests-exec "$REPO_ROOT/lib/run-bats-with-junit.sh" \- "$SCRIPT_DIR/test/" \- "${BATS_JUNIT_XML:-$SCRIPT_DIR/test-results/junit.xml}"+# BATS_JUNIT_XML used to name a single file; the suite now writes one report per+# bats file, so the variable names the directory they land in.+export RESULTS_DIR="${RESULTS_DIR:-$SCRIPT_DIR/test-results}"++exec "$SCRIPT_DIR/run-bats-in-container.sh"diff --git a/compatibility-tests/compat-terraform/.dockerignore b/compatibility-tests/compat-terraform/.dockerignorenew file mode 100644index 0000000000..2bea3ccfb5--- /dev/null+++ b/compatibility-tests/compat-terraform/.dockerignore@@ -0,0 +1,11 @@+# Keep host-side Terraform working state out of the image. CI builds from a fresh+# checkout where none of this exists (.terraform/ and .terraform.lock.hcl are+# gitignored), so copying a local run's leftovers in would make the image behave+# differently from CI and quietly skew any timing comparison.+.terraform+.terraform.lock.hcl+*.tfstate+*.tfstate.*+errored.tfstate+test-results+large-object.bindiff --git a/compatibility-tests/compat-terraform/main.tf b/compatibility-tests/compat-terraform/main.tfindex a341338da8..9e26ea1717 100644--- a/compatibility-tests/compat-terraform/main.tf+++ b/compatibility-tests/compat-terraform/main.tf@@ -291,6 +291,9 @@ resource "aws_elasticache_replication_group" "compat" {# port it would hand out anyway, so pinning it asserts nothing. A non-default port is# what actually exercises CreateReplicationGroup's Port input, and a mismatch shows up# as permanent drift because Terraform treats the port as replacement-forcing.+ #+ # It must also stay above the block the cluster-mode fixture allocates: the bats files+ # run concurrently and that fixture takes one port per node from the base of the range.port = 6395}diff --git a/compatibility-tests/compat-terraform/run-bats-in-container.sh b/compatibility-tests/compat-terraform/run-bats-in-container.shold mode 100644new mode 100755index f89f6897ab..18a3c19dfd--- a/compatibility-tests/compat-terraform/run-bats-in-container.sh+++ b/compatibility-tests/compat-terraform/run-bats-in-container.sh@@ -1,16 +1,175 @@#!/usr/bin/env bash-set -euo pipefail+# Runs this suite's bats files, one process per file, concurrently by default.+#+# Each test/*.bats file is an independent root module: it creates its own state+# (local state for test/*-tf/, the S3 backend for the main suite), uses distinct+# resource identifiers, and cleans up in teardown_file. Nothing is shared, so the+# only thing serialising them was running `bats test/` as a single process.+#+# bats' own --jobs needs GNU parallel or rush, neither of which is in this image,+# so the files are forked here and reaped with wait.+#+# Environment:+# RESULTS_DIR where junit-<file>.xml is written (default /results)+# BATS_PARALLEL_FILES 0 to run the files sequentially (bisecting a flake)+# IAC_BIN terraform | tofu (default: whichever is on PATH)+set -uo pipefail-report_dir="$(mktemp -d /tmp/bats-junit-XXXXXX)"-trap 'rm -rf "$report_dir"' EXIT+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"+cd "$SCRIPT_DIR"-set +e-/opt/bats-core/bin/bats --report-formatter junit -o "$report_dir" test/-status=$?-set -e+RESULTS_DIR="${RESULTS_DIR:-/results}"+PARALLEL="${BATS_PARALLEL_FILES:-1}"-if [ -f "$report_dir/report.xml" ]; then- mv "$report_dir/report.xml" /results/junit.xml+if [ -n "${IAC_BIN:-}" ]; then+ :+elif command -v terraform >/dev/null 2>&1; then+ IAC_BIN=terraform+elif command -v tofu >/dev/null 2>&1; then+ IAC_BIN=tofu+else+ echo "Error: neither terraform nor tofu is on PATH" >&2+ exit 1fi+# Same resolution order as test/test_helper/common-setup.bash, so this script+# behaves identically in the image (BATS_LIB_PATH=/opt) and from a local checkout.+if [ -n "${BATS_LIB_PATH:-}" ] && [ -x "${BATS_LIB_PATH}/bats-core/bin/bats" ]; then+ BATS_BIN="${BATS_LIB_PATH}/bats-core/bin/bats"+elif [ -x "${SCRIPT_DIR}/../lib/bats-core/bin/bats" ]; then+ BATS_BIN="${SCRIPT_DIR}/../lib/bats-core/bin/bats"+else+ echo "Error: bats-core not found. Run 'just setup-bats' first." >&2+ exit 1+fi++FILES=()+for f in test/*.bats; do+ [ -e "$f" ] || continue+ FILES+=("$f")+done+if [ "${#FILES[@]}" -eq 0 ]; then+ echo "Error: no test/*.bats files found in $SCRIPT_DIR" >&2+ exit 1+fi++WORK_DIR="$(mktemp -d "${TMPDIR:-/tmp}/bats-run-XXXXXX")"+trap 'rm -rf "$WORK_DIR"' EXIT+mkdir -p "$RESULTS_DIR"+# Clear only the reports this run will write. A run that stops before rewriting every one+# would otherwise leave the survivors beside the new results, and every consumer globs the+# directory. Naming the files rather than globbing junit*.xml is deliberate:+# docker/run-docker-tests.sh mounts one shared test-results directory for all eight suites, so+# a glob here would delete compat-terraform's reports when compat-opentofu starts, and both+# would delete the junit.xml that sdk-test-awscli and compat-cdk write.+for report in "${FILES[@]}"; do+ rm -f "${RESULTS_DIR}/junit-$(basename "$report" .bats).xml"+done++# One shared provider download instead of one per file. Every fixture deletes its+# lock file in setup_file and re-resolves, so without this the concurrent inits+# would each fetch the provider at the same time. Warmed sequentially here because+# the plugin cache is only safe for concurrent readers, not concurrent writers.+warm_provider_cache() {+ local constraint warm_dir+ constraint="$(sed -n 's/.*version[[:space:]]*=[[:space:]]*"\([^"]*\)".*/\1/p' provider.tf | head -1)"+ if [ -z "$constraint" ]; then+ echo "warn: no provider version constraint found in provider.tf, skipping cache warm-up" >&2+ return 0+ fi++ export TF_PLUGIN_CACHE_DIR="${TF_PLUGIN_CACHE_DIR:-${WORK_DIR}/plugin-cache}"+ export TF_PLUGIN_CACHE_MAY_BREAK_DEPENDENCY_LOCK_FILE=true+ mkdir -p "$TF_PLUGIN_CACHE_DIR"++ warm_dir="${WORK_DIR}/warm"+ mkdir -p "$warm_dir"+ cat > "${warm_dir}/versions.tf" <<EOF+terraform {+ required_providers {+ aws = {+ source = "hashicorp/aws"+ version = "${constraint}"+ }+ }+}+EOF+ echo "# --- warming provider cache (aws ${constraint}) into ${TF_PLUGIN_CACHE_DIR} ---"+ if ! (cd "$warm_dir" && "$IAC_BIN" init -backend=false -input=false -no-color >/dev/null 2>&1); then+ # Not fatal: every fixture still resolves the provider itself, just without+ # the shared cache. Losing the warm-up costs time, never correctness.+ echo "warn: provider cache warm-up failed; files will resolve the provider individually" >&2+ unset TF_PLUGIN_CACHE_DIR TF_PLUGIN_CACHE_MAY_BREAK_DEPENDENCY_LOCK_FILE+ fi+}++run_one() {+ local file="$1" name="$2"+ "$BATS_BIN" --report-formatter junit -o "${WORK_DIR}/${name}" "$file" \+ > "${WORK_DIR}/${name}.log" 2>&1+ echo "$?" > "${WORK_DIR}/${name}.status"+}++warm_provider_cache++NAMES=()+for f in "${FILES[@]}"; do+ name="$(basename "$f" .bats)"+ NAMES+=("$name")+ mkdir -p "${WORK_DIR}/${name}"+done++start=$(date +%s)+if [ "$PARALLEL" = "0" ]; then+ echo "# --- running ${#FILES[@]} bats files sequentially ---"+ for i in "${!FILES[@]}"; do+ run_one "${FILES[$i]}" "${NAMES[$i]}"+ done+else+ echo "# --- running ${#FILES[@]} bats files concurrently ---"+ pids=()+ for i in "${!FILES[@]}"; do+ run_one "${FILES[$i]}" "${NAMES[$i]}" &+ pids+=("$!")+ done+ for pid in "${pids[@]}"; do+ wait "$pid"+ done+fi+elapsed=$(( $(date +%s) - start ))++# Output is collected per file and replayed in a fixed order: six interleaved bats+# streams are unreadable, and grouped blocks keep a failure next to its own TAP plan.+status=0+for name in "${NAMES[@]}"; do+ echo+ echo "=============== ${name}.bats ==============="+ cat "${WORK_DIR}/${name}.log" 2>/dev/null || true++ file_status=1+ if [ -f "${WORK_DIR}/${name}.status" ]; then+ file_status="$(cat "${WORK_DIR}/${name}.status")"+ fi+ [ "$file_status" -eq 0 ] || status=1++ if [ -f "${WORK_DIR}/${name}/report.xml" ]; then+ mv "${WORK_DIR}/${name}/report.xml" "${RESULTS_DIR}/junit-${name}.xml"+ else+ echo "Error: no JUnit report produced for ${name}.bats" >&2+ status=1+ fi+done++echo+echo "=============== summary ==============="+for name in "${NAMES[@]}"; do+ file_status="$(cat "${WORK_DIR}/${name}.status" 2>/dev/null || echo 1)"+ if [ "$file_status" -eq 0 ]; then+ echo " PASS ${name}"+ else+ echo " FAIL ${name} (exit ${file_status})"+ fi+done+echo "total ${elapsed}s"+exit "$status"diff --git a/compatibility-tests/compat-terraform/run.sh b/compatibility-tests/compat-terraform/run.shindex 2678aa3207..dca3e0007e 100755--- a/compatibility-tests/compat-terraform/run.sh+++ b/compatibility-tests/compat-terraform/run.sh@@ -1,7 +1,10 @@#!/bin/bashset -euo pipefail-# Environment setup for Docker container+# Runs the Terraform compatibility suite against a Floci already listening on+# FLOCI_ENDPOINT. Delegates to run-bats-in-container.sh, the same entrypoint the+# Docker image uses, so a local run and a CI run execute identical logic.+export AWS_REGION=us-east-1export AWS_DEFAULT_REGION=us-east-1export AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID:-test}"@@ -12,13 +15,13 @@ export AWS_ENDPOINT_URL="$FLOCI_ENDPOINT"SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"-# Ensure bats is availableif [ ! -d "$REPO_ROOT/lib/bats-core" ]; thenecho "Error: bats-core not found. Run 'just setup-bats' first."exit 1fi-# Run bats tests-exec "$REPO_ROOT/lib/run-bats-with-junit.sh" \- "$SCRIPT_DIR/test/" \- "${BATS_JUNIT_XML:-$SCRIPT_DIR/test-results/junit.xml}"+# BATS_JUNIT_XML used to name a single file; the suite now writes one report per+# bats file, so the variable names the directory they land in.+export RESULTS_DIR="${RESULTS_DIR:-$SCRIPT_DIR/test-results}"++exec "$SCRIPT_DIR/run-bats-in-container.sh"diff --git a/compatibility-tests/justfile b/compatibility-tests/justfileindex 6e88d0b4db..1d23c88dd1 100644--- a/compatibility-tests/justfile+++ b/compatibility-tests/justfile@@ -100,13 +100,13 @@ setup-bats:test-cdk:./lib/run-bats-with-junit.sh compat-cdk/test/ compat-cdk/test-results/junit.xml-# Run Terraform compatibility tests (JUnit XML output via bats)+# Run Terraform compatibility tests (one JUnit XML per bats file, files run concurrently)test-terraform:- ./lib/run-bats-with-junit.sh compat-terraform/test/ compat-terraform/test-results/junit.xml+ ./compat-terraform/run.sh-# Run OpenTofu compatibility tests (JUnit XML output via bats)+# Run OpenTofu compatibility tests (one JUnit XML per bats file, files run concurrently)test-opentofu:- ./lib/run-bats-with-junit.sh compat-opentofu/test/ compat-opentofu/test-results/junit.xml+ ./compat-opentofu/run.sh# Run all IaC compatibility tests (continues on failure)test-compat: