Light, fluffy, and always free - The AWS Local Emulator alternative
ci: bound each terraform bats file and keep partial results on a cancelled compat job (#3720)
Hector Ventura32594a9dd5fa
diff --git a/.github/workflows/compatibility.yml b/.github/workflows/compatibility.ymlindex d12a55baea..a6ef4e744e 100644--- a/.github/workflows/compatibility.yml+++ b/.github/workflows/compatibility.yml@@ -441,5 +441,7 @@ jobs:if-no-files-found: warn- name: Dump floci logs- if: failure()+ # A job cancelled at timeout-minutes is not a failure, but a hung suite is the case+ # where the server log matters most.+ if: failure() || cancelled()run: docker logs flocidiff --git a/compatibility-tests/README.md b/compatibility-tests/README.mdindex 54fe9150cf..667c745c7d 100644--- a/compatibility-tests/README.md+++ b/compatibility-tests/README.md@@ -122,6 +122,10 @@ The Terraform and OpenTofu suites run each `test/*.bats` file in its own processconcurrently, so they write one report per file rather than a single `junit.xml`.Every consumer globs the directory, so nothing downstream changes. Set`BATS_PARALLEL_FILES=0` to run the files one at a time when isolating a failure.+Each file also runs under a `BATS_FILE_TIMEOUT` (default `12m`): a fixture that hangs+fails under its own name with its TAP output ending at the stuck test, and the reports of+the files that finished are still published, instead of the whole job dying at the+workflow's 20 minute limit with nothing reported.## Configurationdiff --git a/compatibility-tests/compat-opentofu/run-bats-in-container.sh b/compatibility-tests/compat-opentofu/run-bats-in-container.shindex 18a3c19dfd..7c642ed422 100755--- a/compatibility-tests/compat-opentofu/run-bats-in-container.sh+++ b/compatibility-tests/compat-opentofu/run-bats-in-container.sh@@ -20,6 +20,11 @@ cd "$SCRIPT_DIR"RESULTS_DIR="${RESULTS_DIR:-/results}"PARALLEL="${BATS_PARALLEL_FILES:-1}"+# Bound one bats file so a hung fixture fails under its own name instead of the whole job+# dying at the workflow's timeout-minutes with nothing reported. The workflow gives the+# job 20 minutes and a healthy concurrent run finishes in about seven, so twelve leaves+# room for a slow runner while still naming the culprit before the job limit.+FILE_TIMEOUT="${BATS_FILE_TIMEOUT:-12m}"if [ -n "${IAC_BIN:-}" ]; then:@@ -104,10 +109,29 @@ EOF}run_one() {- local file="$1" name="$2"- "$BATS_BIN" --report-formatter junit -o "${WORK_DIR}/${name}" "$file" \- > "${WORK_DIR}/${name}.log" 2>&1- echo "$?" > "${WORK_DIR}/${name}.status"+ local file="$1" name="$2" started status+ started=$(date +%s)+ if command -v timeout >/dev/null 2>&1; then+ timeout -k 30s "$FILE_TIMEOUT" \+ "$BATS_BIN" --report-formatter junit -o "${WORK_DIR}/${name}" "$file" \+ > "${WORK_DIR}/${name}.log" 2>&1+ else+ "$BATS_BIN" --report-formatter junit -o "${WORK_DIR}/${name}" "$file" \+ > "${WORK_DIR}/${name}.log" 2>&1+ fi+ status=$?+ if [ "$status" -eq 124 ]; then+ echo "# ${name}.bats: timed out after ${FILE_TIMEOUT}; the TAP output above ends at the test that hung" \+ >> "${WORK_DIR}/${name}.log"+ fi+ echo "$status" > "${WORK_DIR}/${name}.status"+ # Publish the report as soon as this file is done rather than after every file has+ # finished: if the job is cancelled while another file hangs, the reports of the files+ # that completed are still uploaded, and the missing one names the hang.+ if [ -f "${WORK_DIR}/${name}/report.xml" ]; then+ mv "${WORK_DIR}/${name}/report.xml" "${RESULTS_DIR}/junit-${name}.xml"+ fi+ echo "# --- ${name}.bats finished in $(( $(date +%s) - started ))s (exit ${status}) ---"}warm_provider_cache@@ -152,9 +176,7 @@ for name in "${NAMES[@]}"; dofi[ "$file_status" -eq 0 ] || status=1- if [ -f "${WORK_DIR}/${name}/report.xml" ]; then- mv "${WORK_DIR}/${name}/report.xml" "${RESULTS_DIR}/junit-${name}.xml"- else+ if [ ! -f "${RESULTS_DIR}/junit-${name}.xml" ]; thenecho "Error: no JUnit report produced for ${name}.bats" >&2status=1fidiff --git a/compatibility-tests/compat-terraform/run-bats-in-container.sh b/compatibility-tests/compat-terraform/run-bats-in-container.shindex 18a3c19dfd..7c642ed422 100755--- a/compatibility-tests/compat-terraform/run-bats-in-container.sh+++ b/compatibility-tests/compat-terraform/run-bats-in-container.sh@@ -20,6 +20,11 @@ cd "$SCRIPT_DIR"RESULTS_DIR="${RESULTS_DIR:-/results}"PARALLEL="${BATS_PARALLEL_FILES:-1}"+# Bound one bats file so a hung fixture fails under its own name instead of the whole job+# dying at the workflow's timeout-minutes with nothing reported. The workflow gives the+# job 20 minutes and a healthy concurrent run finishes in about seven, so twelve leaves+# room for a slow runner while still naming the culprit before the job limit.+FILE_TIMEOUT="${BATS_FILE_TIMEOUT:-12m}"if [ -n "${IAC_BIN:-}" ]; then:@@ -104,10 +109,29 @@ EOF}run_one() {- local file="$1" name="$2"- "$BATS_BIN" --report-formatter junit -o "${WORK_DIR}/${name}" "$file" \- > "${WORK_DIR}/${name}.log" 2>&1- echo "$?" > "${WORK_DIR}/${name}.status"+ local file="$1" name="$2" started status+ started=$(date +%s)+ if command -v timeout >/dev/null 2>&1; then+ timeout -k 30s "$FILE_TIMEOUT" \+ "$BATS_BIN" --report-formatter junit -o "${WORK_DIR}/${name}" "$file" \+ > "${WORK_DIR}/${name}.log" 2>&1+ else+ "$BATS_BIN" --report-formatter junit -o "${WORK_DIR}/${name}" "$file" \+ > "${WORK_DIR}/${name}.log" 2>&1+ fi+ status=$?+ if [ "$status" -eq 124 ]; then+ echo "# ${name}.bats: timed out after ${FILE_TIMEOUT}; the TAP output above ends at the test that hung" \+ >> "${WORK_DIR}/${name}.log"+ fi+ echo "$status" > "${WORK_DIR}/${name}.status"+ # Publish the report as soon as this file is done rather than after every file has+ # finished: if the job is cancelled while another file hangs, the reports of the files+ # that completed are still uploaded, and the missing one names the hang.+ if [ -f "${WORK_DIR}/${name}/report.xml" ]; then+ mv "${WORK_DIR}/${name}/report.xml" "${RESULTS_DIR}/junit-${name}.xml"+ fi+ echo "# --- ${name}.bats finished in $(( $(date +%s) - started ))s (exit ${status}) ---"}warm_provider_cache@@ -152,9 +176,7 @@ for name in "${NAMES[@]}"; dofi[ "$file_status" -eq 0 ] || status=1- if [ -f "${WORK_DIR}/${name}/report.xml" ]; then- mv "${WORK_DIR}/${name}/report.xml" "${RESULTS_DIR}/junit-${name}.xml"- else+ if [ ! -f "${RESULTS_DIR}/junit-${name}.xml" ]; thenecho "Error: no JUnit report produced for ${name}.bats" >&2status=1fi