Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(github-actions): continue Docker compatibility tests after failures (#3351)

Niels Kootstra051a2f09db36
diff --git a/.github/workflows/compatibility.yml b/.github/workflows/compatibility.yml
index d614f53df8..59bc128d99 100644
--- a/.github/workflows/compatibility.yml
+++ b/.github/workflows/compatibility.yml
@@ -10,6 +10,8 @@ on:
- 'docker/entrypoint.sh'
- 'docker/localstack-parity.sh'
- 'docker/healthcheck.sh'
+ - 'docker/run-docker-tests.sh'
+ - 'docker/test-run-docker-tests.sh'
- 'compatibility-tests/**'
- '.github/ci/compat-timing-report.py'
- '.github/ci/compat-opentofu-drift-check.py'
@@ -25,6 +27,16 @@ concurrency:
cancel-in-progress: true
jobs:
+ docker-runner-regression:
+ name: compat / Docker runner failure aggregation
+ runs-on: ubuntu-24.04
+ timeout-minutes: 5
+ steps:
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+
+ - name: Test Docker runner failure aggregation
+ run: bash docker/test-run-docker-tests.sh
+
compat-aws-cli-shim:
name: compat / aws CLI endpoint resolution
runs-on: ubuntu-24.04-arm
diff --git a/docker/run-docker-tests.sh b/docker/run-docker-tests.sh
index fec32cb31d..a8ebdeab84 100755
--- a/docker/run-docker-tests.sh
+++ b/docker/run-docker-tests.sh
@@ -47,6 +47,7 @@ SUITES=(
# results dir
mkdir -p test-results
+FAILED=0
for suite in "${SUITES[@]}"; do
echo "=== Running $suite in Docker ==="
@@ -54,7 +55,11 @@ for suite in "${SUITES[@]}"; do
IMAGE_NAME="compat-$suite"
# Build
- docker build -q -t "$IMAGE_NAME" "compatibility-tests/$suite"
+ if ! docker build -q -t "$IMAGE_NAME" "compatibility-tests/$suite"; then
+ echo "Test suite $suite failed to build"
+ FAILED=1
+ continue
+ fi
# Build DNS args: if we resolved Floci's IP, inject it as the DNS server so
# wildcard subdomains like <bucket>.floci resolve inside test containers.
@@ -73,15 +78,19 @@ for suite in "${SUITES[@]}"; do
fi
# Run
- docker run --rm --network "$NETWORK" \
- "${DNS_ARGS[@]}" \
- -e FLOCI_ENDPOINT=http://floci:4566 \
- -e FLOCI_S3_VHOST_ENDPOINT=http://floci:4566 \
- -v "$(pwd)/test-results:/results" \
- -v /var/run/docker.sock:/var/run/docker.sock \
- --group-add "$DOCKER_GID" \
- "${EXTRA_ARGS[@]}" \
- "$IMAGE_NAME" || echo "Test suite $suite failed"
+ if ! docker run --rm --network "$NETWORK" \
+ "${DNS_ARGS[@]}" \
+ -e FLOCI_ENDPOINT=http://floci:4566 \
+ -e FLOCI_S3_VHOST_ENDPOINT=http://floci:4566 \
+ -v "$(pwd)/test-results:/results" \
+ -v /var/run/docker.sock:/var/run/docker.sock \
+ --group-add "$DOCKER_GID" \
+ "${EXTRA_ARGS[@]}" \
+ "$IMAGE_NAME"; then
+ echo "Test suite $suite failed"
+ FAILED=1
+ fi
done
echo "=== All Docker tests completed ==="
+exit "$FAILED"
diff --git a/docker/test-run-docker-tests.sh b/docker/test-run-docker-tests.sh
new file mode 100755
index 0000000000..4854d38883
--- /dev/null
+++ b/docker/test-run-docker-tests.sh
@@ -0,0 +1,117 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+REPO_ROOT=$(cd "$SCRIPT_DIR/.." && pwd)
+LOG_FILE=$(mktemp)
+trap 'rm -f "$LOG_FILE"' EXIT
+
+export DOCKER_TEST_LOG="$LOG_FILE"
+export FAIL_RUN_SUITE=sdk-test-node
+export FAIL_BUILD_SUITE=
+
+docker() {
+ if [[ "$1" == compose && "$2" == up ]]; then
+ return 0
+ fi
+
+ if [[ "$1" == compose && "$2" == ps ]]; then
+ printf 'floci-container\n'
+ return 0
+ fi
+
+ if [[ "$1" == inspect ]]; then
+ printf '172.20.0.2\n'
+ return 0
+ fi
+
+ if [[ "$1" == build ]]; then
+ suite="${!#}"
+ suite="${suite#compatibility-tests/}"
+ printf 'build:%s\n' "$suite" >> "$DOCKER_TEST_LOG"
+ [[ "$suite" != "$FAIL_BUILD_SUITE" ]]
+ return
+ fi
+
+ if [[ "$1" == run ]]; then
+ suite="${!#}"
+ suite="${suite#compat-}"
+ printf 'run:%s\n' "$suite" >> "$DOCKER_TEST_LOG"
+ [[ "$suite" != "$FAIL_RUN_SUITE" ]]
+ return
+ fi
+
+ printf 'unexpected docker command: %s\n' "$*" >&2
+ return 1
+}
+
+curl() {
+ return 0
+}
+
+export -f docker curl
+
+if bash "$REPO_ROOT/docker/run-docker-tests.sh"; then
+ status=0
+else
+ status=$?
+fi
+
+if [[ "$status" -eq 0 ]]; then
+ echo "expected a failed suite to produce a non-zero exit status" >&2
+ exit 1
+fi
+
+expected_suites=(
+ sdk-test-python
+ sdk-test-node
+ sdk-test-java
+ sdk-test-go
+ sdk-test-awscli
+ compat-cdk
+ compat-terraform
+ compat-opentofu
+)
+
+for suite in "${expected_suites[@]}"; do
+ grep -qx "run:$suite" "$LOG_FILE" || {
+ echo "expected suite to run after failure: $suite" >&2
+ exit 1
+ }
+done
+
+: > "$LOG_FILE"
+export FAIL_RUN_SUITE=
+
+bash "$REPO_ROOT/docker/run-docker-tests.sh" || {
+ echo "expected all successful suites to produce a zero exit status" >&2
+ exit 1
+}
+
+: > "$LOG_FILE"
+export FAIL_BUILD_SUITE=compat-cdk
+
+if bash "$REPO_ROOT/docker/run-docker-tests.sh"; then
+ status=0
+else
+ status=$?
+fi
+
+if [[ "$status" -eq 0 ]]; then
+ echo "expected a failed image build to produce a non-zero exit status" >&2
+ exit 1
+fi
+
+for suite in "${expected_suites[@]}"; do
+ if [[ "$suite" == "compat-cdk" ]]; then
+ grep -q "run:$suite" "$LOG_FILE" && {
+ echo "failed image build should not run suite: $suite" >&2
+ exit 1
+ }
+ continue
+ fi
+ grep -q "run:$suite" "$LOG_FILE" || {
+ echo "expected suite to run after failed image build: $suite" >&2
+ exit 1
+ }
+done