AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
ci(platform): speed up and harden single-container validation (#14282)
Nicholas Tindle43a7bc38d210
diff --git a/.github/scripts/platform-single-container-publish.sh b/.github/scripts/platform-single-container-publish.shindex 2440dddc97..f4cfc4ce12 100755--- a/.github/scripts/platform-single-container-publish.sh+++ b/.github/scripts/platform-single-container-publish.sh@@ -173,10 +173,6 @@ resolve_publication() {publish_latest=falsepublication_name="Single-container SHA image published"- if [[ "$GITHUB_EVENT_NAME" == "workflow_dispatch" && "$GITHUB_REF" == "refs/heads/dev" ]]; then- return- fi-if [[ "$GITHUB_EVENT_NAME" == "release" && "$GITHUB_REF" == "refs/tags/${RELEASE_TAG}" ]]; thenif [[ ! "$RELEASE_TAG" =~ ^autogpt-platform-beta-v([0-9]+\.[0-9]+\.[0-9]+)$ ]]; thenecho "refusing unsupported release tag: $RELEASE_TAG" >&2@@ -194,18 +190,6 @@ resolve_publication() {}publication_allowed() {- if [[ "$GITHUB_EVENT_NAME" == "workflow_dispatch" ]]; then- if [[ "$PUBLISH_REQUESTED" != "true" ]]; then- printf 'false\n'- return- fi- if ! resolve_publication; then- return 1- fi- printf 'true\n'- return- fi-if [[ "$GITHUB_EVENT_NAME" == "release" ]]; thenif [[ "$RELEASE_PRERELEASE" != "false" ]]; thenprintf 'false\n'@@ -356,7 +340,7 @@ ensure_immutable_manifest() {ensure_release_tag() {local manifest_digest="$1"- local state raw_manifest+ local state raw_manifest published_release_versionrelease_manifest_digest="$manifest_digest"[[ -n "$release_ref" ]] || return 0@@ -370,7 +354,11 @@ ensure_release_tag() {release_manifest_digest="$(inspect_manifest "$release_ref")"verify_manifest "$release_ref" "${expected_rows[@]}"raw_manifest="$(docker buildx imagetools inspect --raw "$release_ref")"- if [[ "$(release_version_from_manifest <<<"$raw_manifest")" != "$release_version" ]]; then+ if ! published_release_version="$(release_version_from_manifest <<<"$raw_manifest")"; then+ echo "could not read the immutable release-version annotation from $release_ref" >&2+ return 1+ fi+ if [[ "$published_release_version" != "$release_version" ]]; thenecho "$release_ref is missing its immutable release-version annotation" >&2return 1fi@@ -418,10 +406,11 @@ latest_update_allowed() {returnfi- set +e- semver_is_newer "$current_version" "$target_version"- compare_status=$?- set -e+ if semver_is_newer "$current_version" "$target_version"; then+ compare_status=0+ else+ compare_status=$?+ fiif ((compare_status == 0)); thenecho "refusing to move latest backward from $current_version to $target_version" >&2return 1@@ -458,13 +447,22 @@ publish_latest_tag() {fi}+list_digest_files() {+ find "$DIGEST_DIR" -maxdepth 1 -type f -print | sort+}+load_verified_digests() {local digest_file descriptor expected_arch digest_hex image_ref actual_platform raw_manifest expected_rowlocal runnable_digest image_json source_revision+ local digest_listinglocal -a digest_files=()declare -A seen_platforms=()- mapfile -t digest_files < <(find "$DIGEST_DIR" -maxdepth 1 -type f -print | sort)+ if ! digest_listing="$(list_digest_files)"; then+ echo "could not enumerate verified platform digests" >&2+ return 1+ fi+ mapfile -t digest_files <<<"$digest_listing"if ((${#digest_files[@]} != 2)); thenecho "expected exactly two verified platform digests" >&2return 1@@ -695,12 +693,12 @@ self_test() {assert_equal 3 "$actual" "confirmed absent retry count"rm -f "$tag_retry_state"- DEPLOY_IMAGE=docker.io/significantgravitas/autogpt \+ if DEPLOY_IMAGE=docker.io/significantgravitas/autogpt \GITHUB_EVENT_NAME=workflow_dispatch GITHUB_REF=refs/heads/dev GITHUB_SHA=abc123 \- RELEASE_TAG='' resolve_publication- assert_equal docker.io/significantgravitas/autogpt:sha-abc123 "$immutable_ref" "dev immutable tag"- assert_equal '' "$release_ref" "dev release tag"- assert_equal false "$publish_latest" "dev latest policy"+ RELEASE_TAG='' resolve_publication 2>/dev/null; then+ echo "workflow dispatch publication was accepted" >&2+ return 1+ firelease_manifest_digest=""ensure_release_tag sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa@@ -726,26 +724,20 @@ self_test() {actual="$(DEPLOY_IMAGE=docker.io/significantgravitas/autogpt \GITHUB_EVENT_NAME=workflow_dispatch GITHUB_REF=refs/heads/dev GITHUB_SHA=abc123 \- PUBLISH_REQUESTED=true RELEASE_PRERELEASE='' RELEASE_TAG='' publication_allowed+ RELEASE_PRERELEASE='' RELEASE_TAG='' publication_allowed)"- assert_equal true "$actual" "dev publication authorization"- actual="$(- DEPLOY_IMAGE=docker.io/significantgravitas/autogpt \- GITHUB_EVENT_NAME=workflow_dispatch GITHUB_REF=refs/heads/dev GITHUB_SHA=abc123 \- PUBLISH_REQUESTED=false RELEASE_PRERELEASE='' RELEASE_TAG='' publication_allowed- )"- assert_equal false "$actual" "validation-only dispatch authorization"+ assert_equal false "$actual" "dispatch publication authorization"actual="$(DEPLOY_IMAGE=docker.io/significantgravitas/autogpt \GITHUB_EVENT_NAME=release GITHUB_REF=refs/tags/autogpt-platform-beta-v0.7.1 \- GITHUB_SHA=abc123 PUBLISH_REQUESTED='' RELEASE_PRERELEASE=false \+ GITHUB_SHA=abc123 RELEASE_PRERELEASE=false \RELEASE_TAG=autogpt-platform-beta-v0.7.1 publication_allowed)"assert_equal true "$actual" "release publication authorization"actual="$(DEPLOY_IMAGE=docker.io/significantgravitas/autogpt \GITHUB_EVENT_NAME=release GITHUB_REF=refs/tags/autogpt-platform-beta-v0.7.1 \- GITHUB_SHA=abc123 PUBLISH_REQUESTED='' RELEASE_PRERELEASE=true \+ GITHUB_SHA=abc123 RELEASE_PRERELEASE=true \RELEASE_TAG=autogpt-platform-beta-v0.7.1 publication_allowed)"assert_equal false "$actual" "prerelease publication authorization"@@ -754,9 +746,9 @@ self_test() {GITHUB_OUTPUT="$authorization_output" \DEPLOY_IMAGE=docker.io/significantgravitas/autogpt \GITHUB_EVENT_NAME=workflow_dispatch GITHUB_REF=refs/heads/dev GITHUB_SHA=abc123 \- PUBLISH_REQUESTED=true RELEASE_PRERELEASE='' RELEASE_TAG='' authorize+ RELEASE_PRERELEASE='' RELEASE_TAG='' authorizeactual="$(<"$authorization_output")"- assert_equal allowed=true "$actual" "authorization output"+ assert_equal allowed=false "$actual" "dispatch authorization output": >"$authorization_output"if DEPLOY_IMAGE=docker.io/significantgravitas/autogpt \@@ -767,7 +759,7 @@ self_test() {fiif DEPLOY_IMAGE=docker.io/significantgravitas/autogpt \GITHUB_EVENT_NAME=release GITHUB_REF=refs/tags/autogpt-platform-beta-v0.7 \- GITHUB_SHA=abc123 PUBLISH_REQUESTED='' RELEASE_PRERELEASE=false \+ GITHUB_SHA=abc123 RELEASE_PRERELEASE=false \RELEASE_TAG=autogpt-platform-beta-v0.7 publication_allowed 2>/dev/null; thenecho "malformed release tag was authorized" >&2return 1@@ -775,7 +767,7 @@ self_test() {if GITHUB_OUTPUT="$authorization_output" \DEPLOY_IMAGE=docker.io/significantgravitas/autogpt \GITHUB_EVENT_NAME=release GITHUB_REF=refs/tags/autogpt-platform-beta-v0.7 \- GITHUB_SHA=abc123 PUBLISH_REQUESTED='' RELEASE_PRERELEASE=false \+ GITHUB_SHA=abc123 RELEASE_PRERELEASE=false \RELEASE_TAG=autogpt-platform-beta-v0.7 authorize 2>/dev/null; thenecho "malformed release tag produced an authorization output" >&2return 1@@ -875,6 +867,32 @@ JSONecho "invalid release-version annotation was accepted" >&2return 1fi+ if (+ release_ref=docker.io/significantgravitas/autogpt:v0.7.1+ release_version=v0.7.1+ expected_rows=()+ tag_state() {+ printf 'present\n'+ }+ inspect_manifest() {+ printf 'sha256:%064d\n' 0+ }+ verify_manifest() {+ :+ }+ docker() {+ printf '{}\n'+ }+ release_version_from_manifest() {+ printf 'v0.7.1\n'+ return 1+ }+ ensure_release_tag sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa \+ >/dev/null 2>&1+ ); then+ echo "failed release-version extraction was accepted" >&2+ return 1+ fisemver_is_newer v0.7.2 v0.7.1semver_is_newer v1.0.0 v0.99.99@@ -899,6 +917,19 @@ JSONreturn 1fi+ if (+ list_digest_files() {+ printf '%s\n' \+ /tmp/amd64-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa \+ /tmp/arm64-bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb+ return 1+ }+ DIGEST_DIR=/tmp load_verified_digests >/dev/null 2>&1+ ); then+ echo "failed digest enumeration was accepted" >&2+ return 1+ fi+echo "single-container publication helper tests passed"}diff --git a/.github/scripts/platform-single-container-smoke.sh b/.github/scripts/platform-single-container-smoke.shindex 9bcdafcf3d..08d437d4e6 100644--- a/.github/scripts/platform-single-container-smoke.sh+++ b/.github/scripts/platform-single-container-smoke.sh@@ -22,10 +22,13 @@ readonly STOCK_DOCKER_STOP_TIMEOUT=10readonly SHUTDOWN_MARGIN_SECONDS=1readonly MAX_CLEAN_STOP_SECONDS=$((STOCK_DOCKER_STOP_TIMEOUT - SHUTDOWN_MARGIN_SECONDS))readonly TIMEOUT_SECONDS="${SMOKE_TIMEOUT_SECONDS:-2700}"+readonly SCAN_COMPLETION_FILE="${SMOKE_SCAN_COMPLETION_FILE:-}"readonly SAFE_PLATFORM="${SMOKE_PLATFORM//\//-}"readonly RUN_TOKEN="${GITHUB_RUN_ID:-local}-${GITHUB_RUN_ATTEMPT:-0}-${SAFE_PLATFORM}-${RANDOM}"readonly RUN_CONTAINER_NAME="autogpt-single-smoke-${RUN_TOKEN}"readonly NEGATIVE_CONTAINER_NAME="${RUN_CONTAINER_NAME}-negative"+readonly SMOKE_STARTED_SECONDS=${SECONDS}+SMOKE_LAST_TIMING_SECONDS=${SMOKE_STARTED_SECONDS}HEADERS_FILE="$(mktemp)"readonly HEADERS_FILEAUTH_COOKIE_FILE="$(mktemp)"@@ -34,40 +37,175 @@ readonly AUTH_COOKIE_FILECONTAINER_NAME="${RUN_CONTAINER_NAME}"DATA_VOLUME=+record_timing() {+ local phase="$1"+ local now=${SECONDS}+ printf 'single-container smoke timing: phase=%s duration=%ss elapsed=%ss\n' \+ "${phase}" \+ "$((now - SMOKE_LAST_TIMING_SECONDS))" \+ "$((now - SMOKE_STARTED_SECONDS))"+ SMOKE_LAST_TIMING_SECONDS=${now}+}++record_boot_milestones() {+ local phase="$1"+ local started_at+ local health_history+ local container_logs+ local line+ local milestone+ local timestamp++ if ! started_at="$(+ docker inspect --format '{{.State.StartedAt}}' "${CONTAINER_NAME}"+ )"; then+ echo "could not record ${phase} container start time" >&2+ return 0+ fi+ printf 'single-container boot milestone: phase=%s container-started-at=%s\n' \+ "${phase}" "${started_at}"++ if health_history="$(+ docker inspect --format '{{json .State.Health.Log}}' "${CONTAINER_NAME}"+ )"; then+ if ! jq -r --arg phase "${phase}" \+ '.[] | "single-container boot milestone: phase=\($phase) docker-health start=\(.Start) end=\(.End) exit=\(.ExitCode)"' \+ <<<"${health_history}"; then+ echo "could not format ${phase} Docker health milestones" >&2+ fi+ else+ echo "could not record ${phase} Docker health milestones" >&2+ fi++ if ! container_logs="$(+ docker logs --timestamps --since "${started_at}" "${CONTAINER_NAME}" 2>&1+ )"; then+ echo "could not record ${phase} appliance milestones" >&2+ return 0+ fi+ # Raw service logs and Docker health output can contain request or provider+ # data and stay private. A matching line selects a fixed event name; none of+ # the original log content is emitted.+ while IFS= read -r line; do+ milestone=+ case "${line}" in+ *"[single-container] starting process supervisor"*) milestone=supervisor-start ;;+ *"[single-container] initializing PostgreSQL data directory"*) milestone=postgres-init ;;+ *"[single-container] PostgreSQL is ready"*) milestone=postgres-ready ;;+ *"[single-container] Valkey node "*" is ready"*) milestone=valkey-node-ready ;;+ *"[single-container] RabbitMQ is ready"*) milestone=rabbitmq-ready ;;+ *"[single-container] FalkorDB is ready"*) milestone=falkordb-ready ;;+ *"[single-container] forming three-node Valkey cluster"*) milestone=valkey-cluster-forming ;;+ *"[single-container] Valkey cluster is ready"*) milestone=valkey-cluster-ready ;;+ *"[single-container] Valkey cluster is already healthy"*) milestone=valkey-cluster-already-healthy ;;+ *"[single-container] Valkey cluster recovered"*) milestone=valkey-cluster-recovered ;;+ *"[single-container] RabbitMQ application user is present"*) milestone=rabbitmq-user-ready ;;+ *"[single-container] ensuring platform database schemas exist"*) milestone=database-schemas-start ;;+ *"[single-container] applying Prisma migrations"*) milestone=prisma-migrations-start ;;+ *"[single-container] configuring least-privilege frontend database role"*) milestone=frontend-role-start ;;+ *"[single-container] bootstrap complete"*) milestone=bootstrap-complete ;;+ *"[single-container] starting database-manager"*) milestone=database-manager-start ;;+ *"[single-container] starting scheduler"*) milestone=scheduler-start ;;+ *"[single-container] starting batch-executor"*) milestone=batch-executor-start ;;+ *"[single-container] starting notification"*) milestone=notification-start ;;+ *"[single-container] starting executor"*) milestone=executor-start ;;+ *"[single-container] starting copilot-executor"*) milestone=copilot-executor-start ;;+ *"[single-container] starting copilot-bot"*) milestone=copilot-bot-start ;;+ *"[single-container] starting platform-linking-manager"*) milestone=platform-linking-manager-start ;;+ *"[single-container] starting websocket"*) milestone=websocket-start ;;+ *"[single-container] starting rest"*) milestone=rest-start ;;+ *"[single-container] starting next"*) milestone=next-start ;;+ *"[single-container] starting nginx"*) milestone=nginx-start ;;+ *"[single-container] watchdog armed after initial healthy state"*) milestone=watchdog-armed ;;+ esac+ [[ -n "${milestone}" ]] || continue+ timestamp="${line%% *}"+ if [[ ! "${timestamp}" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(\.[0-9]+)?Z$ ]]; then+ timestamp=unknown+ fi+ printf 'single-container boot milestone: phase=%s at=%s event=%s\n' \+ "${phase}" "${timestamp}" "${milestone}"+ done <<<"${container_logs}"+}+diagnostics() {if docker container inspect "${CONTAINER_NAME}" >/dev/null 2>&1; then- docker inspect --format '{{json .State}}' "${CONTAINER_NAME}" || true- docker logs --timestamps --tail 2000 "${CONTAINER_NAME}" || true+ if ! docker inspect --format '{{json .State}}' "${CONTAINER_NAME}"; then+ echo "could not inspect ${CONTAINER_NAME}" >&2+ fi+ if ! docker logs --timestamps --tail 2000 "${CONTAINER_NAME}"; then+ echo "could not read logs for ${CONTAINER_NAME}" >&2+ fifi}cleanup() {local result=$?+ local cleanup_failed=0local container+ local container_id+ local volume_nametrap - EXIT INT TERMif ((result != 0)); thendiagnosticsfifor container in "${CONTAINER_NAME}" "${NEGATIVE_CONTAINER_NAME}"; do- if docker container inspect "${container}" >/dev/null 2>&1; then- docker stop --timeout "${STOCK_DOCKER_STOP_TIMEOUT}" "${container}" >/dev/null 2>&1 || true- docker rm --force --volumes "${container}" >/dev/null 2>&1 || true+ if ! container_id="$(+ docker container ls --all --quiet --filter "name=^/${container}$"+ )"; then+ echo "could not determine whether ${container} needs cleanup" >&2+ cleanup_failed=1+ continue+ fi+ if [[ -n "${container_id}" ]]; then+ if ! docker stop --timeout "${STOCK_DOCKER_STOP_TIMEOUT}" "${container}" >/dev/null 2>&1; then+ echo "could not stop ${container} during cleanup" >&2+ cleanup_failed=1+ fi+ if ! docker rm --force --volumes "${container}" >/dev/null 2>&1; then+ echo "could not remove ${container} during cleanup" >&2+ cleanup_failed=1+ fifidone- if [[ -n "${DATA_VOLUME}" ]] && docker volume inspect "${DATA_VOLUME}" >/dev/null 2>&1; then- docker volume rm "${DATA_VOLUME}" >/dev/null 2>&1 || true+ if [[ -n "${DATA_VOLUME}" ]]; then+ if ! volume_name="$(+ docker volume ls --quiet --filter "name=^${DATA_VOLUME}$"+ )"; then+ echo "could not determine whether ${DATA_VOLUME} needs cleanup" >&2+ cleanup_failed=1+ elif [[ -n "${volume_name}" ]]; then+ if ! docker volume rm "${DATA_VOLUME}" >/dev/null 2>&1; then+ echo "could not remove ${DATA_VOLUME} during cleanup" >&2+ cleanup_failed=1+ fi+ fi+ fi+ if ! rm -f "${HEADERS_FILE}" "${AUTH_COOKIE_FILE}"; then+ echo "could not remove single-container smoke temporary files" >&2+ cleanup_failed=1+ fi+ if ((result == 0 && cleanup_failed != 0)); then+ result=1fi- rm -f "${HEADERS_FILE}" "${AUTH_COOKIE_FILE}"exit "${result}"}assert_clean_stop() {local reason="$1"- local started elapsed exit_code- # Integer SECONDS truncates at both ends, so a real 8.9s stop reads as 8.+ local started finished_at finished_epoch elapsed exit_codestarted="${EPOCHREALTIME}"docker stop --timeout "${STOCK_DOCKER_STOP_TIMEOUT}" "${CONTAINER_NAME}" >/dev/null- elapsed="$(awk -v a="${started}" -v b="${EPOCHREALTIME}" 'BEGIN { printf "%.2f", b - a }')"+ finished_at="$(docker inspect --format '{{.State.FinishedAt}}' "${CONTAINER_NAME}")"+ if ! finished_epoch="$(date --date="${finished_at}" +%s.%N)" || [[ -z "${finished_epoch}" ]]; then+ echo "invalid container finish timestamp ${reason}: ${finished_at}" >&2+ return 1+ fi+ awk -v a="${started}" -v b="${finished_epoch}" 'BEGIN { exit !(b >= a) }' || {+ echo "container finish timestamp precedes the stop request ${reason}" >&2+ return 1+ }+ elapsed="$(awk -v a="${started}" -v b="${finished_epoch}" 'BEGIN { printf "%.2f", b - a }')"exit_code="$(docker inspect --format '{{.State.ExitCode}}' "${CONTAINER_NAME}")"[[ "${exit_code}" == 0 ]] || {echo "container did not exit cleanly ${reason}: exit ${exit_code} after" \@@ -104,12 +242,17 @@ wait_for_healthy() {local statelocal healthlocal exit_code+ local state_recordwhile ((SECONDS < deadline)); do- read -r state health exit_code < <(+ if ! state_record="$(docker inspect \--format '{{.State.Status}} {{if .State.Health}}{{.State.Health.Status}}{{else}}missing{{end}} {{.State.ExitCode}}' \"${CONTAINER_NAME}"- )+ )"; then+ echo "could not inspect container state while waiting for health" >&2+ return 1+ fi+ read -r state health exit_code <<<"${state_record}"case "${state}:${health}" inrunning:healthy)if appliance_is_healthy_and_armed; then@@ -121,7 +264,7 @@ wait_for_healthy() {return 1;;esac- sleep 10+ sleep 1doneecho "container did not become healthy within ${TIMEOUT_SECONDS}s" >&2return 1@@ -133,12 +276,17 @@ wait_for_automatic_restart() {local statelocal healthlocal restart_count+ local state_recordwhile ((SECONDS < deadline)); do- read -r state health restart_count < <(+ if ! state_record="$(docker inspect \--format '{{.State.Status}} {{if .State.Health}}{{.State.Health.Status}}{{else}}missing{{end}} {{.RestartCount}}' \"${CONTAINER_NAME}"- )+ )"; then+ echo "could not inspect container state while waiting for restart" >&2+ return 1+ fi+ read -r state health restart_count <<<"${state_record}"if [[ "${state}:${health}" == running:healthy ]] &&((restart_count >= minimum_restart_count)) &&appliance_is_healthy_and_armed; then@@ -150,12 +298,107 @@ wait_for_automatic_restart() {return 1;;esac- sleep 10+ sleep 1doneecho "container did not automatically restart and become healthy" >&2return 1}+wait_for_concurrent_scans() {+ if [[ -z "${SCAN_COMPLETION_FILE}" ]]; then+ return 0+ fi+ local deadline=$((SECONDS + TIMEOUT_SECONDS))+ while ((SECONDS < deadline)); do+ if [[ -f "${SCAN_COMPLETION_FILE}" ]]; then+ return 0+ fi+ sleep 1+ done+ echo "concurrent image scans did not complete within ${TIMEOUT_SECONDS}s" >&2+ return 1+}++count_container_log_evidence() {+ local expected="$1"+ local container_logs+ local count=0+ local line+ if ! container_logs="$(docker logs "${CONTAINER_NAME}" 2>&1)"; then+ echo "could not read container logs while counting watchdog evidence" >&2+ return 1+ fi+ while IFS= read -r line; do+ if [[ "${line}" == *"${expected}"* ]]; then+ count=$((count + 1))+ fi+ done <<<"${container_logs}"+ printf '%s\n' "${count}"+}++wait_for_new_container_log_evidence() {+ local expected="$1"+ local previous_count="$2"+ local deadline=$((SECONDS + TIMEOUT_SECONDS))+ local current_count+ if [[ ! "${previous_count}" =~ ^[0-9]+$ ]]; then+ echo "previous watchdog evidence count is invalid" >&2+ return 1+ fi+ while ((SECONDS < deadline)); do+ if ! current_count="$(count_container_log_evidence "${expected}")"; then+ return 1+ fi+ if [[ ! "${current_count}" =~ ^[0-9]+$ ]]; then+ echo "watchdog evidence count is invalid" >&2+ return 1+ fi+ if ((current_count > previous_count)); then+ return 0+ fi+ sleep 1+ done+ echo "container log did not report new watchdog evidence: ${expected}" >&2+ return 1+}++force_watchdog_check() {+ local watchdog_pid="$1"+ local expected="$2"+ local previous_count+ if ! previous_count="$(count_container_log_evidence "${expected}")"; then+ return 1+ fi+ if [[ ! "${previous_count}" =~ ^[0-9]+$ ]]; then+ echo "watchdog evidence count is invalid before forced check" >&2+ return 1+ fi+ docker exec "${CONTAINER_NAME}" kill -USR1 "${watchdog_pid}"+ wait_for_new_container_log_evidence "${expected}" "${previous_count}"+}++advance_watchdog_failure() {+ local watchdog_pid="$1"+ local ordinal="$2"+ local previous_count="$3"+ local expected="watchdog health failure ${ordinal}/3 trigger="+ local current_count+ current_count="$(count_container_log_evidence "${expected}")" || return 1+ [[ "${current_count}" =~ ^[0-9]+$ && "${previous_count}" =~ ^[0-9]+$ ]] || return 1+ if ((current_count > previous_count)); then+ return 0+ fi+ if ! docker exec "${CONTAINER_NAME}" kill -USR1 "${watchdog_pid}"; then+ # A scheduled third failure can stop the appliance before the signal.+ current_count="$(count_container_log_evidence "${expected}")" || return 1+ if [[ ! "${current_count}" =~ ^[0-9]+$ ]] || ((current_count <= previous_count)); then+ echo "could not advance watchdog failure ${ordinal}/3" >&2+ return 1+ fi+ fi+ wait_for_new_container_log_evidence "${expected}" "${previous_count}"+}+runtime_config_hash() {docker exec "${CONTAINER_NAME}" sha256sum /data/config/runtime.env | awk '{print $1}'}@@ -233,6 +476,24 @@ assert_pinned_topology_environment() {done}+assert_command_rejected_with() {+ local accepted_message="$1"+ local expected_message="$2"+ shift 2+ local output++ if output="$("$@" 2>&1)"; then+ echo "${accepted_message}" >&2+ return 1+ fi+ if [[ "${output,,}" != *"${expected_message,,}"* ]]; then+ echo "negative isolation probe failed for an unexpected reason" >&2+ echo "expected error containing: ${expected_message}" >&2+ printf '%s\n' "${output}" >&2+ return 1+ fi+}+assert_frontend_database_isolation() {local next_pidlocal nginx_pid@@ -241,6 +502,7 @@ assert_frontend_database_isolation() {local nginx_uidlocal rest_uidlocal assertions+ local frontend_role_passwordlessnext_pid="$(docker exec "${CONTAINER_NAME}" \@@ -412,29 +674,28 @@ SQLprintf 'database privilege assertions:\n%s\n' "${assertions}" >&2return 1}- if docker exec --user autogpt_frontend "${CONTAINER_NAME}" \+ assert_command_rejected_with \+ "frontend database role can assume postgres" \+ 'permission denied to set role "postgres"' \+ docker exec --user autogpt_frontend "${CONTAINER_NAME}" \/usr/bin/env -i \PATH=/usr/lib/postgresql/15/bin:/usr/bin:/bin \PGHOST=/run/postgresql \PGDATABASE=postgres \PGUSER=autogpt_frontend \psql --no-psqlrc --set=ON_ERROR_STOP=1 \- --command='SET ROLE postgres' >/dev/null 2>&1; then- echo "frontend database role can assume postgres" >&2- return 1- fi+ --command='SET ROLE postgres'- if docker exec --user autogpt_proxy "${CONTAINER_NAME}" \+ assert_command_rejected_with \+ "nginx operating-system user can access Valkey without authentication" \+ "noauth authentication required" \+ docker exec --user autogpt_proxy "${CONTAINER_NAME}" \/usr/bin/env -i \PATH=/usr/bin:/bin \/app/autogpt_platform/backend/.venv/bin/python \- /opt/autogpt/single-container/probe.py redis --port 17000 \- >/dev/null 2>&1; then- echo "nginx operating-system user can access Valkey without authentication" >&2- return 1- fi+ /opt/autogpt/single-container/probe.py redis --port 17000- [[ "$(+ frontend_role_passwordless="$(docker exec --user postgres "${CONTAINER_NAME}" \/usr/bin/env -i \PATH=/usr/lib/postgresql/15/bin:/usr/bin:/bin \@@ -443,32 +704,32 @@ SQLPGUSER=postgres \psql --no-psqlrc --tuples-only --no-align --set=ON_ERROR_STOP=1 \--command="SELECT rolpassword IS NULL FROM pg_catalog.pg_authid WHERE rolname = 'autogpt_frontend'"- )" == t ]] || {+ )"+ [[ "${frontend_role_passwordless}" == t ]] || {echo "frontend database role unexpectedly has a password" >&2return 1}- if docker exec --user autogpt_frontend "${CONTAINER_NAME}" \+ assert_command_rejected_with \+ "frontend operating-system user can authenticate as postgres" \+ 'peer authentication failed for user "postgres"' \+ docker exec --user autogpt_frontend "${CONTAINER_NAME}" \/usr/bin/env -i \PATH=/usr/lib/postgresql/15/bin:/usr/bin:/bin \PGHOST=/run/postgresql \PGDATABASE=postgres \PGUSER=postgres \psql --no-psqlrc --set=ON_ERROR_STOP=1 \- --command='SELECT 1' >/dev/null 2>&1; then- echo "frontend operating-system user can authenticate as postgres" >&2- return 1- fi+ --command='SELECT 1'- if docker exec --user autogpt_frontend "${CONTAINER_NAME}" \+ assert_command_rejected_with \+ "frontend operating-system user can access Valkey without authentication" \+ "noauth authentication required" \+ docker exec --user autogpt_frontend "${CONTAINER_NAME}" \/usr/bin/env -i \PATH=/usr/bin:/bin \/app/autogpt_platform/backend/.venv/bin/python \- /opt/autogpt/single-container/probe.py redis --port 17000 \- >/dev/null 2>&1; then- echo "frontend operating-system user can access Valkey without authentication" >&2- return 1- fi+ /opt/autogpt/single-container/probe.py redis --port 17000docker exec "${CONTAINER_NAME}" \curl --fail --silent --show-error --max-time 30 \@@ -609,6 +870,7 @@ assert_request_tokens_absent_from_logs() {local sentinel=AUTOGPT_LOG_SENTINEL_6f2b3cb87e9alocal websocket_key=dGhlIHNhbXBsZSBub25jZQ== # pragma: allowlist secret # gitleaks:allowlocal websocket_status+ local container_logscurl --fail --silent --show-error --max-time 30 \"${PUBLIC_URL}/_agpt/health?token=${sentinel}" >/dev/null@@ -629,9 +891,11 @@ assert_request_tokens_absent_from_logs() {return 1}sleep 2- # Do not use grep -q here: with pipefail an early grep exit can SIGPIPE- # `docker logs` and accidentally turn a positive match into a false result.- if docker logs "${CONTAINER_NAME}" 2>&1 | grep -F "${sentinel}" >/dev/null; then+ if ! container_logs="$(docker logs "${CONTAINER_NAME}" 2>&1)"; then+ echo "could not inspect container logs for request-token leakage" >&2+ return 1+ fi+ if [[ "${container_logs}" == *"${sentinel}"* ]]; thenecho "request token sentinel leaked into container logs" >&2return 1fi@@ -667,11 +931,15 @@ assert_runtime_config_mode() {}assert_prisma_cli_is_prebundled() {+ local container_logsdocker exec "${CONTAINER_NAME}" \/usr/bin/test -f \/opt/prisma-python/binaries/node_modules/prisma/build/index.js- if docker logs "${CONTAINER_NAME}" 2>&1 | \- grep -F "Installing Prisma CLI" >/dev/null; then+ if ! container_logs="$(docker logs "${CONTAINER_NAME}" 2>&1)"; then+ echo "could not inspect container logs for runtime Prisma installation" >&2+ return 1+ fi+ if [[ "${container_logs}" == *"Installing Prisma CLI"* ]]; thenecho "Prisma CLI was installed during container startup" >&2return 1fi@@ -680,27 +948,36 @@ assert_prisma_cli_is_prebundled() {assert_falkordb_binary_contract() {local linkagelocal listeners+ local published_portlinkage="$(docker exec "${CONTAINER_NAME}" /bin/bash -Eeuo pipefail -c 'ldd /opt/falkordb/redis-serverldd /opt/falkordb/falkordb.so')"- if grep -F "not found" <<<"${linkage}" >/dev/null; then+ if [[ "${linkage}" == *"not found"* ]]; thenecho "FalkorDB has an unresolved shared-library dependency" >&2return 1filisteners="$(docker exec "${CONTAINER_NAME}" ss -lnt)"- grep -Eq '127\.0\.0\.1:6380([[:space:]]|$)' <<<"${listeners}" || {+ [[ "${listeners}" =~ 127\.0\.0\.1:6380([[:space:]]|$) ]] || {echo "FalkorDB is not listening on the private loopback address" >&2return 1}- if grep -Eq '(0\.0\.0\.0|\[::\]):6380([[:space:]]|$)' <<<"${listeners}"; then+ if [[ "${listeners}" =~ (0\.0\.0\.0|\[::\]):6380([[:space:]]|$) ]]; thenecho "FalkorDB is listening on a public container interface" >&2return 1fi- if [[ -n "$(docker port "${CONTAINER_NAME}" 6380 2>/dev/null)" ]]; then+ if ! published_port="$(+ docker inspect \+ --format '{{with index .NetworkSettings.Ports "6380/tcp"}}{{json .}}{{end}}' \+ "${CONTAINER_NAME}"+ )"; then+ echo "could not inspect Docker port bindings for FalkorDB" >&2+ return 1+ fi+ if [[ -n "${published_port}" ]]; thenecho "FalkorDB port 6380 is published by Docker" >&2return 1fi@@ -776,17 +1053,19 @@ wait_for_falkordb_restart() {local deadline=$((SECONDS + TIMEOUT_SECONDS))local current_pidwhile ((SECONDS < deadline)); do- current_pid="$(+ if ! current_pid="$(docker exec "${CONTAINER_NAME}" \supervisorctl -c /opt/autogpt/single-container/supervisor/supervisord.conf \- pid state:falkordb 2>/dev/null || true- )"+ pid state:falkordb 2>/dev/null+ )"; then+ current_pid=""+ fiif [[ "${current_pid}" =~ ^[0-9]+$ ]] &&[[ "${current_pid}" != "${previous_pid}" ]] &&appliance_is_healthy_and_armed; thenreturn 0fi- sleep 5+ sleep 1doneecho "FalkorDB did not restart and return the container to healthy" >&2return 1@@ -852,7 +1131,6 @@ assert_unsupported_email_verification_rejected() {elsestatus=$?fi- docker rm --force --volumes "${NEGATIVE_CONTAINER_NAME}" >/dev/null 2>&1 || true((status != 0)) || {echo "image accepted unsupported email verification" >&2return 1@@ -861,9 +1139,8 @@ assert_unsupported_email_verification_rejected() {echo "email-verification rejection probe timed out" >&2return 1}- grep -Fq \- "email verification is not supported by the single-container distribution" \- <<<"${output}" || {+ [[ "${output}" == \+ *"email verification is not supported by the single-container distribution"* ]] || {echo "unsupported email verification failed without an actionable error" >&2return 1}@@ -872,12 +1149,15 @@ assert_unsupported_email_verification_rejected() {# Phase one deliberately supplies no environment, port, volume, entrypoint, or# command override. This is the CI proof for literal `docker run IMAGE`.assert_unsupported_email_verification_rejected+record_timing "unsupported-config-rejection"docker run --detach \--platform "${SMOKE_PLATFORM}" \--name "${CONTAINER_NAME}" \"${SMOKE_IMAGE}" >/dev/nulldiscover_data_volumewait_for_healthy+record_boot_milestones "initial-appliance-boot"+record_timing "initial-appliance-boot"assert_codex_runtime_contractassert_prisma_cli_is_prebundledassert_falkordb_binary_contract@@ -891,8 +1171,10 @@ first_hash="$(runtime_config_hash)"assert_runtime_config_modeassert_frontend_database_isolation+record_timing "phase-one-contracts"assert_clean_stop "after docker stop"docker rm "${CONTAINER_NAME}" >/dev/null+record_timing "phase-one-clean-stop"# A persistent user config must not be able to move internal listeners or bind# them publicly. Environment-pinned topology has higher Pydantic precedence.@@ -909,8 +1191,11 @@ docker run --detach \--volume "${DATA_VOLUME}:/data" \"${SMOKE_IMAGE}" >/dev/nullwait_for_healthy+record_boot_milestones "replacement-appliance-boot"+record_timing "replacement-appliance-boot"-[[ "$(runtime_config_hash)" == "${first_hash}" ]] || {+replacement_config_hash="$(runtime_config_hash)"+[[ "${replacement_config_hash}" == "${first_hash}" ]] || {echo "runtime config changed across container replacement" >&2exit 1}@@ -929,6 +1214,7 @@ assert_redirect http://127.0.0.1:3300/copilot \assert_prefixed_backend_redirectassert_internal_tooling_is_privateassert_request_tokens_absent_from_logs+record_timing "phase-two-contracts"falkordb_pid="$(docker exec "${CONTAINER_NAME}" \@@ -941,8 +1227,11 @@ falkordb_restart_count="$()"docker exec "${CONTAINER_NAME}" kill -TERM "${falkordb_pid}"wait_for_falkordb_restart "${falkordb_pid}"-[[ "$(docker inspect --format '{{.RestartCount}}' "${CONTAINER_NAME}")" == \- "${falkordb_restart_count}" ]] || {+record_timing "falkordb-process-recovery"+current_restart_count="$(+ docker inspect --format '{{.RestartCount}}' "${CONTAINER_NAME}"+)"+[[ "${current_restart_count}" == "${falkordb_restart_count}" ]] || {echo "FalkorDB recovery unexpectedly restarted the whole container" >&2exit 1}@@ -950,12 +1239,40 @@ assert_memory_contract verifywait_for_healthyrestart_count="$(docker inspect --format '{{.RestartCount}}' "${CONTAINER_NAME}")"+watchdog_pid="$(+ docker exec "${CONTAINER_NAME}" \+ supervisorctl -c /opt/autogpt/single-container/supervisor/supervisord.conf \+ pid runtime:watchdog+)"+[[ "${watchdog_pid}" =~ ^[0-9]+$ ]] || {+ echo "watchdog PID is invalid before forced health checks" >&2+ exit 1+}+# Reset the failure counter while healthy. Snapshot every ordinal before+# stopping nginx: scheduled checks can advance the same counter while Docker+# calls are in flight, so either trigger is valid evidence for each failure.+force_watchdog_check \+ "${watchdog_pid}" \+ "watchdog health check passed trigger=forced"+record_timing "watchdog-forced-check-arm"++watchdog_failure_counts=()+for ordinal in 1 2 3; do+ watchdog_failure_counts+=("$(count_container_log_evidence "watchdog health failure ${ordinal}/3 trigger=")")+donedocker exec "${CONTAINER_NAME}" \supervisorctl \-c /opt/autogpt/single-container/supervisor/supervisord.conf \stop runtime:nginx >/dev/null+for ordinal in 1 2 3; do+ advance_watchdog_failure \+ "${watchdog_pid}" "${ordinal}" "${watchdog_failure_counts[ordinal - 1]}"+donewait_for_automatic_restart "$((restart_count + 1))"-[[ "$(runtime_config_hash)" == "${first_hash}" ]] || {+record_boot_milestones "automatic-appliance-restart"+record_timing "automatic-appliance-restart"+restarted_config_hash="$(runtime_config_hash)"+[[ "${restarted_config_hash}" == "${first_hash}" ]] || {echo "runtime config changed across automatic Docker restart" >&2exit 1}@@ -967,5 +1284,11 @@ assert_memory_contract cleanupassert_clean_stop "after the restart test"docker rm "${CONTAINER_NAME}" >/dev/null+record_timing "post-restart-contracts-and-clean-stop"+# Trivy scans the immutable image, independently of either runtime container.+# Synchronizing here preserves foreground scan failure/report propagation+# without holding phase one open before the replacement and restart checks.+wait_for_concurrent_scans+record_timing "final-scan-sync"echo "single-container smoke test passed for ${SMOKE_PLATFORM}"diff --git a/.github/scripts/single_container_cache.py b/.github/scripts/single_container_cache.pynew file mode 100644index 0000000000..8fc9508b25--- /dev/null+++ b/.github/scripts/single_container_cache.py@@ -0,0 +1,42 @@+#!/usr/bin/env python3++import hashlib+import os+++def cache_settings(event: str, ref: str, image: str, arch: str) -> list[str]:+ if arch not in {"amd64", "arm64"}:+ raise ValueError(f"Unsupported cache architecture: {arch}")+ if event == "pull_request":+ return []+ shared = f"{image}:v4-{arch}"+ imports = [shared]+ export = None+ if event == "workflow_dispatch":+ branch_digest = hashlib.sha256(ref.encode()).hexdigest()[:20]+ export = f"{image}:v4-branch-{branch_digest}-{arch}"+ imports.insert(0, export)+ elif event == "push" and ref == "refs/heads/dev":+ export = shared+ settings = [+ f"{target}.cache-from=type=registry,ref={cache_ref}"+ for target in ("backend-server", "single-container")+ for cache_ref in imports+ ]+ if export:+ settings.append(+ f"single-container.cache-to=type=registry,ref={export},mode=max,"+ "oci-mediatypes=true,image-manifest=false,ignore-error=true"+ )+ return settings+++if __name__ == "__main__":+ settings = cache_settings(+ os.environ["GITHUB_EVENT_NAME"],+ os.environ["GITHUB_REF"],+ os.environ["BUILD_CACHE_IMAGE"],+ os.environ["BUILD_CACHE_ARCH"],+ )+ with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as output:+ output.write("set<<CACHE_EOF\n" + "\n".join(settings) + "\nCACHE_EOF\n")diff --git a/.github/scripts/verify_single_container_reports.py b/.github/scripts/verify_single_container_reports.pynew file mode 100644index 0000000000..5102af7745--- /dev/null+++ b/.github/scripts/verify_single_container_reports.py@@ -0,0 +1,151 @@+#!/usr/bin/env python3++import argparse+import json+import re+import xml.etree.ElementTree as ET+from pathlib import Path+++JUNIT_REPORTS = (+ "runtime-helpers.xml",+ "publication-policy.xml",+ "smoke.xml",+)+TRIVY_REPORTS = (+ "trivy-critical.json",+ "trivy-secrets.json",+)+TRIVY_FINDING_KEYS = (+ "Vulnerabilities",+ "Misconfigurations",+ "Secrets",+ "Licenses",+)+++def parse_args() -> argparse.Namespace:+ parser = argparse.ArgumentParser()+ parser.add_argument("--report-dir", type=Path, required=True)+ parser.add_argument("--expected-image", required=True)+ return parser.parse_args()+++def nonnegative_count(element: ET.Element, attribute: str, report: str) -> int:+ value = element.get(attribute)+ if value is None:+ raise ValueError(f"{report} is missing {attribute}")+ try:+ count = int(value)+ except ValueError as error:+ raise ValueError(f"{report} has invalid {attribute}={value!r}") from error+ if count < 0:+ raise ValueError(f"{report} has negative {attribute}={count}")+ return count+++def verify_junit(path: Path) -> None:+ root = ET.parse(path).getroot()+ if root.tag not in {"testsuite", "testsuites"}:+ raise ValueError(f"{path.name} is not a JUnit XML report")+ cases = list(root.iter("testcase"))+ summaries = [(root, cases)]+ if root.tag == "testsuites":+ summaries.extend(+ (suite, list(suite.iter("testcase"))) for suite in root.iter("testsuite")+ )+ if not cases:+ raise ValueError(f"{path.name} contains no test cases")++ for summary, summary_cases in summaries:+ actual = {+ "tests": len(summary_cases),+ "failures": sum(case.find("failure") is not None for case in summary_cases),+ "errors": sum(case.find("error") is not None for case in summary_cases),+ "skipped": sum(case.find("skipped") is not None for case in summary_cases),+ }+ for attribute, actual_count in actual.items():+ declared_count = nonnegative_count(summary, attribute, path.name)+ if declared_count != actual_count:+ raise ValueError(+ f"{path.name} declares {attribute}={declared_count}, "+ f"found {actual_count}"+ )++ failures = sum(case.find("failure") is not None for case in cases)+ errors = sum(case.find("error") is not None for case in cases)+ skipped = sum(case.find("skipped") is not None for case in cases)+ if failures:+ raise ValueError(f"{path.name} reports {failures} failures")+ if errors:+ raise ValueError(f"{path.name} reports {errors} errors")+ if len(cases) == skipped:+ raise ValueError(f"{path.name} contains no successful test cases")+++def verify_trivy(path: Path, expected_image: str) -> None:+ document = json.loads(path.read_text(encoding="utf-8"))+ if not isinstance(document, dict) or "SchemaVersion" not in document:+ raise ValueError(f"{path.name} is not a Trivy JSON report")+ if document.get("SchemaVersion") != 2:+ raise ValueError(f"{path.name} has an unexpected schema version")+ if document.get("ArtifactType") != "container_image":+ raise ValueError(f"{path.name} did not scan a container image")+ if document.get("ArtifactName") != expected_image:+ raise ValueError(+ f"{path.name} scanned {document.get('ArtifactName')!r}, "+ f"expected {expected_image!r}"+ )++ results = document.get("Results", [])+ if not isinstance(results, list):+ raise ValueError(f"{path.name} has an invalid Results value")+ if not results:+ metadata = document.get("Metadata")+ image_id = metadata.get("ImageID") if isinstance(metadata, dict) else None+ if (+ path.name != "trivy-secrets.json"+ or not isinstance(image_id, str)+ or re.fullmatch(r"sha256:[0-9a-f]{64}", image_id) is None+ ):+ raise ValueError(+ f"{path.name} has incomplete scan results or image metadata"+ )++ findings: dict[str, int] = {}+ for result in results:+ if not isinstance(result, dict):+ raise ValueError(f"{path.name} has an invalid result entry")+ for key in ("Target", "Class", "Type"):+ if not isinstance(result.get(key), str) or not result[key]:+ raise ValueError(f"{path.name} result is missing {key}")+ for key in TRIVY_FINDING_KEYS:+ if key not in result:+ continue+ value = result[key]+ if not isinstance(value, list):+ raise ValueError(f"{path.name} has an invalid {key} value")+ if value:+ findings[key] = findings.get(key, 0) + len(value)++ if findings:+ summary = ", ".join(f"{key}={count}" for key, count in sorted(findings.items()))+ raise ValueError(f"{path.name} contains findings: {summary}")+++def main() -> int:+ args = parse_args()+ try:+ for report in JUNIT_REPORTS:+ verify_junit(args.report_dir / report)+ for report in TRIVY_REPORTS:+ verify_trivy(args.report_dir / report, args.expected_image)+ except (OSError, ET.ParseError, json.JSONDecodeError, ValueError) as error:+ print(f"single-container report verification failed: {error}")+ return 1+ print("single-container reports contain no failures or findings")+ return 0+++if __name__ == "__main__":+ raise SystemExit(main())diff --git a/.github/workflows/platform-single-container-docker.yml b/.github/workflows/platform-single-container-docker.ymlindex 4fd2a0c84b..2b45700a29 100644--- a/.github/workflows/platform-single-container-docker.yml+++ b/.github/workflows/platform-single-container-docker.yml@@ -3,33 +3,31 @@ name: AutoGPT Platform - Single-container imageon:push:branches: [dev]- paths:- - ".dockerignore"- - ".github/scripts/platform-single-container-publish.sh"- - ".github/scripts/platform-single-container-smoke.sh"- - ".github/workflows/platform-single-container-docker.yml"- - "autogpt_platform/**"- - "docs/platform/single-container.md"pull_request:paths:- ".dockerignore"- ".github/scripts/platform-single-container-publish.sh"+ - ".github/scripts/run_command_junit.py"+ - ".github/scripts/run_unittest_junit.py"+ - ".github/scripts/verify_single_container_reports.py"+ - ".github/scripts/single_container_cache.py"- ".github/scripts/platform-single-container-smoke.sh"- ".github/workflows/platform-single-container-docker.yml"- - "autogpt_platform/**"+ - "autogpt_platform/backend/Dockerfile"+ - "autogpt_platform/backend/poetry.lock"+ - "autogpt_platform/backend/pyproject.toml"+ - "autogpt_platform/docker-compose.yml"+ - "autogpt_platform/docker-compose.platform.yml"+ - "autogpt_platform/frontend/package.json"+ - "autogpt_platform/frontend/pnpm-lock.yaml"+ - "autogpt_platform/single-container/**"- "docs/platform/single-container.md"release:types: [published]workflow_dispatch:- inputs:- publish:- description: "Publish the current dev commit to Docker Hub"- required: true- type: boolean- default: falseconcurrency:- group: ${{ github.workflow }}-${{ (github.event_name == 'workflow_dispatch' || github.event_name == 'release') && 'publication' || github.ref }}+ group: ${{ github.workflow }}-${{ github.event_name == 'release' && 'publication' || github.event_name == 'workflow_dispatch' && github.run_id || github.ref }}cancel-in-progress: ${{ github.event_name == 'pull_request' }}permissions:@@ -37,10 +35,11 @@ permissions:env:DEPLOY_IMAGE: docker.io/significantgravitas/autogpt+ BUILD_CACHE_IMAGE: ghcr.io/significant-gravitas/autogpt-single-container-buildcachejobs:authorize-publication:- name: Authorize publication source+ name: Evaluate publication eligibilityruns-on: ubuntu-24.04outputs:allowed: ${{ steps.policy.outputs.allowed }}@@ -50,19 +49,19 @@ jobs:with:persist-credentials: false- - name: Validate publication source+ - name: Determine publication eligibilityid: policyshell: bashenv:- PUBLISH_REQUESTED: ${{ inputs.publish }}RELEASE_PRERELEASE: ${{ github.event.release.prerelease }}RELEASE_TAG: ${{ github.event.release.tag_name }}run: bash .github/scripts/platform-single-container-publish.sh authorizebuild-and-scan:name: Build, smoke, and scan (${{ matrix.platform }})- needs: authorize-publication- if: ${{ github.event_name != 'release' || needs.authorize-publication.outputs.allowed == 'true' }}+ permissions:+ contents: read+ packages: writeruns-on: ${{ matrix.runner }}timeout-minutes: 240strategy:@@ -79,24 +78,69 @@ jobs:- name: Check out repositoryuses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1with:+ ref: ${{ github.sha }}persist-credentials: false- - name: Test appliance runtime helpers- shell: bash- run: python3 -m unittest discover -s autogpt_platform/single-container/tests -p 'test_*.py' -v-- - name: Test publication helpers+ - name: Initialize validation reportsshell: bash- run: bash .github/scripts/platform-single-container-publish.sh self-test+ env:+ REPORT_DIR: ${{ runner.temp }}/platform-single-container-results+ run: |+ mkdir -p "$REPORT_DIR"+ jq -n \+ --arg sha "$GITHUB_SHA" \+ --arg platform "${{ matrix.platform }}" \+ '{sha: $sha, platform: $platform, validated: false}' \+ > "$REPORT_DIR/status.json"- name: Set up Docker Buildxuses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0with:version: v0.36.0+ cleanup: falsedriver-opts: |image=docker.io/moby/buildkit:v0.32.0@sha256:1f8167fcb0eca5b7126353d35299386945cbb8949cc516c592a49f80cfce4fa2+ - name: Authenticate to GHCR build cache+ if: ${{ github.event_name != 'pull_request' }}+ uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0+ with:+ registry: ghcr.io+ username: ${{ github.actor }}+ password: ${{ secrets.GITHUB_TOKEN }}++ - name: Configure registry build cache+ id: cache+ shell: bash+ env:+ BUILD_CACHE_ARCH: ${{ matrix.suffix }}+ run: python3 .github/scripts/single_container_cache.py++ - name: Test appliance runtime helpers+ shell: bash+ env:+ REPORT_DIR: ${{ runner.temp }}/platform-single-container-results+ run: >-+ python3 .github/scripts/run_unittest_junit.py+ --start-directory autogpt_platform/single-container/tests+ --pattern 'test_*.py'+ --allow-skip test_documented_operations.DocumentedOperationsTest.test_restored_launch_supports_bash_3_when_available+ --output "$REPORT_DIR/runtime-helpers.xml"++ - name: Test publication helpers+ shell: bash+ env:+ REPORT_DIR: ${{ runner.temp }}/platform-single-container-results+ run: >-+ python3 .github/scripts/run_command_junit.py+ --name publication-policy+ --classname single-container.publication+ --output "$REPORT_DIR/publication-policy.xml"+ -- bash .github/scripts/platform-single-container-publish.sh self-test+- name: Build image without publishing+ id: build-image+ background: trueuses: docker/bake-action@d3418bd7d0e9324001bca92fa8ba175ea7e6dc9b # v7.3.0with:source: .@@ -104,47 +148,148 @@ jobs:targets: single-containerset: |*.platform=${{ matrix.platform }}- backend-server.cache-from=type=gha,scope=platform-single-container-backend-${{ matrix.suffix }}- backend-server.cache-to=type=gha,scope=platform-single-container-backend-${{ matrix.suffix }},mode=max,ignore-error=truesingle-container.args.IMAGE_VERSION=sha-${{ github.sha }}single-container.args.VCS_REF=${{ github.sha }}- single-container.cache-from=type=gha,scope=platform-single-container-${{ matrix.suffix }}- single-container.cache-to=type=gha,scope=platform-single-container-${{ matrix.suffix }},mode=max,ignore-error=truesingle-container.output=type=dockersingle-container.tags=autogpt-platform-single-container:ci-${{ matrix.suffix }}+ ${{ steps.cache.outputs.set }}++ - name: Wait for image build+ wait: build-image++ - name: Verify image source revision+ shell: bash+ env:+ REPORT_DIR: ${{ runner.temp }}/platform-single-container-results+ SMOKE_IMAGE: autogpt-platform-single-container:ci-${{ matrix.suffix }}+ run: |+ actual_revision="$(docker image inspect --format '{{ index .Config.Labels "org.opencontainers.image.revision" }}' "$SMOKE_IMAGE")"+ if [[ "$actual_revision" != "$GITHUB_SHA" ]]; then+ echo "expected image revision $GITHUB_SHA, got $actual_revision" >&2+ exit 1+ fi+ image_id="$(docker image inspect --format '{{ .Id }}' "$SMOKE_IMAGE")"+ jq -n \+ --arg sha "$GITHUB_SHA" \+ --arg platform "${{ matrix.platform }}" \+ --arg image "$SMOKE_IMAGE" \+ --arg image_id "$image_id" \+ '{sha: $sha, platform: $platform, image: $image, image_id: $image_id, validated: false}' \+ > "$REPORT_DIR/status.json"- name: Smoke-test the complete image+ id: complete-image-smoke+ background: trueshell: bashenv:+ REPORT_DIR: ${{ runner.temp }}/platform-single-container-resultsSMOKE_IMAGE: autogpt-platform-single-container:ci-${{ matrix.suffix }}SMOKE_PLATFORM: ${{ matrix.platform }}- run: bash .github/scripts/platform-single-container-smoke.sh+ SMOKE_SCAN_COMPLETION_FILE: ${{ runner.temp }}/platform-single-container-results/scans-complete+ run: >-+ python3 .github/scripts/run_command_junit.py+ --name complete-image-smoke+ --classname single-container.smoke+ --output "$REPORT_DIR/smoke.xml"+ -- bash .github/scripts/platform-single-container-smoke.sh- name: Scan for fixable critical vulnerabilities+ id: vulnerability_scan+ continue-on-error: trueuses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0with:image-ref: autogpt-platform-single-container:ci-${{ matrix.suffix }}- format: table+ scanners: vuln,secret+ format: json+ output: ${{ runner.temp }}/platform-single-container-results/trivy-critical.jsonignore-unfixed: trueseverity: CRITICALexit-code: "1"timeout: 30m- name: Scan image filesystem for embedded secrets+ id: secret_scan+ continue-on-error: trueuses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0with:image-ref: autogpt-platform-single-container:ci-${{ matrix.suffix }}scanners: secret- format: table+ format: json+ output: ${{ runner.temp }}/platform-single-container-results/trivy-secrets.jsonseverity: HIGH,CRITICALexit-code: "1"timeout: 30m+ - name: Signal concurrent scans complete+ if: ${{ !cancelled() }}+ shell: bash+ env:+ SCAN_COMPLETION_FILE: ${{ runner.temp }}/platform-single-container-results/scans-complete+ run: touch "$SCAN_COMPLETION_FILE"++ - name: Wait for complete-image smoke test+ wait: complete-image-smoke++ - name: Verify machine-readable reports+ if: ${{ !cancelled() }}+ shell: bash+ env:+ REPORT_DIR: ${{ runner.temp }}/platform-single-container-results+ SMOKE_IMAGE: autogpt-platform-single-container:ci-${{ matrix.suffix }}+ SECRET_SCAN_OUTCOME: ${{ steps.secret_scan.outcome }}+ VULNERABILITY_SCAN_OUTCOME: ${{ steps.vulnerability_scan.outcome }}+ run: |+ scan_failed=0+ if [[ "$VULNERABILITY_SCAN_OUTCOME" != "success" ]]; then+ echo "vulnerability scan action concluded $VULNERABILITY_SCAN_OUTCOME" >&2+ scan_failed=1+ fi+ if [[ "$SECRET_SCAN_OUTCOME" != "success" ]]; then+ echo "secret scan action concluded $SECRET_SCAN_OUTCOME" >&2+ scan_failed=1+ fi+ reports=(+ runtime-helpers.xml+ publication-policy.xml+ smoke.xml+ trivy-critical.json+ trivy-secrets.json+ )+ for report in "${reports[@]}"; do+ if [[ ! -s "$REPORT_DIR/$report" ]]; then+ echo "missing machine-readable report: $report" >&2+ exit 1+ fi+ done+ python3 .github/scripts/verify_single_container_reports.py \+ --report-dir "$REPORT_DIR" \+ --expected-image "$SMOKE_IMAGE"+ ((scan_failed == 0))++ - name: Mark validation complete+ shell: bash+ env:+ REPORT_DIR: ${{ runner.temp }}/platform-single-container-results+ run: |+ jq '.validated = true' "$REPORT_DIR/status.json" > "$REPORT_DIR/status.complete.json"+ mv "$REPORT_DIR/status.complete.json" "$REPORT_DIR/status.json"++ - name: Upload single-container reports+ if: ${{ !cancelled() }}+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1+ with:+ name: platform-single-container-ci-${{ matrix.suffix }}+ path: ${{ runner.temp }}/platform-single-container-results+ if-no-files-found: error+ retention-days: 7+publish-platform-digests:name: Publish, smoke, and scan (${{ matrix.platform }})- if: ${{ needs.authorize-publication.outputs.allowed == 'true' && needs.build-and-scan.result == 'success' }}+ if: ${{ github.event_name == 'release' && needs.authorize-publication.outputs.allowed == 'true' && needs.build-and-scan.result == 'success' }}needs: [authorize-publication, build-and-scan]- environment: ${{ github.event_name == 'release' && 'dockerhub-release' || 'dockerhub-publish' }}+ permissions:+ contents: read+ environment: dockerhub-releaseruns-on: ${{ matrix.runner }}timeout-minutes: 240strategy:@@ -167,6 +312,7 @@ jobs:uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0with:version: v0.36.0+ cleanup: falsedriver-opts: |image=docker.io/moby/buildkit:v0.32.0@sha256:1f8167fcb0eca5b7126353d35299386945cbb8949cc516c592a49f80cfce4fa2@@ -185,12 +331,8 @@ jobs:targets: single-containerset: |*.platform=${{ matrix.platform }}- backend-server.cache-from=type=gha,scope=platform-single-container-backend-${{ matrix.suffix }}- backend-server.cache-to=type=gha,scope=platform-single-container-backend-${{ matrix.suffix }},mode=max,ignore-error=truesingle-container.args.IMAGE_VERSION=sha-${{ github.sha }}single-container.args.VCS_REF=${{ github.sha }}- single-container.cache-from=type=gha,scope=platform-single-container-${{ matrix.suffix }}- single-container.cache-to=type=gha,scope=platform-single-container-${{ matrix.suffix }},mode=max,ignore-error=truesingle-container.tags=${{ env.DEPLOY_IMAGE }}single-container.attest=type=provenance,mode=maxsingle-container.attest=type=sbom@@ -272,9 +414,9 @@ jobs:publish-manifest:name: Publish multi-platform manifest- if: ${{ needs.authorize-publication.outputs.allowed == 'true' && needs.publish-platform-digests.result == 'success' }}+ if: ${{ github.event_name == 'release' && needs.authorize-publication.outputs.allowed == 'true' && needs.publish-platform-digests.result == 'success' }}needs: [authorize-publication, publish-platform-digests]- environment: ${{ github.event_name == 'release' && 'dockerhub-release' || 'dockerhub-publish' }}+ environment: dockerhub-releaseruns-on: ubuntu-24.04timeout-minutes: 30steps:diff --git a/autogpt_platform/backend/backend/executor/scheduler.py b/autogpt_platform/backend/backend/executor/scheduler.pyindex c679e0db01..6e674d8a58 100644--- a/autogpt_platform/backend/backend/executor/scheduler.py+++ b/autogpt_platform/backend/backend/executor/scheduler.py@@ -2109,7 +2109,10 @@ class Scheduler(AppService):hours=6,# Due now rather than called inline below: run_service() is what# starts the event loop uvicorn binds the RPC port on.- next_run_time=datetime.now(timezone.utc),+ next_run_time=datetime.now(timezone.utc)+ + timedelta(+ hours=0 if config.scheduler_startup_embedding_backfill else 6+ ),replace_existing=True,max_instances=1, # Prevent overlapping runsmisfire_grace_time=None,diff --git a/autogpt_platform/backend/backend/executor/scheduler_unit_test.py b/autogpt_platform/backend/backend/executor/scheduler_unit_test.pyindex eaf7df107a..0999fe25f7 100644--- a/autogpt_platform/backend/backend/executor/scheduler_unit_test.py+++ b/autogpt_platform/backend/backend/executor/scheduler_unit_test.py@@ -48,6 +48,7 @@ from backend.util.exceptions import (ExpertPrivateTenancyNotFoundError,UserPaywalledError,)+from backend.util.settings import Config_SCHEDULER_PATH = "backend.executor.scheduler"@@ -1510,7 +1511,9 @@ class _StartupRun(NamedTuple):backfill_calls_at_readiness: int-def _registered_jobs(monkeypatch, interval_hours: int) -> _StartupRun:+def _registered_jobs(+ monkeypatch, interval_hours: int, startup_embedding_backfill: bool = True+) -> _StartupRun:"""Drive ``Scheduler.run_service`` with every heavy dependency stubbed anda mock APScheduler, recording what it registered and what it ran beforehanding over to ``AppService.run_service``."""@@ -1518,6 +1521,10 @@ def _registered_jobs(monkeypatch, interval_hours: int) -> _StartupRun:f"{_SCHEDULER_PATH}.config.stripe_tier_reconcile_interval_hours",interval_hours,)+ monkeypatch.setattr(+ f"{_SCHEDULER_PATH}.config.scheduler_startup_embedding_backfill",+ startup_embedding_backfill,+ )mock_scheduler = MagicMock()embedding_backfill = MagicMock(return_value=None)at_readiness = []@@ -1573,6 +1580,45 @@ def test_reconcile_stripe_tiers_interval_follows_config_setting(monkeypatch):assert match.kwargs["seconds"] == 12 * 3600+def test_startup_embedding_backfill_defaults_on():+ assert Config.model_fields["scheduler_startup_embedding_backfill"].default is True+++def test_startup_embedding_backfill_can_be_disabled(monkeypatch):+ before = datetime.now(timezone.utc)+ run = _registered_jobs(+ monkeypatch, interval_hours=6, startup_embedding_backfill=False+ )++ run.embedding_backfill.assert_not_called()+ job = next(+ c for c in run.add_job_calls if c.kwargs["id"] == "ensure_embeddings_coverage"+ )+ assert job.kwargs["trigger"] == "interval"+ assert job.kwargs["hours"] == 6+ assert before + timedelta(hours=6) <= job.kwargs["next_run_time"]+++def test_startup_embedding_backfill_runs_in_the_existing_background_job(monkeypatch):+ before = datetime.now(timezone.utc)+ run = _registered_jobs(monkeypatch, interval_hours=6)++ run.embedding_backfill.assert_not_called()+ jobs = [+ c for c in run.add_job_calls if c.kwargs["id"] == "ensure_embeddings_coverage"+ ]+ assert len(jobs) == 1+ job = jobs[0]+ assert before <= job.kwargs["next_run_time"] <= datetime.now(timezone.utc)+ assert job.kwargs["trigger"] == "interval"+ assert job.kwargs["hours"] == 6+ assert job.kwargs["max_instances"] == 1+ assert job.kwargs["misfire_grace_time"] is None+ assert job.kwargs["coalesce"] is True+ job.args[0]()+ run.embedding_backfill.assert_called_once_with()++def test_embedding_backfill_does_not_delay_rpc_readiness(monkeypatch):"""``AppService.run_service`` starts the event loop uvicorn is scheduledonto, so anything run before it keeps the RPC port closed. The backfilldiff --git a/autogpt_platform/backend/backend/util/settings.py b/autogpt_platform/backend/backend/util/settings.pyindex feb804130c..825068c1dd 100644--- a/autogpt_platform/backend/backend/util/settings.py+++ b/autogpt_platform/backend/backend/util/settings.py@@ -651,6 +651,14 @@ class Config(UpdateTrackingModel["Config"], BaseSettings):),)+ scheduler_startup_embedding_backfill: bool = Field(+ default=True,+ description=(+ "Run the first search embedding coverage backfill in the background "+ "when the scheduler starts instead of waiting six hours"+ ),+ )+upload_file_size_limit_mb: int = Field(default=256,ge=1,diff --git a/autogpt_platform/single-container/tests/test_ci_registry_cache.py b/autogpt_platform/single-container/tests/test_ci_registry_cache.pynew file mode 100644index 0000000000..13be35c767--- /dev/null+++ b/autogpt_platform/single-container/tests/test_ci_registry_cache.py@@ -0,0 +1,67 @@+import importlib.util+import unittest+from pathlib import Path+++SCRIPT = (+ Path(__file__).resolve().parents[3] / ".github/scripts/single_container_cache.py"+)+SPEC = importlib.util.spec_from_file_location("single_container_cache", SCRIPT)+cache = importlib.util.module_from_spec(SPEC)+SPEC.loader.exec_module(cache)+IMAGE = "ghcr.io/example/appliance-cache"+++class RegistryCacheTests(unittest.TestCase):+ def settings(self, event, ref="refs/heads/dev", arch="amd64"):+ return cache.cache_settings(event, ref, IMAGE, arch)++ def test_pull_requests_never_read_or_write_registry_cache(self):+ self.assertEqual(self.settings("pull_request"), [])++ def test_only_dev_push_writes_the_shared_validation_cache(self):+ settings = self.settings("push")+ self.assertIn(+ f"single-container.cache-to=type=registry,ref={IMAGE}:v4-amd64,mode=max,oci-mediatypes=true,image-manifest=false,ignore-error=true",+ settings,+ )+ self.assertFalse(+ any("cache-to" in s for s in self.settings("push", "refs/heads/test"))+ )++ def test_dispatch_cache_is_stable_branch_scoped_and_never_shared_cache(self):+ for ref in ("refs/heads/dev", "refs/heads/test"):+ settings = self.settings("workflow_dispatch", ref)+ writes = [s for s in settings if "cache-to" in s]+ self.assertEqual(len(writes), 1)+ self.assertIn(":v4-branch-", writes[0])+ self.assertIn("mode=max", writes[0])+ self.assertIn("ignore-error=true", writes[0])+ self.assertIn(+ f"single-container.cache-from=type=registry,ref={IMAGE}:v4-amd64",+ settings,+ )+ self.assertEqual(settings, self.settings("workflow_dispatch", ref))+ self.assertNotEqual(+ self.settings("workflow_dispatch", "refs/heads/one"),+ self.settings("workflow_dispatch", "refs/heads/two"),+ )++ def test_validation_cache_reads_are_architecture_scoped(self):+ settings = self.settings("release", "refs/tags/v1")+ self.assertTrue(all("cache-from" in s and ":v4-amd64" in s for s in settings))+ self.assertTrue(+ all(":v4-arm64" in s for s in self.settings("release", arch="arm64"))+ )++ def test_release_digest_build_does_not_trust_writable_ci_cache(self):+ workflow = (+ SCRIPT.parents[1] / "workflows/platform-single-container-docker.yml"+ ).read_text(encoding="utf-8")+ release_job = workflow.split(" publish-platform-digests:", 1)[1].split(+ " publish-manifest:", 1+ )[0]+ self.assertNotIn("cache-from", release_job)+ self.assertNotIn("cache-to", release_job)+ self.assertNotIn("GHCR", release_job)+ self.assertNotIn("packages:", release_job)diff --git a/autogpt_platform/single-container/tests/test_ci_report_verifier.py b/autogpt_platform/single-container/tests/test_ci_report_verifier.pynew file mode 100644index 0000000000..79e06bc966--- /dev/null+++ b/autogpt_platform/single-container/tests/test_ci_report_verifier.py@@ -0,0 +1,271 @@+import json+import subprocess+import sys+import tempfile+import unittest+from pathlib import Path+++REPOSITORY_ROOT = Path(__file__).resolve().parents[3]+REPORT_VERIFIER = REPOSITORY_ROOT / ".github/scripts/verify_single_container_reports.py"+++class ReportVerifierTests(unittest.TestCase):+ def setUp(self):+ self.temporary_directory = tempfile.TemporaryDirectory()+ self.report_dir = Path(self.temporary_directory.name)+ passing_junit = (+ '<testsuite tests="1" failures="0" errors="0" skipped="0">'+ '<testcase classname="ci" name="passes" />'+ "</testsuite>"+ )+ for name in ("runtime-helpers.xml", "publication-policy.xml", "smoke.xml"):+ (self.report_dir / name).write_text(passing_junit, encoding="utf-8")+ passing_trivy = {+ "SchemaVersion": 2,+ "ArtifactName": "single-container:test",+ "ArtifactType": "container_image",+ "Results": [+ {+ "Target": "single-container:test (debian)",+ "Class": "os-pkgs",+ "Type": "debian",+ "Vulnerabilities": [],+ "Secrets": [],+ }+ ],+ }+ for name in ("trivy-critical.json", "trivy-secrets.json"):+ (self.report_dir / name).write_text(+ json.dumps(passing_trivy), encoding="utf-8"+ )++ def tearDown(self):+ self.temporary_directory.cleanup()++ def run_verifier(self):+ return subprocess.run(+ [+ sys.executable,+ str(REPORT_VERIFIER),+ "--report-dir",+ str(self.report_dir),+ "--expected-image",+ "single-container:test",+ ],+ check=False,+ capture_output=True,+ text=True,+ )++ def test_passing_reports_succeed(self):+ result = self.run_verifier()++ self.assertEqual(result.returncode, 0, result.stdout + result.stderr)++ def test_clean_secret_scan_accepts_empty_or_omitted_results(self):+ path = self.report_dir / "trivy-secrets.json"+ report = json.loads(path.read_text(encoding="utf-8"))+ report["Metadata"] = {"ImageID": "sha256:" + "a" * 64}+ for results_present in (True, False):+ with self.subTest(results_present=results_present):+ if results_present:+ report["Results"] = []+ else:+ del report["Results"]+ path.write_text(json.dumps(report), encoding="utf-8")+ result = self.run_verifier()+ self.assertEqual(result.returncode, 0, result.stdout + result.stderr)++ def test_empty_secret_results_without_image_metadata_are_rejected(self):+ path = self.report_dir / "trivy-secrets.json"+ report = json.loads(path.read_text(encoding="utf-8"))+ report["Results"] = []+ path.write_text(json.dumps(report), encoding="utf-8")+ self.assertNotEqual(self.run_verifier().returncode, 0)++ def test_empty_vulnerability_results_are_rejected(self):+ path = self.report_dir / "trivy-critical.json"+ report = json.loads(path.read_text(encoding="utf-8"))+ report["Metadata"] = {"ImageID": "sha256:" + "a" * 64}+ report["Results"] = []+ path.write_text(json.dumps(report), encoding="utf-8")+ self.assertNotEqual(self.run_verifier().returncode, 0)++ def test_junit_failure_is_rejected_even_when_summary_claims_success(self):+ (self.report_dir / "smoke.xml").write_text(+ '<testsuite tests="1" failures="0" errors="0" skipped="0">'+ '<testcase classname="ci" name="fails"><failure /></testcase>'+ "</testsuite>",+ encoding="utf-8",+ )++ result = self.run_verifier()++ self.assertNotEqual(result.returncode, 0)+ self.assertIn("declares failures=0, found 1", result.stdout)++ def test_junit_error_count_is_rejected(self):+ (self.report_dir / "runtime-helpers.xml").write_text(+ '<testsuite tests="1" failures="0" errors="1" skipped="0">'+ '<testcase classname="ci" name="errors"><error /></testcase>'+ "</testsuite>",+ encoding="utf-8",+ )++ result = self.run_verifier()++ self.assertNotEqual(result.returncode, 0)+ self.assertIn("reports 1 errors", result.stdout)++ def test_junit_aggregate_failure_is_rejected(self):+ (self.report_dir / "smoke.xml").write_text(+ '<testsuites tests="1" failures="1" errors="0" skipped="0">'+ '<testsuite tests="1" failures="0" errors="0" skipped="0">'+ '<testcase classname="ci" name="misreported" />'+ "</testsuite>"+ "</testsuites>",+ encoding="utf-8",+ )++ result = self.run_verifier()++ self.assertNotEqual(result.returncode, 0)+ self.assertIn("declares failures=1, found 0", result.stdout)++ def test_nested_junit_suites_are_accepted(self):+ (self.report_dir / "smoke.xml").write_text(+ '<testsuites tests="1" failures="0" errors="0" skipped="0">'+ '<testsuite tests="1" failures="0" errors="0" skipped="0">'+ '<testsuite tests="1" failures="0" errors="0" skipped="0">'+ '<testcase classname="ci" name="nested" />'+ "</testsuite>"+ "</testsuite>"+ "</testsuites>",+ encoding="utf-8",+ )++ result = self.run_verifier()++ self.assertEqual(result.returncode, 0, result.stdout + result.stderr)++ def test_junit_count_mismatch_is_rejected(self):+ (self.report_dir / "publication-policy.xml").write_text(+ '<testsuite tests="99" failures="0" errors="0" skipped="0">'+ '<testcase classname="ci" name="only-one" />'+ "</testsuite>",+ encoding="utf-8",+ )++ result = self.run_verifier()++ self.assertNotEqual(result.returncode, 0)+ self.assertIn("declares tests=99, found 1", result.stdout)++ def test_junit_skipped_only_report_is_rejected(self):+ (self.report_dir / "smoke.xml").write_text(+ '<testsuite tests="1" failures="0" errors="0" skipped="1">'+ '<testcase classname="ci" name="skipped"><skipped /></testcase>'+ "</testsuite>",+ encoding="utf-8",+ )++ result = self.run_verifier()++ self.assertNotEqual(result.returncode, 0)+ self.assertIn("contains no successful test cases", result.stdout)++ def test_trivy_findings_are_rejected(self):+ report = {+ "SchemaVersion": 2,+ "ArtifactName": "single-container:test",+ "ArtifactType": "container_image",+ "Results": [+ {+ "Target": "single-container:test (debian)",+ "Class": "os-pkgs",+ "Type": "debian",+ "Vulnerabilities": [{"ID": "CVE"}],+ }+ ],+ }+ (self.report_dir / "trivy-critical.json").write_text(+ json.dumps(report), encoding="utf-8"+ )++ result = self.run_verifier()++ self.assertNotEqual(result.returncode, 0)+ self.assertIn("Vulnerabilities=1", result.stdout)++ def test_non_trivy_json_is_rejected(self):+ (self.report_dir / "trivy-secrets.json").write_text("{}", encoding="utf-8")++ result = self.run_verifier()++ self.assertNotEqual(result.returncode, 0)+ self.assertIn("is not a Trivy JSON report", result.stdout)++ def test_trivy_report_without_scanned_results_is_rejected(self):+ (self.report_dir / "trivy-secrets.json").write_text(+ json.dumps(+ {+ "SchemaVersion": 2,+ "ArtifactName": "single-container:test",+ "ArtifactType": "container_image",+ "Results": None,+ }+ ),+ encoding="utf-8",+ )++ result = self.run_verifier()++ self.assertNotEqual(result.returncode, 0)+ self.assertIn("invalid Results value", result.stdout)++ def test_trivy_result_without_scan_identity_is_rejected(self):+ (self.report_dir / "trivy-secrets.json").write_text(+ json.dumps(+ {+ "SchemaVersion": 2,+ "ArtifactName": "single-container:test",+ "ArtifactType": "container_image",+ "Results": [{}],+ }+ ),+ encoding="utf-8",+ )++ result = self.run_verifier()++ self.assertNotEqual(result.returncode, 0)+ self.assertIn("result is missing Target", result.stdout)++ def test_trivy_null_finding_list_is_rejected(self):+ report = json.loads((self.report_dir / "trivy-critical.json").read_text())+ report["Results"][0]["Vulnerabilities"] = None+ (self.report_dir / "trivy-critical.json").write_text(+ json.dumps(report), encoding="utf-8"+ )++ result = self.run_verifier()++ self.assertNotEqual(result.returncode, 0)+ self.assertIn("invalid Vulnerabilities value", result.stdout)++ def test_trivy_wrong_image_is_rejected(self):+ report = json.loads((self.report_dir / "trivy-critical.json").read_text())+ report["ArtifactName"] = "other:image"+ (self.report_dir / "trivy-critical.json").write_text(+ json.dumps(report), encoding="utf-8"+ )++ result = self.run_verifier()++ self.assertNotEqual(result.returncode, 0)+ self.assertIn("scanned 'other:image'", result.stdout)+++if __name__ == "__main__":+ unittest.main()diff --git a/autogpt_platform/single-container/tests/test_ci_smoke_stop.py b/autogpt_platform/single-container/tests/test_ci_smoke_stop.pynew file mode 100644index 0000000000..c301f32b05--- /dev/null+++ b/autogpt_platform/single-container/tests/test_ci_smoke_stop.py@@ -0,0 +1,75 @@+import re+import shutil+import subprocess+import sys+import unittest+from pathlib import Path+from unittest.mock import patch+++SMOKE = (+ Path(__file__).resolve().parents[3]+ / ".github/scripts/platform-single-container-smoke.sh"+)+++def bash_executable() -> str:+ if sys.platform != "win32":+ return "/bin/bash"+ executable = shutil.which("bash")+ if executable is None:+ raise RuntimeError("Git Bash must be installed and on PATH for this test")+ return executable+++class SmokeStopTests(unittest.TestCase):+ def test_uses_fixed_bash_path_on_posix(self):+ with patch("sys.platform", "linux"), patch("shutil.which") as which:+ self.assertEqual(bash_executable(), "/bin/bash")+ which.assert_not_called()++ def test_resolves_bash_executable_on_windows(self):+ selected = "C:/Program Files/Git/bin/bash.exe"+ with patch("sys.platform", "win32"), patch(+ "shutil.which", return_value=selected+ ) as which:+ self.assertEqual(bash_executable(), selected)+ which.assert_called_once_with("bash")++ def test_missing_windows_bash_fails_instead_of_skipping(self):+ with patch("sys.platform", "win32"), patch("shutil.which", return_value=None):+ with self.assertRaisesRegex(RuntimeError, "Git Bash"):+ bash_executable()++ def test_invalid_or_empty_finish_time_fails_with_diagnostic(self):+ function = re.search(+ r"^assert_clean_stop\(\) \{.*?^\}",+ SMOKE.read_text(encoding="utf-8"),+ re.MULTILINE | re.DOTALL,+ ).group()+ for date_status in (0, 1):+ with self.subTest(date_status=date_status):+ script = (+ "set -Eeuo pipefail\n"+ "STOCK_DOCKER_STOP_TIMEOUT=10\n"+ "CONTAINER_NAME=test-container\n"+ "docker() { printf '%s' invalid-timestamp; }\n"+ f"date() {{ return {date_status}; }}\n"+ "awk() { echo UNEXPECTED_AWK >&2; return 1; }\n"+ f"{function}\n"+ "assert_clean_stop test-stop\n"+ )+ result = subprocess.run(+ [bash_executable(), "--noprofile", "--norc", "-s"],+ input=script.encode("utf-8"),+ capture_output=True,+ check=False,+ )+ stderr = result.stderr.decode("utf-8")+ self.assertNotEqual(result.returncode, 0)+ self.assertIn("invalid container finish timestamp", stderr)+ self.assertNotIn("UNEXPECTED_AWK", stderr)+++if __name__ == "__main__":+ unittest.main()diff --git a/autogpt_platform/single-container/tests/test_ci_watchdog_evidence.py b/autogpt_platform/single-container/tests/test_ci_watchdog_evidence.pynew file mode 100644index 0000000000..03c34c460e--- /dev/null+++ b/autogpt_platform/single-container/tests/test_ci_watchdog_evidence.py@@ -0,0 +1,101 @@+import re+import subprocess+import unittest++from test_ci_smoke_stop import SMOKE, bash_executable+++class WatchdogEvidenceTests(unittest.TestCase):+ def run_case(+ self,+ *,+ already_seen=0,+ previous_count=0,+ signal_adds_evidence=1,+ signal_status=0,+ observed_ordinal=2,+ ):+ source = SMOKE.read_text(encoding="utf-8")+ functions = []+ for name in (+ "count_container_log_evidence",+ "wait_for_new_container_log_evidence",+ "advance_watchdog_failure",+ ):+ match = re.search(+ rf"^{name}\(\) \{{.*?^\}}", source, re.MULTILINE | re.DOTALL+ )+ self.assertIsNotNone(match, f"missing {name}")+ functions.append(match.group())+ script = (+ "\n".join(functions)+ + f"""+set -Eeuo pipefail+TIMEOUT_SECONDS=2+CONTAINER_NAME=test+COUNT={already_seen}+docker() {{+ if [[ "$1" == logs ]]; then+ for ((i=0; i<COUNT; i++)); do+ printf '[single-container] watchdog health failure {observed_ordinal}/3 trigger=scheduled\\n'+ done+ return 0+ fi+ printf 'signal\\n'+ COUNT=$((COUNT + {signal_adds_evidence}))+ return {signal_status}+}}+advance_watchdog_failure 123 2 {previous_count}+"""+ )+ result = subprocess.run(+ [bash_executable(), "--noprofile", "--norc", "-s"],+ input=script.encode("utf-8"),+ capture_output=True,+ timeout=5,+ check=False,+ )+ return subprocess.CompletedProcess(+ result.args,+ result.returncode,+ result.stdout.decode("utf-8"),+ result.stderr.decode("utf-8"),+ )++ def test_scheduled_failure_before_signal_needs_no_extra_signal(self):+ result = self.run_case(already_seen=1)+ self.assertEqual(result.returncode, 0, result.stderr)+ self.assertNotIn("signal", result.stdout)++ def test_forced_check_still_requires_new_failure_evidence(self):+ result = self.run_case()+ self.assertEqual(result.returncode, 0, result.stderr)+ self.assertIn("signal", result.stdout)++ def test_restart_racing_with_signal_accepts_observed_failure(self):+ result = self.run_case(signal_status=1)+ self.assertEqual(result.returncode, 0, result.stderr)++ def test_failed_signal_without_evidence_fails(self):+ result = self.run_case(signal_adds_evidence=0, signal_status=1)+ self.assertNotEqual(result.returncode, 0)+ self.assertIn("could not advance watchdog failure", result.stderr)++ def test_successful_signal_without_evidence_fails(self):+ result = self.run_case(signal_adds_evidence=0)+ self.assertNotEqual(result.returncode, 0)+ self.assertIn("did not report new watchdog evidence", result.stderr)++ def test_old_failure_evidence_does_not_satisfy_new_check(self):+ result = self.run_case(already_seen=1, previous_count=1)+ self.assertEqual(result.returncode, 0, result.stderr)+ self.assertIn("signal", result.stdout)++ def test_different_failure_ordinal_does_not_satisfy_check(self):+ result = self.run_case(observed_ordinal=3)+ self.assertNotEqual(result.returncode, 0)+ self.assertIn("did not report new watchdog evidence", result.stderr)+++if __name__ == "__main__":+ unittest.main()diff --git a/autogpt_platform/single-container/tests/test_entrypoint.py b/autogpt_platform/single-container/tests/test_entrypoint.pyindex a125d3c4d5..b0d01211d1 100644--- a/autogpt_platform/single-container/tests/test_entrypoint.py+++ b/autogpt_platform/single-container/tests/test_entrypoint.py@@ -141,6 +141,13 @@ class AccountRegistrationTest(unittest.TestCase):class EnvironmentPolicyTest(unittest.TestCase):+ def test_image_preserves_background_embedding_backfill_at_startup(+ self,+ ) -> None:+ dockerfile = DOCKERFILE_PATH.read_text(encoding="utf-8")++ self.assertNotIn("SCHEDULER_STARTUP_EMBEDDING_BACKFILL=false", dockerfile)+def test_rejects_required_email_verification(self) -> None:result = self._configure(AUTH_REQUIRE_EMAIL_VERIFICATION="true")diff --git a/autogpt_platform/single-container/tests/test_supervisor_config.py b/autogpt_platform/single-container/tests/test_supervisor_config.pyindex 73008c674d..7a146012cc 100644--- a/autogpt_platform/single-container/tests/test_supervisor_config.py+++ b/autogpt_platform/single-container/tests/test_supervisor_config.py@@ -8,6 +8,16 @@ from pathlib import PathASSET_DIR = Path(__file__).resolve().parents[1]SUPERVISOR_PATH = ASSET_DIR / "supervisor" / "supervisord.conf"HEALTHCHECK_PATH = ASSET_DIR / "healthcheck.sh"+WATCHDOG_PATH = ASSET_DIR / "watchdog.sh"+SMOKE_PATH = (+ ASSET_DIR.parents[1] / ".github" / "scripts" / "platform-single-container-smoke.sh"+)+WORKFLOW_PATH = (+ ASSET_DIR.parents[1]+ / ".github"+ / "workflows"+ / "platform-single-container-docker.yml"+)# Unraid ships Docker's stock stop timeout and operators must not have to raise# it host-wide to run this appliance, so the whole supervised shutdown has to@@ -201,5 +211,89 @@ class HealthcheckSupervisorNamesTest(unittest.TestCase):self.assertEqual(checked, expected)+class WatchdogCadenceTest(unittest.TestCase):+ def setUp(self) -> None:+ self.watchdog = WATCHDOG_PATH.read_text(encoding="utf-8")+ self.smoke = SMOKE_PATH.read_text(encoding="utf-8")++ def _constant(self, name: str) -> int:+ match = re.search(rf"^readonly {name}=(\d+)$", self.watchdog, re.MULTILINE)+ self.assertIsNotNone(match)+ assert match is not None+ return int(match.group(1))++ def test_failure_limit_remains_three(self) -> None:+ self.assertEqual(self._constant("FAILURE_LIMIT"), 3)++ force_check = self.smoke.split("force_watchdog_check() {", maxsplit=1)[1].split(+ "\n}\n", maxsplit=1+ )[0]+ count_position = force_check.index(+ 'previous_count="$(count_container_log_evidence "${expected}")"'+ )+ signal_position = force_check.index(+ 'docker exec "${CONTAINER_NAME}" kill -USR1 "${watchdog_pid}"'+ )+ wait_position = force_check.index(+ 'wait_for_new_container_log_evidence "${expected}" "${previous_count}"'+ )+ self.assertLess(count_position, signal_position)+ self.assertLess(signal_position, wait_position)++ evidence_wait = self.smoke.split(+ "wait_for_new_container_log_evidence() {", maxsplit=1+ )[1].split("\n}\n", maxsplit=1)[0]+ self.assertIn("((current_count > previous_count))", evidence_wait)++ def test_steady_state_cadence_remains_thirty_seconds(self) -> None:+ self.assertEqual(self._constant("CHECK_INTERVAL"), 30)+ self.assertIn("while true; do\n wait_for_next_check", self.watchdog)+ self.assertIn('sleep "${CHECK_INTERVAL}" &', self.watchdog)+ self.assertIn("trap queue_forced_check USR1", self.watchdog)++ def test_initial_health_poll_is_faster_than_steady_state_poll(self) -> None:+ self.assertEqual(self._constant("INITIAL_CHECK_INTERVAL"), 1)++ initial_wait = self.watchdog.split("wait_for_initial_health()", maxsplit=1)[+ 1+ ].split("stop_appliance()", maxsplit=1)[0]+ self.assertIn('sleep "${INITIAL_CHECK_INTERVAL}"', initial_wait)+ self.assertNotIn('sleep "${CHECK_INTERVAL}"', initial_wait)+++class WorkflowFailurePropagationTest(unittest.TestCase):+ def test_scans_join_smoke_and_fail_from_recorded_outcomes(self) -> None:+ workflow = WORKFLOW_PATH.read_text(encoding="utf-8")+ validation_job = workflow.split(" build-and-scan:", maxsplit=1)[1].split(+ " publish-platform-digests:", maxsplit=1+ )[0]++ self.assertIn(+ " - name: Scan for fixable critical vulnerabilities\n"+ " id: vulnerability_scan\n"+ " continue-on-error: true",+ validation_job,+ )+ self.assertIn(+ " - name: Scan image filesystem for embedded secrets\n"+ " id: secret_scan\n"+ " continue-on-error: true",+ validation_job,+ )+ self.assertIn(+ " - name: Wait for complete-image smoke test\n"+ " wait: complete-image-smoke",+ validation_job,+ )+ self.assertIn(+ "VULNERABILITY_SCAN_OUTCOME: ${{ steps.vulnerability_scan.outcome }}",+ validation_job,+ )+ self.assertIn(+ "SECRET_SCAN_OUTCOME: ${{ steps.secret_scan.outcome }}", validation_job+ )+ self.assertIn("((scan_failed == 0))", validation_job)++if __name__ == "__main__":unittest.main()diff --git a/autogpt_platform/single-container/tests/test_watchdog.py b/autogpt_platform/single-container/tests/test_watchdog.pynew file mode 100644index 0000000000..81783e593b--- /dev/null+++ b/autogpt_platform/single-container/tests/test_watchdog.py@@ -0,0 +1,260 @@+from __future__ import annotations++import os+import queue+import signal+import subprocess+import tempfile+import threading+import time+import unittest+from pathlib import Path++ASSET_DIR = Path(__file__).resolve().parents[1]+WATCHDOG_PATH = ASSET_DIR / "watchdog.sh"++HARNESS = r"""+set -Eeuo pipefail+export AUTOGPT_RUNTIME_DIR="$1"+export AUTOGPT_READY_FILE="${AUTOGPT_RUNTIME_DIR}/ready"+export AUTOGPT_RUNTIME_ENV="${AUTOGPT_RUNTIME_DIR}/runtime.env"+export AUTOGPT_ASSET_DIR="$2"++source "$3"++HEALTHCHECK_COUNT=0+REAL_SLEEP="$(type -P sleep)"++wait_for_ready_file() {+ :+}++wait_for_initial_health() {+ printf 'harness-ready\n'+}++run_healthcheck() {+ local output="$1"+ ((HEALTHCHECK_COUNT += 1))+ printf 'healthcheck-%s\n' "${HEALTHCHECK_COUNT}"+ if [[ "${WATCHDOG_TEST_MODE}" == forced && "${HEALTHCHECK_COUNT}" -eq 1 ]]; then+ return 0+ fi+ printf 'expected test failure\n' >"${output}"+ return 1+}++stop_appliance() {+ printf 'harness-stopped\n'+ exit 0+}++if [[ "${WATCHDOG_TEST_MODE}" == forced ]]; then+ notify_check_timer_started() {+ local watchdog_pid="$$"+ printf 'timer-ready\n'+ (+ "${REAL_SLEEP}" 0.05+ kill -USR1 "${watchdog_pid}"+ ) &+ }+ sleep() {+ exec "${REAL_SLEEP}" "$@"+ }+elif [[ "${WATCHDOG_TEST_MODE}" == periodic ]]; then+ sleep() {+ :+ }+fi++main+"""+++ os.name == "posix" and hasattr(signal, "SIGUSR1"),+ "watchdog signals require a POSIX host",+)+class WatchdogSchedulingTest(unittest.TestCase):+ def test_sigusr1_queued_before_timer_runs_without_delay(self) -> None:+ harness = r"""+set -Eeuo pipefail+export AUTOGPT_RUNTIME_DIR="$1"+export AUTOGPT_ASSET_DIR="$2"+source "$3"+trap queue_forced_check USR1+kill -USR1 "$$"+wait_for_next_check+printf 'trigger=%s\n' "${CHECK_TRIGGER}"+"""+ with tempfile.TemporaryDirectory() as runtime_dir:+ result = subprocess.run(+ [+ "bash",+ "-c",+ harness,+ "bash",+ runtime_dir,+ str(ASSET_DIR),+ str(WATCHDOG_PATH),+ ],+ capture_output=True,+ text=True,+ timeout=5,+ check=False,+ )++ self.assertEqual(result.returncode, 0, result.stdout + result.stderr)+ self.assertEqual(result.stdout, "trigger=forced\n")++ def test_periodic_timer_failure_is_not_treated_as_forced_check(self) -> None:+ harness = r"""+set -Eeuo pipefail+export AUTOGPT_RUNTIME_DIR="$1"+export AUTOGPT_ASSET_DIR="$2"+source "$3"+sleep() {+ return 7+}+wait_for_next_check+"""+ with tempfile.TemporaryDirectory() as runtime_dir:+ result = subprocess.run(+ [+ "bash",+ "-c",+ harness,+ "bash",+ runtime_dir,+ str(ASSET_DIR),+ str(WATCHDOG_PATH),+ ],+ capture_output=True,+ text=True,+ timeout=5,+ check=False,+ )++ self.assertEqual(result.returncode, 1, result.stdout + result.stderr)+ self.assertIn("watchdog check timer failed with status 7", result.stderr)++ def test_periodic_timer_runs_healthcheck_until_failure_limit(self) -> None:+ with tempfile.TemporaryDirectory() as runtime_dir:+ result = subprocess.run(+ [+ "bash",+ "-c",+ HARNESS,+ "bash",+ runtime_dir,+ str(ASSET_DIR),+ str(WATCHDOG_PATH),+ ],+ capture_output=True,+ text=True,+ timeout=10,+ check=False,+ env={**os.environ, "WATCHDOG_TEST_MODE": "periodic"},+ )++ self.assertEqual(result.returncode, 0, result.stdout + result.stderr)+ self.assertEqual(result.stdout.count("healthcheck-"), 3)+ first = result.stderr.index("health failure 1/3 trigger=scheduled")+ second = result.stderr.index("health failure 2/3 trigger=scheduled")+ third = result.stderr.index("health failure 3/3 trigger=scheduled")+ self.assertLess(first, second)+ self.assertLess(second, third)+ self.assertIn("harness-stopped", result.stdout)++ def test_sigusr1_forces_ordered_checks_through_same_healthcheck(self) -> None:+ with tempfile.TemporaryDirectory() as runtime_dir:+ process = subprocess.Popen(+ [+ "bash",+ "-c",+ HARNESS,+ "bash",+ runtime_dir,+ str(ASSET_DIR),+ str(WATCHDOG_PATH),+ ],+ stdout=subprocess.PIPE,+ stderr=subprocess.STDOUT,+ text=True,+ bufsize=1,+ env={**os.environ, "WATCHDOG_TEST_MODE": "forced"},+ )+ output: list[str] = []+ lines: queue.Queue[str] = queue.Queue()+ reader = threading.Thread(+ target=self._collect_lines,+ args=(process, output, lines),+ daemon=True,+ )+ reader.start()+ try:+ self._wait_for_line(process, output, lines, "harness-ready")+ self._wait_for_line(process, output, lines, "timer-ready")+ self._wait_for_line(+ process,+ output,+ lines,+ "health check passed trigger=forced",+ )+ for failure in range(1, 4):+ self._wait_for_line(process, output, lines, "timer-ready")+ self._wait_for_line(+ process,+ output,+ lines,+ f"health failure {failure}/3 trigger=forced",+ )+ returncode = process.wait(timeout=5)+ finally:+ if process.poll() is None:+ process.terminate()+ process.wait(timeout=5)+ reader.join(timeout=5)+ if process.stdout is not None:+ process.stdout.close()++ rendered = "".join(output)+ self.assertFalse(reader.is_alive(), rendered)+ self.assertEqual(returncode, 0, rendered)+ self.assertEqual(rendered.count("healthcheck-"), 4)+ self.assertIn("harness-stopped", rendered)++ def _wait_for_line(+ self,+ process: subprocess.Popen[str],+ output: list[str],+ lines: queue.Queue[str],+ expected: str,+ ) -> None:+ deadline = time.monotonic() + 5+ while time.monotonic() < deadline:+ try:+ line = lines.get(timeout=deadline - time.monotonic())+ except queue.Empty:+ break+ if expected in line:+ return+ self.fail(+ f"watchdog did not emit {expected!r}; returncode={process.poll()}; "+ f"output={''.join(output)!r}"+ )++ def _collect_lines(+ self,+ process: subprocess.Popen[str],+ output: list[str],+ lines: queue.Queue[str],+ ) -> None:+ assert process.stdout is not None+ for line in process.stdout:+ output.append(line)+ lines.put(line)+++if __name__ == "__main__":+ unittest.main()diff --git a/autogpt_platform/single-container/watchdog.sh b/autogpt_platform/single-container/watchdog.shindex 1f21afb57a..a4049bb803 100644--- a/autogpt_platform/single-container/watchdog.sh+++ b/autogpt_platform/single-container/watchdog.sh@@ -8,29 +8,38 @@ source "${AUTOGPT_ASSET_DIR:-/opt/autogpt/single-container}/common.sh"readonly SUPERVISOR_PID_FILE="${AUTOGPT_RUNTIME_DIR}/supervisord.pid"readonly WATCHDOG_ARMED_FILE="${AUTOGPT_RUNTIME_DIR}/watchdog-armed"readonly FAILURE_LIMIT=3+readonly INITIAL_CHECK_INTERVAL=1readonly CHECK_INTERVAL=30readonly CHECK_TIMEOUT=60readonly INITIAL_HEALTH_TIMEOUT=600+CHECK_TIMER_PID=+FORCED_CHECK_PENDING=false+CHECK_TRIGGER=scheduled+WATCHDOG_OUTPUT=+main() {local failures=0- local outputrm -f "${WATCHDOG_ARMED_FILE}"- output="$(mktemp "${AUTOGPT_RUNTIME_DIR}/watchdog-health.XXXXXX")"- trap 'rm -f "${output}" "${WATCHDOG_ARMED_FILE}"' EXIT+ WATCHDOG_OUTPUT="$(mktemp "${AUTOGPT_RUNTIME_DIR}/watchdog-health.XXXXXX")"+ trap 'rm -f "${WATCHDOG_OUTPUT}" "${WATCHDOG_ARMED_FILE}"' EXIT+ trap queue_forced_check USR1wait_for_ready_file- wait_for_initial_health "${output}"+ wait_for_initial_health "${WATCHDOG_OUTPUT}"while true; do- sleep "${CHECK_INTERVAL}"- if run_healthcheck "${output}"; then+ wait_for_next_check+ if run_healthcheck "${WATCHDOG_OUTPUT}"; thenfailures=0+ if [[ "${CHECK_TRIGGER}" == forced ]]; then+ log "watchdog health check passed trigger=forced"+ ficontinuefi((failures += 1))- log "watchdog health failure ${failures}/${FAILURE_LIMIT}" >&2- sed -n '1,120p' "${output}" >&2+ log "watchdog health failure ${failures}/${FAILURE_LIMIT} trigger=${CHECK_TRIGGER}" >&2+ sed -n '1,120p' "${WATCHDOG_OUTPUT}" >&2if ((failures >= FAILURE_LIMIT)); thenstop_appliance \"watchdog stopping the unhealthy appliance; a Docker restart policy is required to restart it automatically"@@ -38,6 +47,66 @@ main() {done}+queue_forced_check() {+ # USR1 advances one cadence slot; the normal healthcheck and failure limit+ # below remain the only path that can stop the appliance.+ FORCED_CHECK_PENDING=true+ if [[ -n "${CHECK_TIMER_PID}" ]]; then+ if ! kill -TERM "${CHECK_TIMER_PID}" 2>/dev/null; then+ :+ fi+ fi+}++notify_check_timer_started() {+ :+}++wait_for_next_check() {+ local timer_pid+ local timer_status+ CHECK_TRIGGER=scheduled++ if [[ "${FORCED_CHECK_PENDING}" == true ]]; then+ FORCED_CHECK_PENDING=false+ CHECK_TRIGGER=forced+ return 0+ fi++ sleep "${CHECK_INTERVAL}" &+ timer_pid=$!+ CHECK_TIMER_PID="${timer_pid}"+ notify_check_timer_started+ if [[ "${FORCED_CHECK_PENDING}" == true ]]; then+ if ! kill -TERM "${timer_pid}" 2>/dev/null; then+ :+ fi+ fi+ if wait "${timer_pid}"; then+ timer_status=0+ else+ timer_status=$?+ fi+ CHECK_TIMER_PID=++ if [[ "${FORCED_CHECK_PENDING}" == true ]]; then+ FORCED_CHECK_PENDING=false+ CHECK_TRIGGER=forced+ if kill -0 "${timer_pid}" 2>/dev/null; then+ if ! kill -TERM "${timer_pid}" 2>/dev/null; then+ :+ fi+ fi+ # Reap the timer if the signal interrupted `wait` before TERM completed.+ if ! wait "${timer_pid}" 2>/dev/null; then+ :+ fi+ return 0+ fi++ ((timer_status == 0)) || fatal "watchdog check timer failed with status ${timer_status}"+}+run_healthcheck() {local output="$1"timeout --signal=TERM --kill-after=5s "${CHECK_TIMEOUT}" \@@ -55,7 +124,7 @@ wait_for_initial_health() {filog "watchdog waiting for initial healthy state" >&2sed -n '1,120p' "${output}" >&2- sleep "${CHECK_INTERVAL}"+ sleep "${INITIAL_CHECK_INTERVAL}"donestop_appliance \"watchdog did not observe a healthy appliance within ${INITIAL_HEALTH_TIMEOUT}s"@@ -69,4 +138,6 @@ stop_appliance() {exit 0}-main "$@"+if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then+ main "$@"+fi