Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

ci(platform): speed up and harden single-container validation (#14282)

Nicholas Tindle43a7bc38d210
diff --git a/.github/scripts/platform-single-container-publish.sh b/.github/scripts/platform-single-container-publish.sh
index 2440dddc97..f4cfc4ce12 100755
--- a/.github/scripts/platform-single-container-publish.sh
+++ b/.github/scripts/platform-single-container-publish.sh
@@ -173,10 +173,6 @@ resolve_publication() {
publish_latest=false
publication_name="Single-container SHA image published"
- if [[ "$GITHUB_EVENT_NAME" == "workflow_dispatch" && "$GITHUB_REF" == "refs/heads/dev" ]]; then
- return
- fi
-
if [[ "$GITHUB_EVENT_NAME" == "release" && "$GITHUB_REF" == "refs/tags/${RELEASE_TAG}" ]]; then
if [[ ! "$RELEASE_TAG" =~ ^autogpt-platform-beta-v([0-9]+\.[0-9]+\.[0-9]+)$ ]]; then
echo "refusing unsupported release tag: $RELEASE_TAG" >&2
@@ -194,18 +190,6 @@ resolve_publication() {
}
publication_allowed() {
- if [[ "$GITHUB_EVENT_NAME" == "workflow_dispatch" ]]; then
- if [[ "$PUBLISH_REQUESTED" != "true" ]]; then
- printf 'false\n'
- return
- fi
- if ! resolve_publication; then
- return 1
- fi
- printf 'true\n'
- return
- fi
-
if [[ "$GITHUB_EVENT_NAME" == "release" ]]; then
if [[ "$RELEASE_PRERELEASE" != "false" ]]; then
printf 'false\n'
@@ -356,7 +340,7 @@ ensure_immutable_manifest() {
ensure_release_tag() {
local manifest_digest="$1"
- local state raw_manifest
+ local state raw_manifest published_release_version
release_manifest_digest="$manifest_digest"
[[ -n "$release_ref" ]] || return 0
@@ -370,7 +354,11 @@ ensure_release_tag() {
release_manifest_digest="$(inspect_manifest "$release_ref")"
verify_manifest "$release_ref" "${expected_rows[@]}"
raw_manifest="$(docker buildx imagetools inspect --raw "$release_ref")"
- if [[ "$(release_version_from_manifest <<<"$raw_manifest")" != "$release_version" ]]; then
+ if ! published_release_version="$(release_version_from_manifest <<<"$raw_manifest")"; then
+ echo "could not read the immutable release-version annotation from $release_ref" >&2
+ return 1
+ fi
+ if [[ "$published_release_version" != "$release_version" ]]; then
echo "$release_ref is missing its immutable release-version annotation" >&2
return 1
fi
@@ -418,10 +406,11 @@ latest_update_allowed() {
return
fi
- set +e
- semver_is_newer "$current_version" "$target_version"
- compare_status=$?
- set -e
+ if semver_is_newer "$current_version" "$target_version"; then
+ compare_status=0
+ else
+ compare_status=$?
+ fi
if ((compare_status == 0)); then
echo "refusing to move latest backward from $current_version to $target_version" >&2
return 1
@@ -458,13 +447,22 @@ publish_latest_tag() {
fi
}
+list_digest_files() {
+ find "$DIGEST_DIR" -maxdepth 1 -type f -print | sort
+}
+
load_verified_digests() {
local digest_file descriptor expected_arch digest_hex image_ref actual_platform raw_manifest expected_row
local runnable_digest image_json source_revision
+ local digest_listing
local -a digest_files=()
declare -A seen_platforms=()
- mapfile -t digest_files < <(find "$DIGEST_DIR" -maxdepth 1 -type f -print | sort)
+ if ! digest_listing="$(list_digest_files)"; then
+ echo "could not enumerate verified platform digests" >&2
+ return 1
+ fi
+ mapfile -t digest_files <<<"$digest_listing"
if ((${#digest_files[@]} != 2)); then
echo "expected exactly two verified platform digests" >&2
return 1
@@ -695,12 +693,12 @@ self_test() {
assert_equal 3 "$actual" "confirmed absent retry count"
rm -f "$tag_retry_state"
- DEPLOY_IMAGE=docker.io/significantgravitas/autogpt \
+ if DEPLOY_IMAGE=docker.io/significantgravitas/autogpt \
GITHUB_EVENT_NAME=workflow_dispatch GITHUB_REF=refs/heads/dev GITHUB_SHA=abc123 \
- RELEASE_TAG='' resolve_publication
- assert_equal docker.io/significantgravitas/autogpt:sha-abc123 "$immutable_ref" "dev immutable tag"
- assert_equal '' "$release_ref" "dev release tag"
- assert_equal false "$publish_latest" "dev latest policy"
+ RELEASE_TAG='' resolve_publication 2>/dev/null; then
+ echo "workflow dispatch publication was accepted" >&2
+ return 1
+ fi
release_manifest_digest=""
ensure_release_tag sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
@@ -726,26 +724,20 @@ self_test() {
actual="$(
DEPLOY_IMAGE=docker.io/significantgravitas/autogpt \
GITHUB_EVENT_NAME=workflow_dispatch GITHUB_REF=refs/heads/dev GITHUB_SHA=abc123 \
- PUBLISH_REQUESTED=true RELEASE_PRERELEASE='' RELEASE_TAG='' publication_allowed
+ RELEASE_PRERELEASE='' RELEASE_TAG='' publication_allowed
)"
- assert_equal true "$actual" "dev publication authorization"
- actual="$(
- DEPLOY_IMAGE=docker.io/significantgravitas/autogpt \
- GITHUB_EVENT_NAME=workflow_dispatch GITHUB_REF=refs/heads/dev GITHUB_SHA=abc123 \
- PUBLISH_REQUESTED=false RELEASE_PRERELEASE='' RELEASE_TAG='' publication_allowed
- )"
- assert_equal false "$actual" "validation-only dispatch authorization"
+ assert_equal false "$actual" "dispatch publication authorization"
actual="$(
DEPLOY_IMAGE=docker.io/significantgravitas/autogpt \
GITHUB_EVENT_NAME=release GITHUB_REF=refs/tags/autogpt-platform-beta-v0.7.1 \
- GITHUB_SHA=abc123 PUBLISH_REQUESTED='' RELEASE_PRERELEASE=false \
+ GITHUB_SHA=abc123 RELEASE_PRERELEASE=false \
RELEASE_TAG=autogpt-platform-beta-v0.7.1 publication_allowed
)"
assert_equal true "$actual" "release publication authorization"
actual="$(
DEPLOY_IMAGE=docker.io/significantgravitas/autogpt \
GITHUB_EVENT_NAME=release GITHUB_REF=refs/tags/autogpt-platform-beta-v0.7.1 \
- GITHUB_SHA=abc123 PUBLISH_REQUESTED='' RELEASE_PRERELEASE=true \
+ GITHUB_SHA=abc123 RELEASE_PRERELEASE=true \
RELEASE_TAG=autogpt-platform-beta-v0.7.1 publication_allowed
)"
assert_equal false "$actual" "prerelease publication authorization"
@@ -754,9 +746,9 @@ self_test() {
GITHUB_OUTPUT="$authorization_output" \
DEPLOY_IMAGE=docker.io/significantgravitas/autogpt \
GITHUB_EVENT_NAME=workflow_dispatch GITHUB_REF=refs/heads/dev GITHUB_SHA=abc123 \
- PUBLISH_REQUESTED=true RELEASE_PRERELEASE='' RELEASE_TAG='' authorize
+ RELEASE_PRERELEASE='' RELEASE_TAG='' authorize
actual="$(<"$authorization_output")"
- assert_equal allowed=true "$actual" "authorization output"
+ assert_equal allowed=false "$actual" "dispatch authorization output"
: >"$authorization_output"
if DEPLOY_IMAGE=docker.io/significantgravitas/autogpt \
@@ -767,7 +759,7 @@ self_test() {
fi
if DEPLOY_IMAGE=docker.io/significantgravitas/autogpt \
GITHUB_EVENT_NAME=release GITHUB_REF=refs/tags/autogpt-platform-beta-v0.7 \
- GITHUB_SHA=abc123 PUBLISH_REQUESTED='' RELEASE_PRERELEASE=false \
+ GITHUB_SHA=abc123 RELEASE_PRERELEASE=false \
RELEASE_TAG=autogpt-platform-beta-v0.7 publication_allowed 2>/dev/null; then
echo "malformed release tag was authorized" >&2
return 1
@@ -775,7 +767,7 @@ self_test() {
if GITHUB_OUTPUT="$authorization_output" \
DEPLOY_IMAGE=docker.io/significantgravitas/autogpt \
GITHUB_EVENT_NAME=release GITHUB_REF=refs/tags/autogpt-platform-beta-v0.7 \
- GITHUB_SHA=abc123 PUBLISH_REQUESTED='' RELEASE_PRERELEASE=false \
+ GITHUB_SHA=abc123 RELEASE_PRERELEASE=false \
RELEASE_TAG=autogpt-platform-beta-v0.7 authorize 2>/dev/null; then
echo "malformed release tag produced an authorization output" >&2
return 1
@@ -875,6 +867,32 @@ JSON
echo "invalid release-version annotation was accepted" >&2
return 1
fi
+ if (
+ release_ref=docker.io/significantgravitas/autogpt:v0.7.1
+ release_version=v0.7.1
+ expected_rows=()
+ tag_state() {
+ printf 'present\n'
+ }
+ inspect_manifest() {
+ printf 'sha256:%064d\n' 0
+ }
+ verify_manifest() {
+ :
+ }
+ docker() {
+ printf '{}\n'
+ }
+ release_version_from_manifest() {
+ printf 'v0.7.1\n'
+ return 1
+ }
+ ensure_release_tag sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa \
+ >/dev/null 2>&1
+ ); then
+ echo "failed release-version extraction was accepted" >&2
+ return 1
+ fi
semver_is_newer v0.7.2 v0.7.1
semver_is_newer v1.0.0 v0.99.99
@@ -899,6 +917,19 @@ JSON
return 1
fi
+ if (
+ list_digest_files() {
+ printf '%s\n' \
+ /tmp/amd64-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa \
+ /tmp/arm64-bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb
+ return 1
+ }
+ DIGEST_DIR=/tmp load_verified_digests >/dev/null 2>&1
+ ); then
+ echo "failed digest enumeration was accepted" >&2
+ return 1
+ fi
+
echo "single-container publication helper tests passed"
}
diff --git a/.github/scripts/platform-single-container-smoke.sh b/.github/scripts/platform-single-container-smoke.sh
index 9bcdafcf3d..08d437d4e6 100644
--- a/.github/scripts/platform-single-container-smoke.sh
+++ b/.github/scripts/platform-single-container-smoke.sh
@@ -22,10 +22,13 @@ readonly STOCK_DOCKER_STOP_TIMEOUT=10
readonly SHUTDOWN_MARGIN_SECONDS=1
readonly MAX_CLEAN_STOP_SECONDS=$((STOCK_DOCKER_STOP_TIMEOUT - SHUTDOWN_MARGIN_SECONDS))
readonly TIMEOUT_SECONDS="${SMOKE_TIMEOUT_SECONDS:-2700}"
+readonly SCAN_COMPLETION_FILE="${SMOKE_SCAN_COMPLETION_FILE:-}"
readonly SAFE_PLATFORM="${SMOKE_PLATFORM//\//-}"
readonly RUN_TOKEN="${GITHUB_RUN_ID:-local}-${GITHUB_RUN_ATTEMPT:-0}-${SAFE_PLATFORM}-${RANDOM}"
readonly RUN_CONTAINER_NAME="autogpt-single-smoke-${RUN_TOKEN}"
readonly NEGATIVE_CONTAINER_NAME="${RUN_CONTAINER_NAME}-negative"
+readonly SMOKE_STARTED_SECONDS=${SECONDS}
+SMOKE_LAST_TIMING_SECONDS=${SMOKE_STARTED_SECONDS}
HEADERS_FILE="$(mktemp)"
readonly HEADERS_FILE
AUTH_COOKIE_FILE="$(mktemp)"
@@ -34,40 +37,175 @@ readonly AUTH_COOKIE_FILE
CONTAINER_NAME="${RUN_CONTAINER_NAME}"
DATA_VOLUME=
+record_timing() {
+ local phase="$1"
+ local now=${SECONDS}
+ printf 'single-container smoke timing: phase=%s duration=%ss elapsed=%ss\n' \
+ "${phase}" \
+ "$((now - SMOKE_LAST_TIMING_SECONDS))" \
+ "$((now - SMOKE_STARTED_SECONDS))"
+ SMOKE_LAST_TIMING_SECONDS=${now}
+}
+
+record_boot_milestones() {
+ local phase="$1"
+ local started_at
+ local health_history
+ local container_logs
+ local line
+ local milestone
+ local timestamp
+
+ if ! started_at="$(
+ docker inspect --format '{{.State.StartedAt}}' "${CONTAINER_NAME}"
+ )"; then
+ echo "could not record ${phase} container start time" >&2
+ return 0
+ fi
+ printf 'single-container boot milestone: phase=%s container-started-at=%s\n' \
+ "${phase}" "${started_at}"
+
+ if health_history="$(
+ docker inspect --format '{{json .State.Health.Log}}' "${CONTAINER_NAME}"
+ )"; then
+ if ! jq -r --arg phase "${phase}" \
+ '.[] | "single-container boot milestone: phase=\($phase) docker-health start=\(.Start) end=\(.End) exit=\(.ExitCode)"' \
+ <<<"${health_history}"; then
+ echo "could not format ${phase} Docker health milestones" >&2
+ fi
+ else
+ echo "could not record ${phase} Docker health milestones" >&2
+ fi
+
+ if ! container_logs="$(
+ docker logs --timestamps --since "${started_at}" "${CONTAINER_NAME}" 2>&1
+ )"; then
+ echo "could not record ${phase} appliance milestones" >&2
+ return 0
+ fi
+ # Raw service logs and Docker health output can contain request or provider
+ # data and stay private. A matching line selects a fixed event name; none of
+ # the original log content is emitted.
+ while IFS= read -r line; do
+ milestone=
+ case "${line}" in
+ *"[single-container] starting process supervisor"*) milestone=supervisor-start ;;
+ *"[single-container] initializing PostgreSQL data directory"*) milestone=postgres-init ;;
+ *"[single-container] PostgreSQL is ready"*) milestone=postgres-ready ;;
+ *"[single-container] Valkey node "*" is ready"*) milestone=valkey-node-ready ;;
+ *"[single-container] RabbitMQ is ready"*) milestone=rabbitmq-ready ;;
+ *"[single-container] FalkorDB is ready"*) milestone=falkordb-ready ;;
+ *"[single-container] forming three-node Valkey cluster"*) milestone=valkey-cluster-forming ;;
+ *"[single-container] Valkey cluster is ready"*) milestone=valkey-cluster-ready ;;
+ *"[single-container] Valkey cluster is already healthy"*) milestone=valkey-cluster-already-healthy ;;
+ *"[single-container] Valkey cluster recovered"*) milestone=valkey-cluster-recovered ;;
+ *"[single-container] RabbitMQ application user is present"*) milestone=rabbitmq-user-ready ;;
+ *"[single-container] ensuring platform database schemas exist"*) milestone=database-schemas-start ;;
+ *"[single-container] applying Prisma migrations"*) milestone=prisma-migrations-start ;;
+ *"[single-container] configuring least-privilege frontend database role"*) milestone=frontend-role-start ;;
+ *"[single-container] bootstrap complete"*) milestone=bootstrap-complete ;;
+ *"[single-container] starting database-manager"*) milestone=database-manager-start ;;
+ *"[single-container] starting scheduler"*) milestone=scheduler-start ;;
+ *"[single-container] starting batch-executor"*) milestone=batch-executor-start ;;
+ *"[single-container] starting notification"*) milestone=notification-start ;;
+ *"[single-container] starting executor"*) milestone=executor-start ;;
+ *"[single-container] starting copilot-executor"*) milestone=copilot-executor-start ;;
+ *"[single-container] starting copilot-bot"*) milestone=copilot-bot-start ;;
+ *"[single-container] starting platform-linking-manager"*) milestone=platform-linking-manager-start ;;
+ *"[single-container] starting websocket"*) milestone=websocket-start ;;
+ *"[single-container] starting rest"*) milestone=rest-start ;;
+ *"[single-container] starting next"*) milestone=next-start ;;
+ *"[single-container] starting nginx"*) milestone=nginx-start ;;
+ *"[single-container] watchdog armed after initial healthy state"*) milestone=watchdog-armed ;;
+ esac
+ [[ -n "${milestone}" ]] || continue
+ timestamp="${line%% *}"
+ if [[ ! "${timestamp}" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(\.[0-9]+)?Z$ ]]; then
+ timestamp=unknown
+ fi
+ printf 'single-container boot milestone: phase=%s at=%s event=%s\n' \
+ "${phase}" "${timestamp}" "${milestone}"
+ done <<<"${container_logs}"
+}
+
diagnostics() {
if docker container inspect "${CONTAINER_NAME}" >/dev/null 2>&1; then
- docker inspect --format '{{json .State}}' "${CONTAINER_NAME}" || true
- docker logs --timestamps --tail 2000 "${CONTAINER_NAME}" || true
+ if ! docker inspect --format '{{json .State}}' "${CONTAINER_NAME}"; then
+ echo "could not inspect ${CONTAINER_NAME}" >&2
+ fi
+ if ! docker logs --timestamps --tail 2000 "${CONTAINER_NAME}"; then
+ echo "could not read logs for ${CONTAINER_NAME}" >&2
+ fi
fi
}
cleanup() {
local result=$?
+ local cleanup_failed=0
local container
+ local container_id
+ local volume_name
trap - EXIT INT TERM
if ((result != 0)); then
diagnostics
fi
for container in "${CONTAINER_NAME}" "${NEGATIVE_CONTAINER_NAME}"; do
- if docker container inspect "${container}" >/dev/null 2>&1; then
- docker stop --timeout "${STOCK_DOCKER_STOP_TIMEOUT}" "${container}" >/dev/null 2>&1 || true
- docker rm --force --volumes "${container}" >/dev/null 2>&1 || true
+ if ! container_id="$(
+ docker container ls --all --quiet --filter "name=^/${container}$"
+ )"; then
+ echo "could not determine whether ${container} needs cleanup" >&2
+ cleanup_failed=1
+ continue
+ fi
+ if [[ -n "${container_id}" ]]; then
+ if ! docker stop --timeout "${STOCK_DOCKER_STOP_TIMEOUT}" "${container}" >/dev/null 2>&1; then
+ echo "could not stop ${container} during cleanup" >&2
+ cleanup_failed=1
+ fi
+ if ! docker rm --force --volumes "${container}" >/dev/null 2>&1; then
+ echo "could not remove ${container} during cleanup" >&2
+ cleanup_failed=1
+ fi
fi
done
- if [[ -n "${DATA_VOLUME}" ]] && docker volume inspect "${DATA_VOLUME}" >/dev/null 2>&1; then
- docker volume rm "${DATA_VOLUME}" >/dev/null 2>&1 || true
+ if [[ -n "${DATA_VOLUME}" ]]; then
+ if ! volume_name="$(
+ docker volume ls --quiet --filter "name=^${DATA_VOLUME}$"
+ )"; then
+ echo "could not determine whether ${DATA_VOLUME} needs cleanup" >&2
+ cleanup_failed=1
+ elif [[ -n "${volume_name}" ]]; then
+ if ! docker volume rm "${DATA_VOLUME}" >/dev/null 2>&1; then
+ echo "could not remove ${DATA_VOLUME} during cleanup" >&2
+ cleanup_failed=1
+ fi
+ fi
+ fi
+ if ! rm -f "${HEADERS_FILE}" "${AUTH_COOKIE_FILE}"; then
+ echo "could not remove single-container smoke temporary files" >&2
+ cleanup_failed=1
+ fi
+ if ((result == 0 && cleanup_failed != 0)); then
+ result=1
fi
- rm -f "${HEADERS_FILE}" "${AUTH_COOKIE_FILE}"
exit "${result}"
}
assert_clean_stop() {
local reason="$1"
- local started elapsed exit_code
- # Integer SECONDS truncates at both ends, so a real 8.9s stop reads as 8.
+ local started finished_at finished_epoch elapsed exit_code
started="${EPOCHREALTIME}"
docker stop --timeout "${STOCK_DOCKER_STOP_TIMEOUT}" "${CONTAINER_NAME}" >/dev/null
- elapsed="$(awk -v a="${started}" -v b="${EPOCHREALTIME}" 'BEGIN { printf "%.2f", b - a }')"
+ finished_at="$(docker inspect --format '{{.State.FinishedAt}}' "${CONTAINER_NAME}")"
+ if ! finished_epoch="$(date --date="${finished_at}" +%s.%N)" || [[ -z "${finished_epoch}" ]]; then
+ echo "invalid container finish timestamp ${reason}: ${finished_at}" >&2
+ return 1
+ fi
+ awk -v a="${started}" -v b="${finished_epoch}" 'BEGIN { exit !(b >= a) }' || {
+ echo "container finish timestamp precedes the stop request ${reason}" >&2
+ return 1
+ }
+ elapsed="$(awk -v a="${started}" -v b="${finished_epoch}" 'BEGIN { printf "%.2f", b - a }')"
exit_code="$(docker inspect --format '{{.State.ExitCode}}' "${CONTAINER_NAME}")"
[[ "${exit_code}" == 0 ]] || {
echo "container did not exit cleanly ${reason}: exit ${exit_code} after" \
@@ -104,12 +242,17 @@ wait_for_healthy() {
local state
local health
local exit_code
+ local state_record
while ((SECONDS < deadline)); do
- read -r state health exit_code < <(
+ if ! state_record="$(
docker inspect \
--format '{{.State.Status}} {{if .State.Health}}{{.State.Health.Status}}{{else}}missing{{end}} {{.State.ExitCode}}' \
"${CONTAINER_NAME}"
- )
+ )"; then
+ echo "could not inspect container state while waiting for health" >&2
+ return 1
+ fi
+ read -r state health exit_code <<<"${state_record}"
case "${state}:${health}" in
running:healthy)
if appliance_is_healthy_and_armed; then
@@ -121,7 +264,7 @@ wait_for_healthy() {
return 1
;;
esac
- sleep 10
+ sleep 1
done
echo "container did not become healthy within ${TIMEOUT_SECONDS}s" >&2
return 1
@@ -133,12 +276,17 @@ wait_for_automatic_restart() {
local state
local health
local restart_count
+ local state_record
while ((SECONDS < deadline)); do
- read -r state health restart_count < <(
+ if ! state_record="$(
docker inspect \
--format '{{.State.Status}} {{if .State.Health}}{{.State.Health.Status}}{{else}}missing{{end}} {{.RestartCount}}' \
"${CONTAINER_NAME}"
- )
+ )"; then
+ echo "could not inspect container state while waiting for restart" >&2
+ return 1
+ fi
+ read -r state health restart_count <<<"${state_record}"
if [[ "${state}:${health}" == running:healthy ]] &&
((restart_count >= minimum_restart_count)) &&
appliance_is_healthy_and_armed; then
@@ -150,12 +298,107 @@ wait_for_automatic_restart() {
return 1
;;
esac
- sleep 10
+ sleep 1
done
echo "container did not automatically restart and become healthy" >&2
return 1
}
+wait_for_concurrent_scans() {
+ if [[ -z "${SCAN_COMPLETION_FILE}" ]]; then
+ return 0
+ fi
+ local deadline=$((SECONDS + TIMEOUT_SECONDS))
+ while ((SECONDS < deadline)); do
+ if [[ -f "${SCAN_COMPLETION_FILE}" ]]; then
+ return 0
+ fi
+ sleep 1
+ done
+ echo "concurrent image scans did not complete within ${TIMEOUT_SECONDS}s" >&2
+ return 1
+}
+
+count_container_log_evidence() {
+ local expected="$1"
+ local container_logs
+ local count=0
+ local line
+ if ! container_logs="$(docker logs "${CONTAINER_NAME}" 2>&1)"; then
+ echo "could not read container logs while counting watchdog evidence" >&2
+ return 1
+ fi
+ while IFS= read -r line; do
+ if [[ "${line}" == *"${expected}"* ]]; then
+ count=$((count + 1))
+ fi
+ done <<<"${container_logs}"
+ printf '%s\n' "${count}"
+}
+
+wait_for_new_container_log_evidence() {
+ local expected="$1"
+ local previous_count="$2"
+ local deadline=$((SECONDS + TIMEOUT_SECONDS))
+ local current_count
+ if [[ ! "${previous_count}" =~ ^[0-9]+$ ]]; then
+ echo "previous watchdog evidence count is invalid" >&2
+ return 1
+ fi
+ while ((SECONDS < deadline)); do
+ if ! current_count="$(count_container_log_evidence "${expected}")"; then
+ return 1
+ fi
+ if [[ ! "${current_count}" =~ ^[0-9]+$ ]]; then
+ echo "watchdog evidence count is invalid" >&2
+ return 1
+ fi
+ if ((current_count > previous_count)); then
+ return 0
+ fi
+ sleep 1
+ done
+ echo "container log did not report new watchdog evidence: ${expected}" >&2
+ return 1
+}
+
+force_watchdog_check() {
+ local watchdog_pid="$1"
+ local expected="$2"
+ local previous_count
+ if ! previous_count="$(count_container_log_evidence "${expected}")"; then
+ return 1
+ fi
+ if [[ ! "${previous_count}" =~ ^[0-9]+$ ]]; then
+ echo "watchdog evidence count is invalid before forced check" >&2
+ return 1
+ fi
+ docker exec "${CONTAINER_NAME}" kill -USR1 "${watchdog_pid}"
+ wait_for_new_container_log_evidence "${expected}" "${previous_count}"
+}
+
+advance_watchdog_failure() {
+ local watchdog_pid="$1"
+ local ordinal="$2"
+ local previous_count="$3"
+ local expected="watchdog health failure ${ordinal}/3 trigger="
+ local current_count
+ current_count="$(count_container_log_evidence "${expected}")" || return 1
+ [[ "${current_count}" =~ ^[0-9]+$ && "${previous_count}" =~ ^[0-9]+$ ]] || return 1
+ if ((current_count > previous_count)); then
+ return 0
+ fi
+ if ! docker exec "${CONTAINER_NAME}" kill -USR1 "${watchdog_pid}"; then
+ # A scheduled third failure can stop the appliance before the signal.
+ current_count="$(count_container_log_evidence "${expected}")" || return 1
+ if [[ ! "${current_count}" =~ ^[0-9]+$ ]] || ((current_count <= previous_count)); then
+ echo "could not advance watchdog failure ${ordinal}/3" >&2
+ return 1
+ fi
+ fi
+ wait_for_new_container_log_evidence "${expected}" "${previous_count}"
+}
+
runtime_config_hash() {
docker exec "${CONTAINER_NAME}" sha256sum /data/config/runtime.env | awk '{print $1}'
}
@@ -233,6 +476,24 @@ assert_pinned_topology_environment() {
done
}
+assert_command_rejected_with() {
+ local accepted_message="$1"
+ local expected_message="$2"
+ shift 2
+ local output
+
+ if output="$("$@" 2>&1)"; then
+ echo "${accepted_message}" >&2
+ return 1
+ fi
+ if [[ "${output,,}" != *"${expected_message,,}"* ]]; then
+ echo "negative isolation probe failed for an unexpected reason" >&2
+ echo "expected error containing: ${expected_message}" >&2
+ printf '%s\n' "${output}" >&2
+ return 1
+ fi
+}
+
assert_frontend_database_isolation() {
local next_pid
local nginx_pid
@@ -241,6 +502,7 @@ assert_frontend_database_isolation() {
local nginx_uid
local rest_uid
local assertions
+ local frontend_role_passwordless
next_pid="$(
docker exec "${CONTAINER_NAME}" \
@@ -412,29 +674,28 @@ SQL
printf 'database privilege assertions:\n%s\n' "${assertions}" >&2
return 1
}
- if docker exec --user autogpt_frontend "${CONTAINER_NAME}" \
+ assert_command_rejected_with \
+ "frontend database role can assume postgres" \
+ 'permission denied to set role "postgres"' \
+ docker exec --user autogpt_frontend "${CONTAINER_NAME}" \
/usr/bin/env -i \
PATH=/usr/lib/postgresql/15/bin:/usr/bin:/bin \
PGHOST=/run/postgresql \
PGDATABASE=postgres \
PGUSER=autogpt_frontend \
psql --no-psqlrc --set=ON_ERROR_STOP=1 \
- --command='SET ROLE postgres' >/dev/null 2>&1; then
- echo "frontend database role can assume postgres" >&2
- return 1
- fi
+ --command='SET ROLE postgres'
- if docker exec --user autogpt_proxy "${CONTAINER_NAME}" \
+ assert_command_rejected_with \
+ "nginx operating-system user can access Valkey without authentication" \
+ "noauth authentication required" \
+ docker exec --user autogpt_proxy "${CONTAINER_NAME}" \
/usr/bin/env -i \
PATH=/usr/bin:/bin \
/app/autogpt_platform/backend/.venv/bin/python \
- /opt/autogpt/single-container/probe.py redis --port 17000 \
- >/dev/null 2>&1; then
- echo "nginx operating-system user can access Valkey without authentication" >&2
- return 1
- fi
+ /opt/autogpt/single-container/probe.py redis --port 17000
- [[ "$(
+ frontend_role_passwordless="$(
docker exec --user postgres "${CONTAINER_NAME}" \
/usr/bin/env -i \
PATH=/usr/lib/postgresql/15/bin:/usr/bin:/bin \
@@ -443,32 +704,32 @@ SQL
PGUSER=postgres \
psql --no-psqlrc --tuples-only --no-align --set=ON_ERROR_STOP=1 \
--command="SELECT rolpassword IS NULL FROM pg_catalog.pg_authid WHERE rolname = 'autogpt_frontend'"
- )" == t ]] || {
+ )"
+ [[ "${frontend_role_passwordless}" == t ]] || {
echo "frontend database role unexpectedly has a password" >&2
return 1
}
- if docker exec --user autogpt_frontend "${CONTAINER_NAME}" \
+ assert_command_rejected_with \
+ "frontend operating-system user can authenticate as postgres" \
+ 'peer authentication failed for user "postgres"' \
+ docker exec --user autogpt_frontend "${CONTAINER_NAME}" \
/usr/bin/env -i \
PATH=/usr/lib/postgresql/15/bin:/usr/bin:/bin \
PGHOST=/run/postgresql \
PGDATABASE=postgres \
PGUSER=postgres \
psql --no-psqlrc --set=ON_ERROR_STOP=1 \
- --command='SELECT 1' >/dev/null 2>&1; then
- echo "frontend operating-system user can authenticate as postgres" >&2
- return 1
- fi
+ --command='SELECT 1'
- if docker exec --user autogpt_frontend "${CONTAINER_NAME}" \
+ assert_command_rejected_with \
+ "frontend operating-system user can access Valkey without authentication" \
+ "noauth authentication required" \
+ docker exec --user autogpt_frontend "${CONTAINER_NAME}" \
/usr/bin/env -i \
PATH=/usr/bin:/bin \
/app/autogpt_platform/backend/.venv/bin/python \
- /opt/autogpt/single-container/probe.py redis --port 17000 \
- >/dev/null 2>&1; then
- echo "frontend operating-system user can access Valkey without authentication" >&2
- return 1
- fi
+ /opt/autogpt/single-container/probe.py redis --port 17000
docker exec "${CONTAINER_NAME}" \
curl --fail --silent --show-error --max-time 30 \
@@ -609,6 +870,7 @@ assert_request_tokens_absent_from_logs() {
local sentinel=AUTOGPT_LOG_SENTINEL_6f2b3cb87e9a
local websocket_key=dGhlIHNhbXBsZSBub25jZQ== # pragma: allowlist secret # gitleaks:allow
local websocket_status
+ local container_logs
curl --fail --silent --show-error --max-time 30 \
"${PUBLIC_URL}/_agpt/health?token=${sentinel}" >/dev/null
@@ -629,9 +891,11 @@ assert_request_tokens_absent_from_logs() {
return 1
}
sleep 2
- # Do not use grep -q here: with pipefail an early grep exit can SIGPIPE
- # `docker logs` and accidentally turn a positive match into a false result.
- if docker logs "${CONTAINER_NAME}" 2>&1 | grep -F "${sentinel}" >/dev/null; then
+ if ! container_logs="$(docker logs "${CONTAINER_NAME}" 2>&1)"; then
+ echo "could not inspect container logs for request-token leakage" >&2
+ return 1
+ fi
+ if [[ "${container_logs}" == *"${sentinel}"* ]]; then
echo "request token sentinel leaked into container logs" >&2
return 1
fi
@@ -667,11 +931,15 @@ assert_runtime_config_mode() {
}
assert_prisma_cli_is_prebundled() {
+ local container_logs
docker exec "${CONTAINER_NAME}" \
/usr/bin/test -f \
/opt/prisma-python/binaries/node_modules/prisma/build/index.js
- if docker logs "${CONTAINER_NAME}" 2>&1 | \
- grep -F "Installing Prisma CLI" >/dev/null; then
+ if ! container_logs="$(docker logs "${CONTAINER_NAME}" 2>&1)"; then
+ echo "could not inspect container logs for runtime Prisma installation" >&2
+ return 1
+ fi
+ if [[ "${container_logs}" == *"Installing Prisma CLI"* ]]; then
echo "Prisma CLI was installed during container startup" >&2
return 1
fi
@@ -680,27 +948,36 @@ assert_prisma_cli_is_prebundled() {
assert_falkordb_binary_contract() {
local linkage
local listeners
+ local published_port
linkage="$(
docker exec "${CONTAINER_NAME}" /bin/bash -Eeuo pipefail -c '
ldd /opt/falkordb/redis-server
ldd /opt/falkordb/falkordb.so
'
)"
- if grep -F "not found" <<<"${linkage}" >/dev/null; then
+ if [[ "${linkage}" == *"not found"* ]]; then
echo "FalkorDB has an unresolved shared-library dependency" >&2
return 1
fi
listeners="$(docker exec "${CONTAINER_NAME}" ss -lnt)"
- grep -Eq '127\.0\.0\.1:6380([[:space:]]|$)' <<<"${listeners}" || {
+ [[ "${listeners}" =~ 127\.0\.0\.1:6380([[:space:]]|$) ]] || {
echo "FalkorDB is not listening on the private loopback address" >&2
return 1
}
- if grep -Eq '(0\.0\.0\.0|\[::\]):6380([[:space:]]|$)' <<<"${listeners}"; then
+ if [[ "${listeners}" =~ (0\.0\.0\.0|\[::\]):6380([[:space:]]|$) ]]; then
echo "FalkorDB is listening on a public container interface" >&2
return 1
fi
- if [[ -n "$(docker port "${CONTAINER_NAME}" 6380 2>/dev/null)" ]]; then
+ if ! published_port="$(
+ docker inspect \
+ --format '{{with index .NetworkSettings.Ports "6380/tcp"}}{{json .}}{{end}}' \
+ "${CONTAINER_NAME}"
+ )"; then
+ echo "could not inspect Docker port bindings for FalkorDB" >&2
+ return 1
+ fi
+ if [[ -n "${published_port}" ]]; then
echo "FalkorDB port 6380 is published by Docker" >&2
return 1
fi
@@ -776,17 +1053,19 @@ wait_for_falkordb_restart() {
local deadline=$((SECONDS + TIMEOUT_SECONDS))
local current_pid
while ((SECONDS < deadline)); do
- current_pid="$(
+ if ! current_pid="$(
docker exec "${CONTAINER_NAME}" \
supervisorctl -c /opt/autogpt/single-container/supervisor/supervisord.conf \
- pid state:falkordb 2>/dev/null || true
- )"
+ pid state:falkordb 2>/dev/null
+ )"; then
+ current_pid=""
+ fi
if [[ "${current_pid}" =~ ^[0-9]+$ ]] &&
[[ "${current_pid}" != "${previous_pid}" ]] &&
appliance_is_healthy_and_armed; then
return 0
fi
- sleep 5
+ sleep 1
done
echo "FalkorDB did not restart and return the container to healthy" >&2
return 1
@@ -852,7 +1131,6 @@ assert_unsupported_email_verification_rejected() {
else
status=$?
fi
- docker rm --force --volumes "${NEGATIVE_CONTAINER_NAME}" >/dev/null 2>&1 || true
((status != 0)) || {
echo "image accepted unsupported email verification" >&2
return 1
@@ -861,9 +1139,8 @@ assert_unsupported_email_verification_rejected() {
echo "email-verification rejection probe timed out" >&2
return 1
}
- grep -Fq \
- "email verification is not supported by the single-container distribution" \
- <<<"${output}" || {
+ [[ "${output}" == \
+ *"email verification is not supported by the single-container distribution"* ]] || {
echo "unsupported email verification failed without an actionable error" >&2
return 1
}
@@ -872,12 +1149,15 @@ assert_unsupported_email_verification_rejected() {
# Phase one deliberately supplies no environment, port, volume, entrypoint, or
# command override. This is the CI proof for literal `docker run IMAGE`.
assert_unsupported_email_verification_rejected
+record_timing "unsupported-config-rejection"
docker run --detach \
--platform "${SMOKE_PLATFORM}" \
--name "${CONTAINER_NAME}" \
"${SMOKE_IMAGE}" >/dev/null
discover_data_volume
wait_for_healthy
+record_boot_milestones "initial-appliance-boot"
+record_timing "initial-appliance-boot"
assert_codex_runtime_contract
assert_prisma_cli_is_prebundled
assert_falkordb_binary_contract
@@ -891,8 +1171,10 @@ first_hash="$(runtime_config_hash)"
assert_runtime_config_mode
assert_frontend_database_isolation
+record_timing "phase-one-contracts"
assert_clean_stop "after docker stop"
docker rm "${CONTAINER_NAME}" >/dev/null
+record_timing "phase-one-clean-stop"
# A persistent user config must not be able to move internal listeners or bind
# them publicly. Environment-pinned topology has higher Pydantic precedence.
@@ -909,8 +1191,11 @@ docker run --detach \
--volume "${DATA_VOLUME}:/data" \
"${SMOKE_IMAGE}" >/dev/null
wait_for_healthy
+record_boot_milestones "replacement-appliance-boot"
+record_timing "replacement-appliance-boot"
-[[ "$(runtime_config_hash)" == "${first_hash}" ]] || {
+replacement_config_hash="$(runtime_config_hash)"
+[[ "${replacement_config_hash}" == "${first_hash}" ]] || {
echo "runtime config changed across container replacement" >&2
exit 1
}
@@ -929,6 +1214,7 @@ assert_redirect http://127.0.0.1:3300/copilot \
assert_prefixed_backend_redirect
assert_internal_tooling_is_private
assert_request_tokens_absent_from_logs
+record_timing "phase-two-contracts"
falkordb_pid="$(
docker exec "${CONTAINER_NAME}" \
@@ -941,8 +1227,11 @@ falkordb_restart_count="$(
)"
docker exec "${CONTAINER_NAME}" kill -TERM "${falkordb_pid}"
wait_for_falkordb_restart "${falkordb_pid}"
-[[ "$(docker inspect --format '{{.RestartCount}}' "${CONTAINER_NAME}")" == \
- "${falkordb_restart_count}" ]] || {
+record_timing "falkordb-process-recovery"
+current_restart_count="$(
+ docker inspect --format '{{.RestartCount}}' "${CONTAINER_NAME}"
+)"
+[[ "${current_restart_count}" == "${falkordb_restart_count}" ]] || {
echo "FalkorDB recovery unexpectedly restarted the whole container" >&2
exit 1
}
@@ -950,12 +1239,40 @@ assert_memory_contract verify
wait_for_healthy
restart_count="$(docker inspect --format '{{.RestartCount}}' "${CONTAINER_NAME}")"
+watchdog_pid="$(
+ docker exec "${CONTAINER_NAME}" \
+ supervisorctl -c /opt/autogpt/single-container/supervisor/supervisord.conf \
+ pid runtime:watchdog
+)"
+[[ "${watchdog_pid}" =~ ^[0-9]+$ ]] || {
+ echo "watchdog PID is invalid before forced health checks" >&2
+ exit 1
+}
+# Reset the failure counter while healthy. Snapshot every ordinal before
+# stopping nginx: scheduled checks can advance the same counter while Docker
+# calls are in flight, so either trigger is valid evidence for each failure.
+force_watchdog_check \
+ "${watchdog_pid}" \
+ "watchdog health check passed trigger=forced"
+record_timing "watchdog-forced-check-arm"
+
+watchdog_failure_counts=()
+for ordinal in 1 2 3; do
+ watchdog_failure_counts+=("$(count_container_log_evidence "watchdog health failure ${ordinal}/3 trigger=")")
+done
docker exec "${CONTAINER_NAME}" \
supervisorctl \
-c /opt/autogpt/single-container/supervisor/supervisord.conf \
stop runtime:nginx >/dev/null
+for ordinal in 1 2 3; do
+ advance_watchdog_failure \
+ "${watchdog_pid}" "${ordinal}" "${watchdog_failure_counts[ordinal - 1]}"
+done
wait_for_automatic_restart "$((restart_count + 1))"
-[[ "$(runtime_config_hash)" == "${first_hash}" ]] || {
+record_boot_milestones "automatic-appliance-restart"
+record_timing "automatic-appliance-restart"
+restarted_config_hash="$(runtime_config_hash)"
+[[ "${restarted_config_hash}" == "${first_hash}" ]] || {
echo "runtime config changed across automatic Docker restart" >&2
exit 1
}
@@ -967,5 +1284,11 @@ assert_memory_contract cleanup
assert_clean_stop "after the restart test"
docker rm "${CONTAINER_NAME}" >/dev/null
+record_timing "post-restart-contracts-and-clean-stop"
+# Trivy scans the immutable image, independently of either runtime container.
+# Synchronizing here preserves foreground scan failure/report propagation
+# without holding phase one open before the replacement and restart checks.
+wait_for_concurrent_scans
+record_timing "final-scan-sync"
echo "single-container smoke test passed for ${SMOKE_PLATFORM}"
diff --git a/.github/scripts/single_container_cache.py b/.github/scripts/single_container_cache.py
new file mode 100644
index 0000000000..8fc9508b25
--- /dev/null
+++ b/.github/scripts/single_container_cache.py
@@ -0,0 +1,42 @@
+#!/usr/bin/env python3
+
+import hashlib
+import os
+
+
+def cache_settings(event: str, ref: str, image: str, arch: str) -> list[str]:
+ if arch not in {"amd64", "arm64"}:
+ raise ValueError(f"Unsupported cache architecture: {arch}")
+ if event == "pull_request":
+ return []
+ shared = f"{image}:v4-{arch}"
+ imports = [shared]
+ export = None
+ if event == "workflow_dispatch":
+ branch_digest = hashlib.sha256(ref.encode()).hexdigest()[:20]
+ export = f"{image}:v4-branch-{branch_digest}-{arch}"
+ imports.insert(0, export)
+ elif event == "push" and ref == "refs/heads/dev":
+ export = shared
+ settings = [
+ f"{target}.cache-from=type=registry,ref={cache_ref}"
+ for target in ("backend-server", "single-container")
+ for cache_ref in imports
+ ]
+ if export:
+ settings.append(
+ f"single-container.cache-to=type=registry,ref={export},mode=max,"
+ "oci-mediatypes=true,image-manifest=false,ignore-error=true"
+ )
+ return settings
+
+
+if __name__ == "__main__":
+ settings = cache_settings(
+ os.environ["GITHUB_EVENT_NAME"],
+ os.environ["GITHUB_REF"],
+ os.environ["BUILD_CACHE_IMAGE"],
+ os.environ["BUILD_CACHE_ARCH"],
+ )
+ with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as output:
+ output.write("set<<CACHE_EOF\n" + "\n".join(settings) + "\nCACHE_EOF\n")
diff --git a/.github/scripts/verify_single_container_reports.py b/.github/scripts/verify_single_container_reports.py
new file mode 100644
index 0000000000..5102af7745
--- /dev/null
+++ b/.github/scripts/verify_single_container_reports.py
@@ -0,0 +1,151 @@
+#!/usr/bin/env python3
+
+import argparse
+import json
+import re
+import xml.etree.ElementTree as ET
+from pathlib import Path
+
+
+JUNIT_REPORTS = (
+ "runtime-helpers.xml",
+ "publication-policy.xml",
+ "smoke.xml",
+)
+TRIVY_REPORTS = (
+ "trivy-critical.json",
+ "trivy-secrets.json",
+)
+TRIVY_FINDING_KEYS = (
+ "Vulnerabilities",
+ "Misconfigurations",
+ "Secrets",
+ "Licenses",
+)
+
+
+def parse_args() -> argparse.Namespace:
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--report-dir", type=Path, required=True)
+ parser.add_argument("--expected-image", required=True)
+ return parser.parse_args()
+
+
+def nonnegative_count(element: ET.Element, attribute: str, report: str) -> int:
+ value = element.get(attribute)
+ if value is None:
+ raise ValueError(f"{report} is missing {attribute}")
+ try:
+ count = int(value)
+ except ValueError as error:
+ raise ValueError(f"{report} has invalid {attribute}={value!r}") from error
+ if count < 0:
+ raise ValueError(f"{report} has negative {attribute}={count}")
+ return count
+
+
+def verify_junit(path: Path) -> None:
+ root = ET.parse(path).getroot()
+ if root.tag not in {"testsuite", "testsuites"}:
+ raise ValueError(f"{path.name} is not a JUnit XML report")
+ cases = list(root.iter("testcase"))
+ summaries = [(root, cases)]
+ if root.tag == "testsuites":
+ summaries.extend(
+ (suite, list(suite.iter("testcase"))) for suite in root.iter("testsuite")
+ )
+ if not cases:
+ raise ValueError(f"{path.name} contains no test cases")
+
+ for summary, summary_cases in summaries:
+ actual = {
+ "tests": len(summary_cases),
+ "failures": sum(case.find("failure") is not None for case in summary_cases),
+ "errors": sum(case.find("error") is not None for case in summary_cases),
+ "skipped": sum(case.find("skipped") is not None for case in summary_cases),
+ }
+ for attribute, actual_count in actual.items():
+ declared_count = nonnegative_count(summary, attribute, path.name)
+ if declared_count != actual_count:
+ raise ValueError(
+ f"{path.name} declares {attribute}={declared_count}, "
+ f"found {actual_count}"
+ )
+
+ failures = sum(case.find("failure") is not None for case in cases)
+ errors = sum(case.find("error") is not None for case in cases)
+ skipped = sum(case.find("skipped") is not None for case in cases)
+ if failures:
+ raise ValueError(f"{path.name} reports {failures} failures")
+ if errors:
+ raise ValueError(f"{path.name} reports {errors} errors")
+ if len(cases) == skipped:
+ raise ValueError(f"{path.name} contains no successful test cases")
+
+
+def verify_trivy(path: Path, expected_image: str) -> None:
+ document = json.loads(path.read_text(encoding="utf-8"))
+ if not isinstance(document, dict) or "SchemaVersion" not in document:
+ raise ValueError(f"{path.name} is not a Trivy JSON report")
+ if document.get("SchemaVersion") != 2:
+ raise ValueError(f"{path.name} has an unexpected schema version")
+ if document.get("ArtifactType") != "container_image":
+ raise ValueError(f"{path.name} did not scan a container image")
+ if document.get("ArtifactName") != expected_image:
+ raise ValueError(
+ f"{path.name} scanned {document.get('ArtifactName')!r}, "
+ f"expected {expected_image!r}"
+ )
+
+ results = document.get("Results", [])
+ if not isinstance(results, list):
+ raise ValueError(f"{path.name} has an invalid Results value")
+ if not results:
+ metadata = document.get("Metadata")
+ image_id = metadata.get("ImageID") if isinstance(metadata, dict) else None
+ if (
+ path.name != "trivy-secrets.json"
+ or not isinstance(image_id, str)
+ or re.fullmatch(r"sha256:[0-9a-f]{64}", image_id) is None
+ ):
+ raise ValueError(
+ f"{path.name} has incomplete scan results or image metadata"
+ )
+
+ findings: dict[str, int] = {}
+ for result in results:
+ if not isinstance(result, dict):
+ raise ValueError(f"{path.name} has an invalid result entry")
+ for key in ("Target", "Class", "Type"):
+ if not isinstance(result.get(key), str) or not result[key]:
+ raise ValueError(f"{path.name} result is missing {key}")
+ for key in TRIVY_FINDING_KEYS:
+ if key not in result:
+ continue
+ value = result[key]
+ if not isinstance(value, list):
+ raise ValueError(f"{path.name} has an invalid {key} value")
+ if value:
+ findings[key] = findings.get(key, 0) + len(value)
+
+ if findings:
+ summary = ", ".join(f"{key}={count}" for key, count in sorted(findings.items()))
+ raise ValueError(f"{path.name} contains findings: {summary}")
+
+
+def main() -> int:
+ args = parse_args()
+ try:
+ for report in JUNIT_REPORTS:
+ verify_junit(args.report_dir / report)
+ for report in TRIVY_REPORTS:
+ verify_trivy(args.report_dir / report, args.expected_image)
+ except (OSError, ET.ParseError, json.JSONDecodeError, ValueError) as error:
+ print(f"single-container report verification failed: {error}")
+ return 1
+ print("single-container reports contain no failures or findings")
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/.github/workflows/platform-single-container-docker.yml b/.github/workflows/platform-single-container-docker.yml
index 4fd2a0c84b..2b45700a29 100644
--- a/.github/workflows/platform-single-container-docker.yml
+++ b/.github/workflows/platform-single-container-docker.yml
@@ -3,33 +3,31 @@ name: AutoGPT Platform - Single-container image
on:
push:
branches: [dev]
- paths:
- - ".dockerignore"
- - ".github/scripts/platform-single-container-publish.sh"
- - ".github/scripts/platform-single-container-smoke.sh"
- - ".github/workflows/platform-single-container-docker.yml"
- - "autogpt_platform/**"
- - "docs/platform/single-container.md"
pull_request:
paths:
- ".dockerignore"
- ".github/scripts/platform-single-container-publish.sh"
+ - ".github/scripts/run_command_junit.py"
+ - ".github/scripts/run_unittest_junit.py"
+ - ".github/scripts/verify_single_container_reports.py"
+ - ".github/scripts/single_container_cache.py"
- ".github/scripts/platform-single-container-smoke.sh"
- ".github/workflows/platform-single-container-docker.yml"
- - "autogpt_platform/**"
+ - "autogpt_platform/backend/Dockerfile"
+ - "autogpt_platform/backend/poetry.lock"
+ - "autogpt_platform/backend/pyproject.toml"
+ - "autogpt_platform/docker-compose.yml"
+ - "autogpt_platform/docker-compose.platform.yml"
+ - "autogpt_platform/frontend/package.json"
+ - "autogpt_platform/frontend/pnpm-lock.yaml"
+ - "autogpt_platform/single-container/**"
- "docs/platform/single-container.md"
release:
types: [published]
workflow_dispatch:
- inputs:
- publish:
- description: "Publish the current dev commit to Docker Hub"
- required: true
- type: boolean
- default: false
concurrency:
- group: ${{ github.workflow }}-${{ (github.event_name == 'workflow_dispatch' || github.event_name == 'release') && 'publication' || github.ref }}
+ group: ${{ github.workflow }}-${{ github.event_name == 'release' && 'publication' || github.event_name == 'workflow_dispatch' && github.run_id || github.ref }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
permissions:
@@ -37,10 +35,11 @@ permissions:
env:
DEPLOY_IMAGE: docker.io/significantgravitas/autogpt
+ BUILD_CACHE_IMAGE: ghcr.io/significant-gravitas/autogpt-single-container-buildcache
jobs:
authorize-publication:
- name: Authorize publication source
+ name: Evaluate publication eligibility
runs-on: ubuntu-24.04
outputs:
allowed: ${{ steps.policy.outputs.allowed }}
@@ -50,19 +49,19 @@ jobs:
with:
persist-credentials: false
- - name: Validate publication source
+ - name: Determine publication eligibility
id: policy
shell: bash
env:
- PUBLISH_REQUESTED: ${{ inputs.publish }}
RELEASE_PRERELEASE: ${{ github.event.release.prerelease }}
RELEASE_TAG: ${{ github.event.release.tag_name }}
run: bash .github/scripts/platform-single-container-publish.sh authorize
build-and-scan:
name: Build, smoke, and scan (${{ matrix.platform }})
- needs: authorize-publication
- if: ${{ github.event_name != 'release' || needs.authorize-publication.outputs.allowed == 'true' }}
+ permissions:
+ contents: read
+ packages: write
runs-on: ${{ matrix.runner }}
timeout-minutes: 240
strategy:
@@ -79,24 +78,69 @@ jobs:
- name: Check out repository
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
+ ref: ${{ github.sha }}
persist-credentials: false
- - name: Test appliance runtime helpers
- shell: bash
- run: python3 -m unittest discover -s autogpt_platform/single-container/tests -p 'test_*.py' -v
-
- - name: Test publication helpers
+ - name: Initialize validation reports
shell: bash
- run: bash .github/scripts/platform-single-container-publish.sh self-test
+ env:
+ REPORT_DIR: ${{ runner.temp }}/platform-single-container-results
+ run: |
+ mkdir -p "$REPORT_DIR"
+ jq -n \
+ --arg sha "$GITHUB_SHA" \
+ --arg platform "${{ matrix.platform }}" \
+ '{sha: $sha, platform: $platform, validated: false}' \
+ > "$REPORT_DIR/status.json"
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
with:
version: v0.36.0
+ cleanup: false
driver-opts: |
image=docker.io/moby/buildkit:v0.32.0@sha256:1f8167fcb0eca5b7126353d35299386945cbb8949cc516c592a49f80cfce4fa2
+ - name: Authenticate to GHCR build cache
+ if: ${{ github.event_name != 'pull_request' }}
+ uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
+ with:
+ registry: ghcr.io
+ username: ${{ github.actor }}
+ password: ${{ secrets.GITHUB_TOKEN }}
+
+ - name: Configure registry build cache
+ id: cache
+ shell: bash
+ env:
+ BUILD_CACHE_ARCH: ${{ matrix.suffix }}
+ run: python3 .github/scripts/single_container_cache.py
+
+ - name: Test appliance runtime helpers
+ shell: bash
+ env:
+ REPORT_DIR: ${{ runner.temp }}/platform-single-container-results
+ run: >-
+ python3 .github/scripts/run_unittest_junit.py
+ --start-directory autogpt_platform/single-container/tests
+ --pattern 'test_*.py'
+ --allow-skip test_documented_operations.DocumentedOperationsTest.test_restored_launch_supports_bash_3_when_available
+ --output "$REPORT_DIR/runtime-helpers.xml"
+
+ - name: Test publication helpers
+ shell: bash
+ env:
+ REPORT_DIR: ${{ runner.temp }}/platform-single-container-results
+ run: >-
+ python3 .github/scripts/run_command_junit.py
+ --name publication-policy
+ --classname single-container.publication
+ --output "$REPORT_DIR/publication-policy.xml"
+ -- bash .github/scripts/platform-single-container-publish.sh self-test
+
- name: Build image without publishing
+ id: build-image
+ background: true
uses: docker/bake-action@d3418bd7d0e9324001bca92fa8ba175ea7e6dc9b # v7.3.0
with:
source: .
@@ -104,47 +148,148 @@ jobs:
targets: single-container
set: |
*.platform=${{ matrix.platform }}
- backend-server.cache-from=type=gha,scope=platform-single-container-backend-${{ matrix.suffix }}
- backend-server.cache-to=type=gha,scope=platform-single-container-backend-${{ matrix.suffix }},mode=max,ignore-error=true
single-container.args.IMAGE_VERSION=sha-${{ github.sha }}
single-container.args.VCS_REF=${{ github.sha }}
- single-container.cache-from=type=gha,scope=platform-single-container-${{ matrix.suffix }}
- single-container.cache-to=type=gha,scope=platform-single-container-${{ matrix.suffix }},mode=max,ignore-error=true
single-container.output=type=docker
single-container.tags=autogpt-platform-single-container:ci-${{ matrix.suffix }}
+ ${{ steps.cache.outputs.set }}
+
+ - name: Wait for image build
+ wait: build-image
+
+ - name: Verify image source revision
+ shell: bash
+ env:
+ REPORT_DIR: ${{ runner.temp }}/platform-single-container-results
+ SMOKE_IMAGE: autogpt-platform-single-container:ci-${{ matrix.suffix }}
+ run: |
+ actual_revision="$(docker image inspect --format '{{ index .Config.Labels "org.opencontainers.image.revision" }}' "$SMOKE_IMAGE")"
+ if [[ "$actual_revision" != "$GITHUB_SHA" ]]; then
+ echo "expected image revision $GITHUB_SHA, got $actual_revision" >&2
+ exit 1
+ fi
+ image_id="$(docker image inspect --format '{{ .Id }}' "$SMOKE_IMAGE")"
+ jq -n \
+ --arg sha "$GITHUB_SHA" \
+ --arg platform "${{ matrix.platform }}" \
+ --arg image "$SMOKE_IMAGE" \
+ --arg image_id "$image_id" \
+ '{sha: $sha, platform: $platform, image: $image, image_id: $image_id, validated: false}' \
+ > "$REPORT_DIR/status.json"
- name: Smoke-test the complete image
+ id: complete-image-smoke
+ background: true
shell: bash
env:
+ REPORT_DIR: ${{ runner.temp }}/platform-single-container-results
SMOKE_IMAGE: autogpt-platform-single-container:ci-${{ matrix.suffix }}
SMOKE_PLATFORM: ${{ matrix.platform }}
- run: bash .github/scripts/platform-single-container-smoke.sh
+ SMOKE_SCAN_COMPLETION_FILE: ${{ runner.temp }}/platform-single-container-results/scans-complete
+ run: >-
+ python3 .github/scripts/run_command_junit.py
+ --name complete-image-smoke
+ --classname single-container.smoke
+ --output "$REPORT_DIR/smoke.xml"
+ -- bash .github/scripts/platform-single-container-smoke.sh
- name: Scan for fixable critical vulnerabilities
+ id: vulnerability_scan
+ continue-on-error: true
uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
with:
image-ref: autogpt-platform-single-container:ci-${{ matrix.suffix }}
- format: table
+ scanners: vuln,secret
+ format: json
+ output: ${{ runner.temp }}/platform-single-container-results/trivy-critical.json
ignore-unfixed: true
severity: CRITICAL
exit-code: "1"
timeout: 30m
- name: Scan image filesystem for embedded secrets
+ id: secret_scan
+ continue-on-error: true
uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
with:
image-ref: autogpt-platform-single-container:ci-${{ matrix.suffix }}
scanners: secret
- format: table
+ format: json
+ output: ${{ runner.temp }}/platform-single-container-results/trivy-secrets.json
severity: HIGH,CRITICAL
exit-code: "1"
timeout: 30m
+ - name: Signal concurrent scans complete
+ if: ${{ !cancelled() }}
+ shell: bash
+ env:
+ SCAN_COMPLETION_FILE: ${{ runner.temp }}/platform-single-container-results/scans-complete
+ run: touch "$SCAN_COMPLETION_FILE"
+
+ - name: Wait for complete-image smoke test
+ wait: complete-image-smoke
+
+ - name: Verify machine-readable reports
+ if: ${{ !cancelled() }}
+ shell: bash
+ env:
+ REPORT_DIR: ${{ runner.temp }}/platform-single-container-results
+ SMOKE_IMAGE: autogpt-platform-single-container:ci-${{ matrix.suffix }}
+ SECRET_SCAN_OUTCOME: ${{ steps.secret_scan.outcome }}
+ VULNERABILITY_SCAN_OUTCOME: ${{ steps.vulnerability_scan.outcome }}
+ run: |
+ scan_failed=0
+ if [[ "$VULNERABILITY_SCAN_OUTCOME" != "success" ]]; then
+ echo "vulnerability scan action concluded $VULNERABILITY_SCAN_OUTCOME" >&2
+ scan_failed=1
+ fi
+ if [[ "$SECRET_SCAN_OUTCOME" != "success" ]]; then
+ echo "secret scan action concluded $SECRET_SCAN_OUTCOME" >&2
+ scan_failed=1
+ fi
+ reports=(
+ runtime-helpers.xml
+ publication-policy.xml
+ smoke.xml
+ trivy-critical.json
+ trivy-secrets.json
+ )
+ for report in "${reports[@]}"; do
+ if [[ ! -s "$REPORT_DIR/$report" ]]; then
+ echo "missing machine-readable report: $report" >&2
+ exit 1
+ fi
+ done
+ python3 .github/scripts/verify_single_container_reports.py \
+ --report-dir "$REPORT_DIR" \
+ --expected-image "$SMOKE_IMAGE"
+ ((scan_failed == 0))
+
+ - name: Mark validation complete
+ shell: bash
+ env:
+ REPORT_DIR: ${{ runner.temp }}/platform-single-container-results
+ run: |
+ jq '.validated = true' "$REPORT_DIR/status.json" > "$REPORT_DIR/status.complete.json"
+ mv "$REPORT_DIR/status.complete.json" "$REPORT_DIR/status.json"
+
+ - name: Upload single-container reports
+ if: ${{ !cancelled() }}
+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+ with:
+ name: platform-single-container-ci-${{ matrix.suffix }}
+ path: ${{ runner.temp }}/platform-single-container-results
+ if-no-files-found: error
+ retention-days: 7
+
publish-platform-digests:
name: Publish, smoke, and scan (${{ matrix.platform }})
- if: ${{ needs.authorize-publication.outputs.allowed == 'true' && needs.build-and-scan.result == 'success' }}
+ if: ${{ github.event_name == 'release' && needs.authorize-publication.outputs.allowed == 'true' && needs.build-and-scan.result == 'success' }}
needs: [authorize-publication, build-and-scan]
- environment: ${{ github.event_name == 'release' && 'dockerhub-release' || 'dockerhub-publish' }}
+ permissions:
+ contents: read
+ environment: dockerhub-release
runs-on: ${{ matrix.runner }}
timeout-minutes: 240
strategy:
@@ -167,6 +312,7 @@ jobs:
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
with:
version: v0.36.0
+ cleanup: false
driver-opts: |
image=docker.io/moby/buildkit:v0.32.0@sha256:1f8167fcb0eca5b7126353d35299386945cbb8949cc516c592a49f80cfce4fa2
@@ -185,12 +331,8 @@ jobs:
targets: single-container
set: |
*.platform=${{ matrix.platform }}
- backend-server.cache-from=type=gha,scope=platform-single-container-backend-${{ matrix.suffix }}
- backend-server.cache-to=type=gha,scope=platform-single-container-backend-${{ matrix.suffix }},mode=max,ignore-error=true
single-container.args.IMAGE_VERSION=sha-${{ github.sha }}
single-container.args.VCS_REF=${{ github.sha }}
- single-container.cache-from=type=gha,scope=platform-single-container-${{ matrix.suffix }}
- single-container.cache-to=type=gha,scope=platform-single-container-${{ matrix.suffix }},mode=max,ignore-error=true
single-container.tags=${{ env.DEPLOY_IMAGE }}
single-container.attest=type=provenance,mode=max
single-container.attest=type=sbom
@@ -272,9 +414,9 @@ jobs:
publish-manifest:
name: Publish multi-platform manifest
- if: ${{ needs.authorize-publication.outputs.allowed == 'true' && needs.publish-platform-digests.result == 'success' }}
+ if: ${{ github.event_name == 'release' && needs.authorize-publication.outputs.allowed == 'true' && needs.publish-platform-digests.result == 'success' }}
needs: [authorize-publication, publish-platform-digests]
- environment: ${{ github.event_name == 'release' && 'dockerhub-release' || 'dockerhub-publish' }}
+ environment: dockerhub-release
runs-on: ubuntu-24.04
timeout-minutes: 30
steps:
diff --git a/autogpt_platform/backend/backend/executor/scheduler.py b/autogpt_platform/backend/backend/executor/scheduler.py
index c679e0db01..6e674d8a58 100644
--- a/autogpt_platform/backend/backend/executor/scheduler.py
+++ b/autogpt_platform/backend/backend/executor/scheduler.py
@@ -2109,7 +2109,10 @@ class Scheduler(AppService):
hours=6,
# Due now rather than called inline below: run_service() is what
# starts the event loop uvicorn binds the RPC port on.
- next_run_time=datetime.now(timezone.utc),
+ next_run_time=datetime.now(timezone.utc)
+ + timedelta(
+ hours=0 if config.scheduler_startup_embedding_backfill else 6
+ ),
replace_existing=True,
max_instances=1, # Prevent overlapping runs
misfire_grace_time=None,
diff --git a/autogpt_platform/backend/backend/executor/scheduler_unit_test.py b/autogpt_platform/backend/backend/executor/scheduler_unit_test.py
index eaf7df107a..0999fe25f7 100644
--- a/autogpt_platform/backend/backend/executor/scheduler_unit_test.py
+++ b/autogpt_platform/backend/backend/executor/scheduler_unit_test.py
@@ -48,6 +48,7 @@ from backend.util.exceptions import (
ExpertPrivateTenancyNotFoundError,
UserPaywalledError,
)
+from backend.util.settings import Config
_SCHEDULER_PATH = "backend.executor.scheduler"
@@ -1510,7 +1511,9 @@ class _StartupRun(NamedTuple):
backfill_calls_at_readiness: int
-def _registered_jobs(monkeypatch, interval_hours: int) -> _StartupRun:
+def _registered_jobs(
+ monkeypatch, interval_hours: int, startup_embedding_backfill: bool = True
+) -> _StartupRun:
"""Drive ``Scheduler.run_service`` with every heavy dependency stubbed and
a mock APScheduler, recording what it registered and what it ran before
handing over to ``AppService.run_service``."""
@@ -1518,6 +1521,10 @@ def _registered_jobs(monkeypatch, interval_hours: int) -> _StartupRun:
f"{_SCHEDULER_PATH}.config.stripe_tier_reconcile_interval_hours",
interval_hours,
)
+ monkeypatch.setattr(
+ f"{_SCHEDULER_PATH}.config.scheduler_startup_embedding_backfill",
+ startup_embedding_backfill,
+ )
mock_scheduler = MagicMock()
embedding_backfill = MagicMock(return_value=None)
at_readiness = []
@@ -1573,6 +1580,45 @@ def test_reconcile_stripe_tiers_interval_follows_config_setting(monkeypatch):
assert match.kwargs["seconds"] == 12 * 3600
+def test_startup_embedding_backfill_defaults_on():
+ assert Config.model_fields["scheduler_startup_embedding_backfill"].default is True
+
+
+def test_startup_embedding_backfill_can_be_disabled(monkeypatch):
+ before = datetime.now(timezone.utc)
+ run = _registered_jobs(
+ monkeypatch, interval_hours=6, startup_embedding_backfill=False
+ )
+
+ run.embedding_backfill.assert_not_called()
+ job = next(
+ c for c in run.add_job_calls if c.kwargs["id"] == "ensure_embeddings_coverage"
+ )
+ assert job.kwargs["trigger"] == "interval"
+ assert job.kwargs["hours"] == 6
+ assert before + timedelta(hours=6) <= job.kwargs["next_run_time"]
+
+
+def test_startup_embedding_backfill_runs_in_the_existing_background_job(monkeypatch):
+ before = datetime.now(timezone.utc)
+ run = _registered_jobs(monkeypatch, interval_hours=6)
+
+ run.embedding_backfill.assert_not_called()
+ jobs = [
+ c for c in run.add_job_calls if c.kwargs["id"] == "ensure_embeddings_coverage"
+ ]
+ assert len(jobs) == 1
+ job = jobs[0]
+ assert before <= job.kwargs["next_run_time"] <= datetime.now(timezone.utc)
+ assert job.kwargs["trigger"] == "interval"
+ assert job.kwargs["hours"] == 6
+ assert job.kwargs["max_instances"] == 1
+ assert job.kwargs["misfire_grace_time"] is None
+ assert job.kwargs["coalesce"] is True
+ job.args[0]()
+ run.embedding_backfill.assert_called_once_with()
+
+
def test_embedding_backfill_does_not_delay_rpc_readiness(monkeypatch):
"""``AppService.run_service`` starts the event loop uvicorn is scheduled
onto, so anything run before it keeps the RPC port closed. The backfill
diff --git a/autogpt_platform/backend/backend/util/settings.py b/autogpt_platform/backend/backend/util/settings.py
index feb804130c..825068c1dd 100644
--- a/autogpt_platform/backend/backend/util/settings.py
+++ b/autogpt_platform/backend/backend/util/settings.py
@@ -651,6 +651,14 @@ class Config(UpdateTrackingModel["Config"], BaseSettings):
),
)
+ scheduler_startup_embedding_backfill: bool = Field(
+ default=True,
+ description=(
+ "Run the first search embedding coverage backfill in the background "
+ "when the scheduler starts instead of waiting six hours"
+ ),
+ )
+
upload_file_size_limit_mb: int = Field(
default=256,
ge=1,
diff --git a/autogpt_platform/single-container/tests/test_ci_registry_cache.py b/autogpt_platform/single-container/tests/test_ci_registry_cache.py
new file mode 100644
index 0000000000..13be35c767
--- /dev/null
+++ b/autogpt_platform/single-container/tests/test_ci_registry_cache.py
@@ -0,0 +1,67 @@
+import importlib.util
+import unittest
+from pathlib import Path
+
+
+SCRIPT = (
+ Path(__file__).resolve().parents[3] / ".github/scripts/single_container_cache.py"
+)
+SPEC = importlib.util.spec_from_file_location("single_container_cache", SCRIPT)
+cache = importlib.util.module_from_spec(SPEC)
+SPEC.loader.exec_module(cache)
+IMAGE = "ghcr.io/example/appliance-cache"
+
+
+class RegistryCacheTests(unittest.TestCase):
+ def settings(self, event, ref="refs/heads/dev", arch="amd64"):
+ return cache.cache_settings(event, ref, IMAGE, arch)
+
+ def test_pull_requests_never_read_or_write_registry_cache(self):
+ self.assertEqual(self.settings("pull_request"), [])
+
+ def test_only_dev_push_writes_the_shared_validation_cache(self):
+ settings = self.settings("push")
+ self.assertIn(
+ f"single-container.cache-to=type=registry,ref={IMAGE}:v4-amd64,mode=max,oci-mediatypes=true,image-manifest=false,ignore-error=true",
+ settings,
+ )
+ self.assertFalse(
+ any("cache-to" in s for s in self.settings("push", "refs/heads/test"))
+ )
+
+ def test_dispatch_cache_is_stable_branch_scoped_and_never_shared_cache(self):
+ for ref in ("refs/heads/dev", "refs/heads/test"):
+ settings = self.settings("workflow_dispatch", ref)
+ writes = [s for s in settings if "cache-to" in s]
+ self.assertEqual(len(writes), 1)
+ self.assertIn(":v4-branch-", writes[0])
+ self.assertIn("mode=max", writes[0])
+ self.assertIn("ignore-error=true", writes[0])
+ self.assertIn(
+ f"single-container.cache-from=type=registry,ref={IMAGE}:v4-amd64",
+ settings,
+ )
+ self.assertEqual(settings, self.settings("workflow_dispatch", ref))
+ self.assertNotEqual(
+ self.settings("workflow_dispatch", "refs/heads/one"),
+ self.settings("workflow_dispatch", "refs/heads/two"),
+ )
+
+ def test_validation_cache_reads_are_architecture_scoped(self):
+ settings = self.settings("release", "refs/tags/v1")
+ self.assertTrue(all("cache-from" in s and ":v4-amd64" in s for s in settings))
+ self.assertTrue(
+ all(":v4-arm64" in s for s in self.settings("release", arch="arm64"))
+ )
+
+ def test_release_digest_build_does_not_trust_writable_ci_cache(self):
+ workflow = (
+ SCRIPT.parents[1] / "workflows/platform-single-container-docker.yml"
+ ).read_text(encoding="utf-8")
+ release_job = workflow.split(" publish-platform-digests:", 1)[1].split(
+ " publish-manifest:", 1
+ )[0]
+ self.assertNotIn("cache-from", release_job)
+ self.assertNotIn("cache-to", release_job)
+ self.assertNotIn("GHCR", release_job)
+ self.assertNotIn("packages:", release_job)
diff --git a/autogpt_platform/single-container/tests/test_ci_report_verifier.py b/autogpt_platform/single-container/tests/test_ci_report_verifier.py
new file mode 100644
index 0000000000..79e06bc966
--- /dev/null
+++ b/autogpt_platform/single-container/tests/test_ci_report_verifier.py
@@ -0,0 +1,271 @@
+import json
+import subprocess
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+
+REPOSITORY_ROOT = Path(__file__).resolve().parents[3]
+REPORT_VERIFIER = REPOSITORY_ROOT / ".github/scripts/verify_single_container_reports.py"
+
+
+class ReportVerifierTests(unittest.TestCase):
+ def setUp(self):
+ self.temporary_directory = tempfile.TemporaryDirectory()
+ self.report_dir = Path(self.temporary_directory.name)
+ passing_junit = (
+ '<testsuite tests="1" failures="0" errors="0" skipped="0">'
+ '<testcase classname="ci" name="passes" />'
+ "</testsuite>"
+ )
+ for name in ("runtime-helpers.xml", "publication-policy.xml", "smoke.xml"):
+ (self.report_dir / name).write_text(passing_junit, encoding="utf-8")
+ passing_trivy = {
+ "SchemaVersion": 2,
+ "ArtifactName": "single-container:test",
+ "ArtifactType": "container_image",
+ "Results": [
+ {
+ "Target": "single-container:test (debian)",
+ "Class": "os-pkgs",
+ "Type": "debian",
+ "Vulnerabilities": [],
+ "Secrets": [],
+ }
+ ],
+ }
+ for name in ("trivy-critical.json", "trivy-secrets.json"):
+ (self.report_dir / name).write_text(
+ json.dumps(passing_trivy), encoding="utf-8"
+ )
+
+ def tearDown(self):
+ self.temporary_directory.cleanup()
+
+ def run_verifier(self):
+ return subprocess.run(
+ [
+ sys.executable,
+ str(REPORT_VERIFIER),
+ "--report-dir",
+ str(self.report_dir),
+ "--expected-image",
+ "single-container:test",
+ ],
+ check=False,
+ capture_output=True,
+ text=True,
+ )
+
+ def test_passing_reports_succeed(self):
+ result = self.run_verifier()
+
+ self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
+
+ def test_clean_secret_scan_accepts_empty_or_omitted_results(self):
+ path = self.report_dir / "trivy-secrets.json"
+ report = json.loads(path.read_text(encoding="utf-8"))
+ report["Metadata"] = {"ImageID": "sha256:" + "a" * 64}
+ for results_present in (True, False):
+ with self.subTest(results_present=results_present):
+ if results_present:
+ report["Results"] = []
+ else:
+ del report["Results"]
+ path.write_text(json.dumps(report), encoding="utf-8")
+ result = self.run_verifier()
+ self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
+
+ def test_empty_secret_results_without_image_metadata_are_rejected(self):
+ path = self.report_dir / "trivy-secrets.json"
+ report = json.loads(path.read_text(encoding="utf-8"))
+ report["Results"] = []
+ path.write_text(json.dumps(report), encoding="utf-8")
+ self.assertNotEqual(self.run_verifier().returncode, 0)
+
+ def test_empty_vulnerability_results_are_rejected(self):
+ path = self.report_dir / "trivy-critical.json"
+ report = json.loads(path.read_text(encoding="utf-8"))
+ report["Metadata"] = {"ImageID": "sha256:" + "a" * 64}
+ report["Results"] = []
+ path.write_text(json.dumps(report), encoding="utf-8")
+ self.assertNotEqual(self.run_verifier().returncode, 0)
+
+ def test_junit_failure_is_rejected_even_when_summary_claims_success(self):
+ (self.report_dir / "smoke.xml").write_text(
+ '<testsuite tests="1" failures="0" errors="0" skipped="0">'
+ '<testcase classname="ci" name="fails"><failure /></testcase>'
+ "</testsuite>",
+ encoding="utf-8",
+ )
+
+ result = self.run_verifier()
+
+ self.assertNotEqual(result.returncode, 0)
+ self.assertIn("declares failures=0, found 1", result.stdout)
+
+ def test_junit_error_count_is_rejected(self):
+ (self.report_dir / "runtime-helpers.xml").write_text(
+ '<testsuite tests="1" failures="0" errors="1" skipped="0">'
+ '<testcase classname="ci" name="errors"><error /></testcase>'
+ "</testsuite>",
+ encoding="utf-8",
+ )
+
+ result = self.run_verifier()
+
+ self.assertNotEqual(result.returncode, 0)
+ self.assertIn("reports 1 errors", result.stdout)
+
+ def test_junit_aggregate_failure_is_rejected(self):
+ (self.report_dir / "smoke.xml").write_text(
+ '<testsuites tests="1" failures="1" errors="0" skipped="0">'
+ '<testsuite tests="1" failures="0" errors="0" skipped="0">'
+ '<testcase classname="ci" name="misreported" />'
+ "</testsuite>"
+ "</testsuites>",
+ encoding="utf-8",
+ )
+
+ result = self.run_verifier()
+
+ self.assertNotEqual(result.returncode, 0)
+ self.assertIn("declares failures=1, found 0", result.stdout)
+
+ def test_nested_junit_suites_are_accepted(self):
+ (self.report_dir / "smoke.xml").write_text(
+ '<testsuites tests="1" failures="0" errors="0" skipped="0">'
+ '<testsuite tests="1" failures="0" errors="0" skipped="0">'
+ '<testsuite tests="1" failures="0" errors="0" skipped="0">'
+ '<testcase classname="ci" name="nested" />'
+ "</testsuite>"
+ "</testsuite>"
+ "</testsuites>",
+ encoding="utf-8",
+ )
+
+ result = self.run_verifier()
+
+ self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
+
+ def test_junit_count_mismatch_is_rejected(self):
+ (self.report_dir / "publication-policy.xml").write_text(
+ '<testsuite tests="99" failures="0" errors="0" skipped="0">'
+ '<testcase classname="ci" name="only-one" />'
+ "</testsuite>",
+ encoding="utf-8",
+ )
+
+ result = self.run_verifier()
+
+ self.assertNotEqual(result.returncode, 0)
+ self.assertIn("declares tests=99, found 1", result.stdout)
+
+ def test_junit_skipped_only_report_is_rejected(self):
+ (self.report_dir / "smoke.xml").write_text(
+ '<testsuite tests="1" failures="0" errors="0" skipped="1">'
+ '<testcase classname="ci" name="skipped"><skipped /></testcase>'
+ "</testsuite>",
+ encoding="utf-8",
+ )
+
+ result = self.run_verifier()
+
+ self.assertNotEqual(result.returncode, 0)
+ self.assertIn("contains no successful test cases", result.stdout)
+
+ def test_trivy_findings_are_rejected(self):
+ report = {
+ "SchemaVersion": 2,
+ "ArtifactName": "single-container:test",
+ "ArtifactType": "container_image",
+ "Results": [
+ {
+ "Target": "single-container:test (debian)",
+ "Class": "os-pkgs",
+ "Type": "debian",
+ "Vulnerabilities": [{"ID": "CVE"}],
+ }
+ ],
+ }
+ (self.report_dir / "trivy-critical.json").write_text(
+ json.dumps(report), encoding="utf-8"
+ )
+
+ result = self.run_verifier()
+
+ self.assertNotEqual(result.returncode, 0)
+ self.assertIn("Vulnerabilities=1", result.stdout)
+
+ def test_non_trivy_json_is_rejected(self):
+ (self.report_dir / "trivy-secrets.json").write_text("{}", encoding="utf-8")
+
+ result = self.run_verifier()
+
+ self.assertNotEqual(result.returncode, 0)
+ self.assertIn("is not a Trivy JSON report", result.stdout)
+
+ def test_trivy_report_without_scanned_results_is_rejected(self):
+ (self.report_dir / "trivy-secrets.json").write_text(
+ json.dumps(
+ {
+ "SchemaVersion": 2,
+ "ArtifactName": "single-container:test",
+ "ArtifactType": "container_image",
+ "Results": None,
+ }
+ ),
+ encoding="utf-8",
+ )
+
+ result = self.run_verifier()
+
+ self.assertNotEqual(result.returncode, 0)
+ self.assertIn("invalid Results value", result.stdout)
+
+ def test_trivy_result_without_scan_identity_is_rejected(self):
+ (self.report_dir / "trivy-secrets.json").write_text(
+ json.dumps(
+ {
+ "SchemaVersion": 2,
+ "ArtifactName": "single-container:test",
+ "ArtifactType": "container_image",
+ "Results": [{}],
+ }
+ ),
+ encoding="utf-8",
+ )
+
+ result = self.run_verifier()
+
+ self.assertNotEqual(result.returncode, 0)
+ self.assertIn("result is missing Target", result.stdout)
+
+ def test_trivy_null_finding_list_is_rejected(self):
+ report = json.loads((self.report_dir / "trivy-critical.json").read_text())
+ report["Results"][0]["Vulnerabilities"] = None
+ (self.report_dir / "trivy-critical.json").write_text(
+ json.dumps(report), encoding="utf-8"
+ )
+
+ result = self.run_verifier()
+
+ self.assertNotEqual(result.returncode, 0)
+ self.assertIn("invalid Vulnerabilities value", result.stdout)
+
+ def test_trivy_wrong_image_is_rejected(self):
+ report = json.loads((self.report_dir / "trivy-critical.json").read_text())
+ report["ArtifactName"] = "other:image"
+ (self.report_dir / "trivy-critical.json").write_text(
+ json.dumps(report), encoding="utf-8"
+ )
+
+ result = self.run_verifier()
+
+ self.assertNotEqual(result.returncode, 0)
+ self.assertIn("scanned 'other:image'", result.stdout)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/autogpt_platform/single-container/tests/test_ci_smoke_stop.py b/autogpt_platform/single-container/tests/test_ci_smoke_stop.py
new file mode 100644
index 0000000000..c301f32b05
--- /dev/null
+++ b/autogpt_platform/single-container/tests/test_ci_smoke_stop.py
@@ -0,0 +1,75 @@
+import re
+import shutil
+import subprocess
+import sys
+import unittest
+from pathlib import Path
+from unittest.mock import patch
+
+
+SMOKE = (
+ Path(__file__).resolve().parents[3]
+ / ".github/scripts/platform-single-container-smoke.sh"
+)
+
+
+def bash_executable() -> str:
+ if sys.platform != "win32":
+ return "/bin/bash"
+ executable = shutil.which("bash")
+ if executable is None:
+ raise RuntimeError("Git Bash must be installed and on PATH for this test")
+ return executable
+
+
+class SmokeStopTests(unittest.TestCase):
+ def test_uses_fixed_bash_path_on_posix(self):
+ with patch("sys.platform", "linux"), patch("shutil.which") as which:
+ self.assertEqual(bash_executable(), "/bin/bash")
+ which.assert_not_called()
+
+ def test_resolves_bash_executable_on_windows(self):
+ selected = "C:/Program Files/Git/bin/bash.exe"
+ with patch("sys.platform", "win32"), patch(
+ "shutil.which", return_value=selected
+ ) as which:
+ self.assertEqual(bash_executable(), selected)
+ which.assert_called_once_with("bash")
+
+ def test_missing_windows_bash_fails_instead_of_skipping(self):
+ with patch("sys.platform", "win32"), patch("shutil.which", return_value=None):
+ with self.assertRaisesRegex(RuntimeError, "Git Bash"):
+ bash_executable()
+
+ def test_invalid_or_empty_finish_time_fails_with_diagnostic(self):
+ function = re.search(
+ r"^assert_clean_stop\(\) \{.*?^\}",
+ SMOKE.read_text(encoding="utf-8"),
+ re.MULTILINE | re.DOTALL,
+ ).group()
+ for date_status in (0, 1):
+ with self.subTest(date_status=date_status):
+ script = (
+ "set -Eeuo pipefail\n"
+ "STOCK_DOCKER_STOP_TIMEOUT=10\n"
+ "CONTAINER_NAME=test-container\n"
+ "docker() { printf '%s' invalid-timestamp; }\n"
+ f"date() {{ return {date_status}; }}\n"
+ "awk() { echo UNEXPECTED_AWK >&2; return 1; }\n"
+ f"{function}\n"
+ "assert_clean_stop test-stop\n"
+ )
+ result = subprocess.run(
+ [bash_executable(), "--noprofile", "--norc", "-s"],
+ input=script.encode("utf-8"),
+ capture_output=True,
+ check=False,
+ )
+ stderr = result.stderr.decode("utf-8")
+ self.assertNotEqual(result.returncode, 0)
+ self.assertIn("invalid container finish timestamp", stderr)
+ self.assertNotIn("UNEXPECTED_AWK", stderr)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/autogpt_platform/single-container/tests/test_ci_watchdog_evidence.py b/autogpt_platform/single-container/tests/test_ci_watchdog_evidence.py
new file mode 100644
index 0000000000..03c34c460e
--- /dev/null
+++ b/autogpt_platform/single-container/tests/test_ci_watchdog_evidence.py
@@ -0,0 +1,101 @@
+import re
+import subprocess
+import unittest
+
+from test_ci_smoke_stop import SMOKE, bash_executable
+
+
+class WatchdogEvidenceTests(unittest.TestCase):
+ def run_case(
+ self,
+ *,
+ already_seen=0,
+ previous_count=0,
+ signal_adds_evidence=1,
+ signal_status=0,
+ observed_ordinal=2,
+ ):
+ source = SMOKE.read_text(encoding="utf-8")
+ functions = []
+ for name in (
+ "count_container_log_evidence",
+ "wait_for_new_container_log_evidence",
+ "advance_watchdog_failure",
+ ):
+ match = re.search(
+ rf"^{name}\(\) \{{.*?^\}}", source, re.MULTILINE | re.DOTALL
+ )
+ self.assertIsNotNone(match, f"missing {name}")
+ functions.append(match.group())
+ script = (
+ "\n".join(functions)
+ + f"""
+set -Eeuo pipefail
+TIMEOUT_SECONDS=2
+CONTAINER_NAME=test
+COUNT={already_seen}
+docker() {{
+ if [[ "$1" == logs ]]; then
+ for ((i=0; i<COUNT; i++)); do
+ printf '[single-container] watchdog health failure {observed_ordinal}/3 trigger=scheduled\\n'
+ done
+ return 0
+ fi
+ printf 'signal\\n'
+ COUNT=$((COUNT + {signal_adds_evidence}))
+ return {signal_status}
+}}
+advance_watchdog_failure 123 2 {previous_count}
+"""
+ )
+ result = subprocess.run(
+ [bash_executable(), "--noprofile", "--norc", "-s"],
+ input=script.encode("utf-8"),
+ capture_output=True,
+ timeout=5,
+ check=False,
+ )
+ return subprocess.CompletedProcess(
+ result.args,
+ result.returncode,
+ result.stdout.decode("utf-8"),
+ result.stderr.decode("utf-8"),
+ )
+
+ def test_scheduled_failure_before_signal_needs_no_extra_signal(self):
+ result = self.run_case(already_seen=1)
+ self.assertEqual(result.returncode, 0, result.stderr)
+ self.assertNotIn("signal", result.stdout)
+
+ def test_forced_check_still_requires_new_failure_evidence(self):
+ result = self.run_case()
+ self.assertEqual(result.returncode, 0, result.stderr)
+ self.assertIn("signal", result.stdout)
+
+ def test_restart_racing_with_signal_accepts_observed_failure(self):
+ result = self.run_case(signal_status=1)
+ self.assertEqual(result.returncode, 0, result.stderr)
+
+ def test_failed_signal_without_evidence_fails(self):
+ result = self.run_case(signal_adds_evidence=0, signal_status=1)
+ self.assertNotEqual(result.returncode, 0)
+ self.assertIn("could not advance watchdog failure", result.stderr)
+
+ def test_successful_signal_without_evidence_fails(self):
+ result = self.run_case(signal_adds_evidence=0)
+ self.assertNotEqual(result.returncode, 0)
+ self.assertIn("did not report new watchdog evidence", result.stderr)
+
+ def test_old_failure_evidence_does_not_satisfy_new_check(self):
+ result = self.run_case(already_seen=1, previous_count=1)
+ self.assertEqual(result.returncode, 0, result.stderr)
+ self.assertIn("signal", result.stdout)
+
+ def test_different_failure_ordinal_does_not_satisfy_check(self):
+ result = self.run_case(observed_ordinal=3)
+ self.assertNotEqual(result.returncode, 0)
+ self.assertIn("did not report new watchdog evidence", result.stderr)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/autogpt_platform/single-container/tests/test_entrypoint.py b/autogpt_platform/single-container/tests/test_entrypoint.py
index a125d3c4d5..b0d01211d1 100644
--- a/autogpt_platform/single-container/tests/test_entrypoint.py
+++ b/autogpt_platform/single-container/tests/test_entrypoint.py
@@ -141,6 +141,13 @@ class AccountRegistrationTest(unittest.TestCase):
class EnvironmentPolicyTest(unittest.TestCase):
+ def test_image_preserves_background_embedding_backfill_at_startup(
+ self,
+ ) -> None:
+ dockerfile = DOCKERFILE_PATH.read_text(encoding="utf-8")
+
+ self.assertNotIn("SCHEDULER_STARTUP_EMBEDDING_BACKFILL=false", dockerfile)
+
def test_rejects_required_email_verification(self) -> None:
result = self._configure(AUTH_REQUIRE_EMAIL_VERIFICATION="true")
diff --git a/autogpt_platform/single-container/tests/test_supervisor_config.py b/autogpt_platform/single-container/tests/test_supervisor_config.py
index 73008c674d..7a146012cc 100644
--- a/autogpt_platform/single-container/tests/test_supervisor_config.py
+++ b/autogpt_platform/single-container/tests/test_supervisor_config.py
@@ -8,6 +8,16 @@ from pathlib import Path
ASSET_DIR = Path(__file__).resolve().parents[1]
SUPERVISOR_PATH = ASSET_DIR / "supervisor" / "supervisord.conf"
HEALTHCHECK_PATH = ASSET_DIR / "healthcheck.sh"
+WATCHDOG_PATH = ASSET_DIR / "watchdog.sh"
+SMOKE_PATH = (
+ ASSET_DIR.parents[1] / ".github" / "scripts" / "platform-single-container-smoke.sh"
+)
+WORKFLOW_PATH = (
+ ASSET_DIR.parents[1]
+ / ".github"
+ / "workflows"
+ / "platform-single-container-docker.yml"
+)
# Unraid ships Docker's stock stop timeout and operators must not have to raise
# it host-wide to run this appliance, so the whole supervised shutdown has to
@@ -201,5 +211,89 @@ class HealthcheckSupervisorNamesTest(unittest.TestCase):
self.assertEqual(checked, expected)
+class WatchdogCadenceTest(unittest.TestCase):
+ def setUp(self) -> None:
+ self.watchdog = WATCHDOG_PATH.read_text(encoding="utf-8")
+ self.smoke = SMOKE_PATH.read_text(encoding="utf-8")
+
+ def _constant(self, name: str) -> int:
+ match = re.search(rf"^readonly {name}=(\d+)$", self.watchdog, re.MULTILINE)
+ self.assertIsNotNone(match)
+ assert match is not None
+ return int(match.group(1))
+
+ def test_failure_limit_remains_three(self) -> None:
+ self.assertEqual(self._constant("FAILURE_LIMIT"), 3)
+
+ force_check = self.smoke.split("force_watchdog_check() {", maxsplit=1)[1].split(
+ "\n}\n", maxsplit=1
+ )[0]
+ count_position = force_check.index(
+ 'previous_count="$(count_container_log_evidence "${expected}")"'
+ )
+ signal_position = force_check.index(
+ 'docker exec "${CONTAINER_NAME}" kill -USR1 "${watchdog_pid}"'
+ )
+ wait_position = force_check.index(
+ 'wait_for_new_container_log_evidence "${expected}" "${previous_count}"'
+ )
+ self.assertLess(count_position, signal_position)
+ self.assertLess(signal_position, wait_position)
+
+ evidence_wait = self.smoke.split(
+ "wait_for_new_container_log_evidence() {", maxsplit=1
+ )[1].split("\n}\n", maxsplit=1)[0]
+ self.assertIn("((current_count > previous_count))", evidence_wait)
+
+ def test_steady_state_cadence_remains_thirty_seconds(self) -> None:
+ self.assertEqual(self._constant("CHECK_INTERVAL"), 30)
+ self.assertIn("while true; do\n wait_for_next_check", self.watchdog)
+ self.assertIn('sleep "${CHECK_INTERVAL}" &', self.watchdog)
+ self.assertIn("trap queue_forced_check USR1", self.watchdog)
+
+ def test_initial_health_poll_is_faster_than_steady_state_poll(self) -> None:
+ self.assertEqual(self._constant("INITIAL_CHECK_INTERVAL"), 1)
+
+ initial_wait = self.watchdog.split("wait_for_initial_health()", maxsplit=1)[
+ 1
+ ].split("stop_appliance()", maxsplit=1)[0]
+ self.assertIn('sleep "${INITIAL_CHECK_INTERVAL}"', initial_wait)
+ self.assertNotIn('sleep "${CHECK_INTERVAL}"', initial_wait)
+
+
+class WorkflowFailurePropagationTest(unittest.TestCase):
+ def test_scans_join_smoke_and_fail_from_recorded_outcomes(self) -> None:
+ workflow = WORKFLOW_PATH.read_text(encoding="utf-8")
+ validation_job = workflow.split(" build-and-scan:", maxsplit=1)[1].split(
+ " publish-platform-digests:", maxsplit=1
+ )[0]
+
+ self.assertIn(
+ " - name: Scan for fixable critical vulnerabilities\n"
+ " id: vulnerability_scan\n"
+ " continue-on-error: true",
+ validation_job,
+ )
+ self.assertIn(
+ " - name: Scan image filesystem for embedded secrets\n"
+ " id: secret_scan\n"
+ " continue-on-error: true",
+ validation_job,
+ )
+ self.assertIn(
+ " - name: Wait for complete-image smoke test\n"
+ " wait: complete-image-smoke",
+ validation_job,
+ )
+ self.assertIn(
+ "VULNERABILITY_SCAN_OUTCOME: ${{ steps.vulnerability_scan.outcome }}",
+ validation_job,
+ )
+ self.assertIn(
+ "SECRET_SCAN_OUTCOME: ${{ steps.secret_scan.outcome }}", validation_job
+ )
+ self.assertIn("((scan_failed == 0))", validation_job)
+
+
if __name__ == "__main__":
unittest.main()
diff --git a/autogpt_platform/single-container/tests/test_watchdog.py b/autogpt_platform/single-container/tests/test_watchdog.py
new file mode 100644
index 0000000000..81783e593b
--- /dev/null
+++ b/autogpt_platform/single-container/tests/test_watchdog.py
@@ -0,0 +1,260 @@
+from __future__ import annotations
+
+import os
+import queue
+import signal
+import subprocess
+import tempfile
+import threading
+import time
+import unittest
+from pathlib import Path
+
+ASSET_DIR = Path(__file__).resolve().parents[1]
+WATCHDOG_PATH = ASSET_DIR / "watchdog.sh"
+
+HARNESS = r"""
+set -Eeuo pipefail
+export AUTOGPT_RUNTIME_DIR="$1"
+export AUTOGPT_READY_FILE="${AUTOGPT_RUNTIME_DIR}/ready"
+export AUTOGPT_RUNTIME_ENV="${AUTOGPT_RUNTIME_DIR}/runtime.env"
+export AUTOGPT_ASSET_DIR="$2"
+
+source "$3"
+
+HEALTHCHECK_COUNT=0
+REAL_SLEEP="$(type -P sleep)"
+
+wait_for_ready_file() {
+ :
+}
+
+wait_for_initial_health() {
+ printf 'harness-ready\n'
+}
+
+run_healthcheck() {
+ local output="$1"
+ ((HEALTHCHECK_COUNT += 1))
+ printf 'healthcheck-%s\n' "${HEALTHCHECK_COUNT}"
+ if [[ "${WATCHDOG_TEST_MODE}" == forced && "${HEALTHCHECK_COUNT}" -eq 1 ]]; then
+ return 0
+ fi
+ printf 'expected test failure\n' >"${output}"
+ return 1
+}
+
+stop_appliance() {
+ printf 'harness-stopped\n'
+ exit 0
+}
+
+if [[ "${WATCHDOG_TEST_MODE}" == forced ]]; then
+ notify_check_timer_started() {
+ local watchdog_pid="$$"
+ printf 'timer-ready\n'
+ (
+ "${REAL_SLEEP}" 0.05
+ kill -USR1 "${watchdog_pid}"
+ ) &
+ }
+ sleep() {
+ exec "${REAL_SLEEP}" "$@"
+ }
+elif [[ "${WATCHDOG_TEST_MODE}" == periodic ]]; then
+ sleep() {
+ :
+ }
+fi
+
+main
+"""
+
+
+ os.name == "posix" and hasattr(signal, "SIGUSR1"),
+ "watchdog signals require a POSIX host",
+)
+class WatchdogSchedulingTest(unittest.TestCase):
+ def test_sigusr1_queued_before_timer_runs_without_delay(self) -> None:
+ harness = r"""
+set -Eeuo pipefail
+export AUTOGPT_RUNTIME_DIR="$1"
+export AUTOGPT_ASSET_DIR="$2"
+source "$3"
+trap queue_forced_check USR1
+kill -USR1 "$$"
+wait_for_next_check
+printf 'trigger=%s\n' "${CHECK_TRIGGER}"
+"""
+ with tempfile.TemporaryDirectory() as runtime_dir:
+ result = subprocess.run(
+ [
+ "bash",
+ "-c",
+ harness,
+ "bash",
+ runtime_dir,
+ str(ASSET_DIR),
+ str(WATCHDOG_PATH),
+ ],
+ capture_output=True,
+ text=True,
+ timeout=5,
+ check=False,
+ )
+
+ self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
+ self.assertEqual(result.stdout, "trigger=forced\n")
+
+ def test_periodic_timer_failure_is_not_treated_as_forced_check(self) -> None:
+ harness = r"""
+set -Eeuo pipefail
+export AUTOGPT_RUNTIME_DIR="$1"
+export AUTOGPT_ASSET_DIR="$2"
+source "$3"
+sleep() {
+ return 7
+}
+wait_for_next_check
+"""
+ with tempfile.TemporaryDirectory() as runtime_dir:
+ result = subprocess.run(
+ [
+ "bash",
+ "-c",
+ harness,
+ "bash",
+ runtime_dir,
+ str(ASSET_DIR),
+ str(WATCHDOG_PATH),
+ ],
+ capture_output=True,
+ text=True,
+ timeout=5,
+ check=False,
+ )
+
+ self.assertEqual(result.returncode, 1, result.stdout + result.stderr)
+ self.assertIn("watchdog check timer failed with status 7", result.stderr)
+
+ def test_periodic_timer_runs_healthcheck_until_failure_limit(self) -> None:
+ with tempfile.TemporaryDirectory() as runtime_dir:
+ result = subprocess.run(
+ [
+ "bash",
+ "-c",
+ HARNESS,
+ "bash",
+ runtime_dir,
+ str(ASSET_DIR),
+ str(WATCHDOG_PATH),
+ ],
+ capture_output=True,
+ text=True,
+ timeout=10,
+ check=False,
+ env={**os.environ, "WATCHDOG_TEST_MODE": "periodic"},
+ )
+
+ self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
+ self.assertEqual(result.stdout.count("healthcheck-"), 3)
+ first = result.stderr.index("health failure 1/3 trigger=scheduled")
+ second = result.stderr.index("health failure 2/3 trigger=scheduled")
+ third = result.stderr.index("health failure 3/3 trigger=scheduled")
+ self.assertLess(first, second)
+ self.assertLess(second, third)
+ self.assertIn("harness-stopped", result.stdout)
+
+ def test_sigusr1_forces_ordered_checks_through_same_healthcheck(self) -> None:
+ with tempfile.TemporaryDirectory() as runtime_dir:
+ process = subprocess.Popen(
+ [
+ "bash",
+ "-c",
+ HARNESS,
+ "bash",
+ runtime_dir,
+ str(ASSET_DIR),
+ str(WATCHDOG_PATH),
+ ],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT,
+ text=True,
+ bufsize=1,
+ env={**os.environ, "WATCHDOG_TEST_MODE": "forced"},
+ )
+ output: list[str] = []
+ lines: queue.Queue[str] = queue.Queue()
+ reader = threading.Thread(
+ target=self._collect_lines,
+ args=(process, output, lines),
+ daemon=True,
+ )
+ reader.start()
+ try:
+ self._wait_for_line(process, output, lines, "harness-ready")
+ self._wait_for_line(process, output, lines, "timer-ready")
+ self._wait_for_line(
+ process,
+ output,
+ lines,
+ "health check passed trigger=forced",
+ )
+ for failure in range(1, 4):
+ self._wait_for_line(process, output, lines, "timer-ready")
+ self._wait_for_line(
+ process,
+ output,
+ lines,
+ f"health failure {failure}/3 trigger=forced",
+ )
+ returncode = process.wait(timeout=5)
+ finally:
+ if process.poll() is None:
+ process.terminate()
+ process.wait(timeout=5)
+ reader.join(timeout=5)
+ if process.stdout is not None:
+ process.stdout.close()
+
+ rendered = "".join(output)
+ self.assertFalse(reader.is_alive(), rendered)
+ self.assertEqual(returncode, 0, rendered)
+ self.assertEqual(rendered.count("healthcheck-"), 4)
+ self.assertIn("harness-stopped", rendered)
+
+ def _wait_for_line(
+ self,
+ process: subprocess.Popen[str],
+ output: list[str],
+ lines: queue.Queue[str],
+ expected: str,
+ ) -> None:
+ deadline = time.monotonic() + 5
+ while time.monotonic() < deadline:
+ try:
+ line = lines.get(timeout=deadline - time.monotonic())
+ except queue.Empty:
+ break
+ if expected in line:
+ return
+ self.fail(
+ f"watchdog did not emit {expected!r}; returncode={process.poll()}; "
+ f"output={''.join(output)!r}"
+ )
+
+ def _collect_lines(
+ self,
+ process: subprocess.Popen[str],
+ output: list[str],
+ lines: queue.Queue[str],
+ ) -> None:
+ assert process.stdout is not None
+ for line in process.stdout:
+ output.append(line)
+ lines.put(line)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/autogpt_platform/single-container/watchdog.sh b/autogpt_platform/single-container/watchdog.sh
index 1f21afb57a..a4049bb803 100644
--- a/autogpt_platform/single-container/watchdog.sh
+++ b/autogpt_platform/single-container/watchdog.sh
@@ -8,29 +8,38 @@ source "${AUTOGPT_ASSET_DIR:-/opt/autogpt/single-container}/common.sh"
readonly SUPERVISOR_PID_FILE="${AUTOGPT_RUNTIME_DIR}/supervisord.pid"
readonly WATCHDOG_ARMED_FILE="${AUTOGPT_RUNTIME_DIR}/watchdog-armed"
readonly FAILURE_LIMIT=3
+readonly INITIAL_CHECK_INTERVAL=1
readonly CHECK_INTERVAL=30
readonly CHECK_TIMEOUT=60
readonly INITIAL_HEALTH_TIMEOUT=600
+CHECK_TIMER_PID=
+FORCED_CHECK_PENDING=false
+CHECK_TRIGGER=scheduled
+WATCHDOG_OUTPUT=
+
main() {
local failures=0
- local output
rm -f "${WATCHDOG_ARMED_FILE}"
- output="$(mktemp "${AUTOGPT_RUNTIME_DIR}/watchdog-health.XXXXXX")"
- trap 'rm -f "${output}" "${WATCHDOG_ARMED_FILE}"' EXIT
+ WATCHDOG_OUTPUT="$(mktemp "${AUTOGPT_RUNTIME_DIR}/watchdog-health.XXXXXX")"
+ trap 'rm -f "${WATCHDOG_OUTPUT}" "${WATCHDOG_ARMED_FILE}"' EXIT
+ trap queue_forced_check USR1
wait_for_ready_file
- wait_for_initial_health "${output}"
+ wait_for_initial_health "${WATCHDOG_OUTPUT}"
while true; do
- sleep "${CHECK_INTERVAL}"
- if run_healthcheck "${output}"; then
+ wait_for_next_check
+ if run_healthcheck "${WATCHDOG_OUTPUT}"; then
failures=0
+ if [[ "${CHECK_TRIGGER}" == forced ]]; then
+ log "watchdog health check passed trigger=forced"
+ fi
continue
fi
((failures += 1))
- log "watchdog health failure ${failures}/${FAILURE_LIMIT}" >&2
- sed -n '1,120p' "${output}" >&2
+ log "watchdog health failure ${failures}/${FAILURE_LIMIT} trigger=${CHECK_TRIGGER}" >&2
+ sed -n '1,120p' "${WATCHDOG_OUTPUT}" >&2
if ((failures >= FAILURE_LIMIT)); then
stop_appliance \
"watchdog stopping the unhealthy appliance; a Docker restart policy is required to restart it automatically"
@@ -38,6 +47,66 @@ main() {
done
}
+queue_forced_check() {
+ # USR1 advances one cadence slot; the normal healthcheck and failure limit
+ # below remain the only path that can stop the appliance.
+ FORCED_CHECK_PENDING=true
+ if [[ -n "${CHECK_TIMER_PID}" ]]; then
+ if ! kill -TERM "${CHECK_TIMER_PID}" 2>/dev/null; then
+ :
+ fi
+ fi
+}
+
+notify_check_timer_started() {
+ :
+}
+
+wait_for_next_check() {
+ local timer_pid
+ local timer_status
+ CHECK_TRIGGER=scheduled
+
+ if [[ "${FORCED_CHECK_PENDING}" == true ]]; then
+ FORCED_CHECK_PENDING=false
+ CHECK_TRIGGER=forced
+ return 0
+ fi
+
+ sleep "${CHECK_INTERVAL}" &
+ timer_pid=$!
+ CHECK_TIMER_PID="${timer_pid}"
+ notify_check_timer_started
+ if [[ "${FORCED_CHECK_PENDING}" == true ]]; then
+ if ! kill -TERM "${timer_pid}" 2>/dev/null; then
+ :
+ fi
+ fi
+ if wait "${timer_pid}"; then
+ timer_status=0
+ else
+ timer_status=$?
+ fi
+ CHECK_TIMER_PID=
+
+ if [[ "${FORCED_CHECK_PENDING}" == true ]]; then
+ FORCED_CHECK_PENDING=false
+ CHECK_TRIGGER=forced
+ if kill -0 "${timer_pid}" 2>/dev/null; then
+ if ! kill -TERM "${timer_pid}" 2>/dev/null; then
+ :
+ fi
+ fi
+ # Reap the timer if the signal interrupted `wait` before TERM completed.
+ if ! wait "${timer_pid}" 2>/dev/null; then
+ :
+ fi
+ return 0
+ fi
+
+ ((timer_status == 0)) || fatal "watchdog check timer failed with status ${timer_status}"
+}
+
run_healthcheck() {
local output="$1"
timeout --signal=TERM --kill-after=5s "${CHECK_TIMEOUT}" \
@@ -55,7 +124,7 @@ wait_for_initial_health() {
fi
log "watchdog waiting for initial healthy state" >&2
sed -n '1,120p' "${output}" >&2
- sleep "${CHECK_INTERVAL}"
+ sleep "${INITIAL_CHECK_INTERVAL}"
done
stop_appliance \
"watchdog did not observe a healthy appliance within ${INITIAL_HEALTH_TIMEOUT}s"
@@ -69,4 +138,6 @@ stop_appliance() {
exit 0
}
-main "$@"
+if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
+ main "$@"
+fi