Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

fix(platform): shut the single container down inside Docker's stock timeout (#14077)

Nicholas Tindle8c65f748b1ea
diff --git a/.github/scripts/platform-single-container-smoke.sh b/.github/scripts/platform-single-container-smoke.sh
index 1a9bbd71f8..70ac881e2b 100644
--- a/.github/scripts/platform-single-container-smoke.sh
+++ b/.github/scripts/platform-single-container-smoke.sh
@@ -7,6 +7,20 @@ set -Eeuo pipefail
readonly PUBLIC_URL=http://localhost:3300
readonly EXPECTED_CODEX_TEMP_ROOT=/dev/shm/autogpt-codex
+# Unraid ships Docker's stock stop timeout and operators must not have to
+# raise it host-wide to run the appliance, so shutdown has to finish inside
+# it. Stopping with a generous timeout would pass here while a default host
+# SIGKILLs the container at 10s (exit 137) with the data stores still up.
+readonly STOCK_DOCKER_STOP_TIMEOUT=10
+# Exit code alone is a cliff: it only changes at the timeout, so a shutdown
+# drifting from 6s to 9.9s passes identically to a fast one until it tips over
+# to 137 on a slower host. Gate on the margin too, so erosion is caught here
+# rather than by an operator. This is the same ceiling the unit test applies --
+# DOCKER_STOP_TIMEOUT_SECONDS minus SHUTDOWN_MARGIN_SECONDS in
+# single-container/tests/test_supervisor_config.py -- derived here rather than
+# written down twice with different values.
+readonly SHUTDOWN_MARGIN_SECONDS=1
+readonly MAX_CLEAN_STOP_SECONDS=$((STOCK_DOCKER_STOP_TIMEOUT - SHUTDOWN_MARGIN_SECONDS))
readonly TIMEOUT_SECONDS="${SMOKE_TIMEOUT_SECONDS:-2700}"
readonly SAFE_PLATFORM="${SMOKE_PLATFORM//\//-}"
readonly RUN_TOKEN="${GITHUB_RUN_ID:-local}-${GITHUB_RUN_ATTEMPT:-0}-${SAFE_PLATFORM}-${RANDOM}"
@@ -36,7 +50,7 @@ cleanup() {
fi
for container in "${CONTAINER_NAME}" "${NEGATIVE_CONTAINER_NAME}"; do
if docker container inspect "${container}" >/dev/null 2>&1; then
- docker stop --timeout 360 "${container}" >/dev/null 2>&1 || true
+ docker stop --timeout "${STOCK_DOCKER_STOP_TIMEOUT}" "${container}" >/dev/null 2>&1 || true
docker rm --force --volumes "${container}" >/dev/null 2>&1 || true
fi
done
@@ -47,6 +61,28 @@ cleanup() {
exit "${result}"
}
+assert_clean_stop() {
+ local reason="$1"
+ local started elapsed exit_code
+ # Integer SECONDS truncates at both ends, so a real 8.9s stop reads as 8.
+ started="${EPOCHREALTIME}"
+ docker stop --timeout "${STOCK_DOCKER_STOP_TIMEOUT}" "${CONTAINER_NAME}" >/dev/null
+ elapsed="$(awk -v a="${started}" -v b="${EPOCHREALTIME}" 'BEGIN { printf "%.2f", b - a }')"
+ exit_code="$(docker inspect --format '{{.State.ExitCode}}' "${CONTAINER_NAME}")"
+ [[ "${exit_code}" == 0 ]] || {
+ echo "container did not exit cleanly ${reason}: exit ${exit_code} after" \
+ "${elapsed}s against the stock ${STOCK_DOCKER_STOP_TIMEOUT}s timeout" >&2
+ return 1
+ }
+ awk -v e="${elapsed}" -v m="${MAX_CLEAN_STOP_SECONDS}" 'BEGIN { exit !(e <= m) }' || {
+ echo "container exited cleanly ${reason} but took ${elapsed}s, over the" \
+ "${MAX_CLEAN_STOP_SECONDS}s budget, close to the stock" \
+ "${STOCK_DOCKER_STOP_TIMEOUT}s timeout" >&2
+ return 1
+ }
+ echo "clean stop ${reason} in ${elapsed}s"
+}
+
trap cleanup EXIT
trap 'exit 130' INT
trap 'exit 143' TERM
@@ -175,7 +211,7 @@ assert_pinned_topology_environment() {
local process_env
pid="$(
docker exec "${CONTAINER_NAME}" \
- supervisorctl -c /opt/autogpt/single-container/supervisor/supervisord.conf pid rest
+ supervisorctl -c /opt/autogpt/single-container/supervisor/supervisord.conf pid runtime:rest
)"
[[ "${pid}" =~ ^[0-9]+$ ]]
# Docker Desktop can deny cross-UID /proc reads even to container root.
@@ -209,17 +245,17 @@ assert_frontend_database_isolation() {
next_pid="$(
docker exec "${CONTAINER_NAME}" \
supervisorctl -c /opt/autogpt/single-container/supervisor/supervisord.conf \
- pid next
+ pid runtime:next
)"
rest_pid="$(
docker exec "${CONTAINER_NAME}" \
supervisorctl -c /opt/autogpt/single-container/supervisor/supervisord.conf \
- pid rest
+ pid runtime:rest
)"
nginx_pid="$(
docker exec "${CONTAINER_NAME}" \
supervisorctl -c /opt/autogpt/single-container/supervisor/supervisord.conf \
- pid nginx
+ pid runtime:nginx
)"
[[ "${next_pid}" =~ ^[0-9]+$ && "${rest_pid}" =~ ^[0-9]+$ && \
"${nginx_pid}" =~ ^[0-9]+$ ]]
@@ -518,7 +554,7 @@ assert_codex_runtime_contract() {
return 1
}
- for process_name in rest executor copilot-executor; do
+ for process_name in runtime:rest runtime:executor runtime:copilot-executor; do
process_id="$(
docker exec "${CONTAINER_NAME}" \
supervisorctl \
@@ -756,7 +792,7 @@ wait_for_falkordb_restart() {
current_pid="$(
docker exec "${CONTAINER_NAME}" \
supervisorctl -c /opt/autogpt/single-container/supervisor/supervisord.conf \
- pid falkordb 2>/dev/null || true
+ pid state:falkordb 2>/dev/null || true
)"
if [[ "${current_pid}" =~ ^[0-9]+$ ]] &&
[[ "${current_pid}" != "${previous_pid}" ]] &&
@@ -868,11 +904,7 @@ first_hash="$(runtime_config_hash)"
assert_runtime_config_mode
assert_frontend_database_isolation
-docker stop --timeout 360 "${CONTAINER_NAME}" >/dev/null
-[[ "$(docker inspect --format '{{.State.ExitCode}}' "${CONTAINER_NAME}")" == 0 ]] || {
- echo "container did not exit cleanly after docker stop" >&2
- exit 1
-}
+assert_clean_stop "after docker stop"
docker rm "${CONTAINER_NAME}" >/dev/null
# A persistent user config must not be able to move internal listeners or bind
@@ -914,7 +946,7 @@ assert_request_tokens_absent_from_logs
falkordb_pid="$(
docker exec "${CONTAINER_NAME}" \
supervisorctl -c /opt/autogpt/single-container/supervisor/supervisord.conf \
- pid falkordb
+ pid state:falkordb
)"
[[ "${falkordb_pid}" =~ ^[0-9]+$ ]]
falkordb_restart_count="$(
@@ -934,7 +966,7 @@ restart_count="$(docker inspect --format '{{.RestartCount}}' "${CONTAINER_NAME}"
docker exec "${CONTAINER_NAME}" \
supervisorctl \
-c /opt/autogpt/single-container/supervisor/supervisord.conf \
- stop nginx >/dev/null
+ stop runtime:nginx >/dev/null
wait_for_automatic_restart "$((restart_count + 1))"
[[ "$(runtime_config_hash)" == "${first_hash}" ]] || {
echo "runtime config changed across automatic Docker restart" >&2
@@ -946,11 +978,7 @@ assert_redirect "${PUBLIC_URL}/copilot" --resolve localhost:3300:127.0.0.1
assert_falkordb_binary_contract
assert_memory_contract cleanup
-docker stop --timeout 360 "${CONTAINER_NAME}" >/dev/null
-[[ "$(docker inspect --format '{{.State.ExitCode}}' "${CONTAINER_NAME}")" == 0 ]] || {
- echo "container did not exit cleanly after the restart test" >&2
- exit 1
-}
+assert_clean_stop "after the restart test"
docker rm "${CONTAINER_NAME}" >/dev/null
echo "single-container smoke test passed for ${SMOKE_PLATFORM}"
diff --git a/autogpt_platform/backend/backend/util/feature_flag.py b/autogpt_platform/backend/backend/util/feature_flag.py
index 5309b3a04b..152232d11f 100644
--- a/autogpt_platform/backend/backend/util/feature_flag.py
+++ b/autogpt_platform/backend/backend/util/feature_flag.py
@@ -25,6 +25,7 @@ P = ParamSpec("P")
T = TypeVar("T")
_is_initialized = False
+_init_attempted = False
class Flag(str, Enum):
@@ -176,12 +177,19 @@ def is_configured() -> bool:
def get_client() -> LDClient:
"""Get the LaunchDarkly client singleton."""
- if not _is_initialized:
+ # Gate on "did we try" rather than "did it work". Without a key
+ # `_is_initialized` never becomes True, so gating on it re-entered
+ # `initialize_launchdarkly` on every flag evaluation -- a warning and a
+ # raise per call on the unconfigured deployments this appliance ships as.
+ if not _init_attempted:
initialize_launchdarkly()
return ldclient.get()
def initialize_launchdarkly() -> None:
+ global _init_attempted
+ _init_attempted = True
+
sdk_key = settings.secrets.launch_darkly_sdk_key
logger.debug(
f"Initializing LaunchDarkly with SDK key: {'present' if sdk_key else 'missing'}"
@@ -194,9 +202,15 @@ def initialize_launchdarkly() -> None:
config = Config(sdk_key)
ldclient.set_config(config)
+ # Read the client before recording that one exists: if constructing it
+ # raised, `shutdown_launchdarkly` would otherwise build a fresh one purely
+ # to close it. Being unreachable is not a construction failure -- that
+ # returns an uninitialized client rather than raising -- so this only
+ # covers the abnormal case.
global _is_initialized
+ connected = ldclient.get().is_initialized()
_is_initialized = True
- if ldclient.get().is_initialized():
+ if connected:
logger.info("LaunchDarkly client initialized successfully")
else:
logger.error("LaunchDarkly client failed to initialize")
@@ -204,9 +218,29 @@ def initialize_launchdarkly() -> None:
def shutdown_launchdarkly() -> None:
"""Shutdown the LaunchDarkly client."""
- if ldclient.get().is_initialized():
- ldclient.get().close()
- logger.info("LaunchDarkly client closed successfully")
+ if not _is_initialized:
+ # `initialize_launchdarkly` returns early when no SDK key is configured,
+ # so `ldclient.set_config` was never called and `ldclient.get()` would
+ # raise "set_config was not called". Callers pair init/shutdown on
+ # app_env alone (see `rest_api.launch_darkly_context` and
+ # `scheduler._shutdown_launchdarkly_for_scheduler`), so an unconfigured
+ # non-LOCAL deployment would otherwise raise out of service teardown and
+ # leave the process alive instead of exiting.
+ return
+
+ # Close whenever a client was constructed, not just when it connected, so
+ # buffered events are flushed and sockets are torn down in order. This is
+ # not about keeping the process alive: every SDK thread is a daemon, so
+ # they never blocked interpreter exit -- it was the escaping exception that
+ # did. Note `close()` flushes synchronously, so an unreachable
+ # LaunchDarkly can make it wait on the SDK's HTTP timeouts.
+ #
+ # `_is_initialized` is deliberately left set: `get_client` reads a false
+ # value as "never started", and clearing it here would let a flag
+ # evaluation arriving during shutdown -- an in-flight request served through
+ # FastAPI's lifespan teardown -- rebuild the client we just closed.
+ ldclient.get().close()
+ logger.info("LaunchDarkly client closed successfully")
async def _fetch_user_context_data(user_id: str) -> Context:
diff --git a/autogpt_platform/backend/backend/util/feature_flag_test.py b/autogpt_platform/backend/backend/util/feature_flag_test.py
index 6740ba8d72..334e487779 100644
--- a/autogpt_platform/backend/backend/util/feature_flag_test.py
+++ b/autogpt_platform/backend/backend/util/feature_flag_test.py
@@ -6,20 +6,23 @@ import pytest
from fastapi import HTTPException
from ldclient import LDClient
+import backend.util.feature_flag as feature_flag_module
from backend.util.feature_flag import (
Flag,
_env_flag_override,
_fetch_user_context_data,
feature_flag,
+ get_client,
is_feature_enabled,
mock_flag_variation,
+ shutdown_launchdarkly,
)
@pytest.fixture
def ld_client(mocker):
client = mocker.Mock(spec=LDClient)
- mocker.patch("ldclient.get", return_value=client)
+ mocker.patch("backend.util.feature_flag.ldclient.get", return_value=client)
client.is_initialized.return_value = True
return client
@@ -411,3 +414,88 @@ class TestUserContextCacheDegradation:
assert ctx.anonymous is True
accessor.assert_not_called()
+
+
+class TestShutdown:
+ @pytest.fixture(autouse=True)
+ def reset_module_state(self):
+ initialized = feature_flag_module._is_initialized
+ attempted = feature_flag_module._init_attempted
+ yield
+ feature_flag_module._is_initialized = initialized
+ feature_flag_module._init_attempted = attempted
+
+ @pytest.fixture
+ def sdk_key(self, mocker):
+ return mocker.patch.object(
+ feature_flag_module.settings.secrets,
+ "launch_darkly_sdk_key",
+ "sdk-key",
+ )
+
+ def test_shutdown_is_a_noop_when_never_initialized(self, mocker):
+ # `initialize_launchdarkly` returns early when no SDK key is set, so
+ # `ldclient.set_config` was never called and `ldclient.get()` raises
+ # "set_config was not called". Callers pair init/shutdown on app_env
+ # alone, so this ran on every unconfigured non-LOCAL deployment and
+ # took the exception out through service teardown, leaving the process
+ # alive until it was killed.
+ feature_flag_module._is_initialized = False
+ get_ldclient = mocker.patch("backend.util.feature_flag.ldclient.get")
+
+ shutdown_launchdarkly()
+
+ get_ldclient.assert_not_called()
+
+ def test_shutdown_closes_an_initialized_client(self, ld_client):
+ feature_flag_module._is_initialized = True
+
+ shutdown_launchdarkly()
+
+ ld_client.close.assert_called_once()
+
+ def test_shutdown_closes_a_client_that_never_connected(self, ld_client):
+ # A configured client that never reached LaunchDarkly still has
+ # streaming and event threads running. Skipping close() there leaves
+ # exactly the kind of live thread that holds a process open past its
+ # stop deadline.
+ feature_flag_module._is_initialized = True
+ ld_client.is_initialized.return_value = False
+
+ shutdown_launchdarkly()
+
+ ld_client.close.assert_called_once()
+
+ def test_shutdown_does_not_rearm_lazy_initialization(
+ self, mocker, ld_client, sdk_key
+ ):
+ # `_is_initialized` is never cleared, which is what keeps a flag
+ # evaluation arriving after teardown from rebuilding the client. This
+ # pins that property; it is not a regression test for the gate swap.
+ feature_flag_module._is_initialized = True
+ feature_flag_module._init_attempted = True
+ set_config = mocker.patch("backend.util.feature_flag.ldclient.set_config")
+
+ shutdown_launchdarkly()
+ get_client()
+
+ set_config.assert_not_called()
+
+ def test_unconfigured_deployment_only_attempts_initialization_once(
+ self, mocker, ld_client
+ ):
+ # Without a key `_is_initialized` never becomes True, so gating the
+ # lazy init on it re-entered initialize_launchdarkly on every flag
+ # evaluation: a warning plus a raise per call, several per request.
+ feature_flag_module._is_initialized = False
+ feature_flag_module._init_attempted = False
+ mocker.patch.object(
+ feature_flag_module.settings.secrets, "launch_darkly_sdk_key", ""
+ )
+ warn = mocker.patch.object(feature_flag_module.logger, "warning")
+
+ get_client()
+ get_client()
+ get_client()
+
+ assert warn.call_count == 1
diff --git a/autogpt_platform/single-container/README.md b/autogpt_platform/single-container/README.md
index bea98c23f5..f7c614b7d9 100644
--- a/autogpt_platform/single-container/README.md
+++ b/autogpt_platform/single-container/README.md
@@ -21,7 +21,10 @@ docker run -d \
significantgravitas/autogpt:latest
```
-The first boot can take several minutes. Wait until Docker reports the container
+The first boot can take several minutes, and it applies the database migrations.
+Let it finish: stopping the container during that window can interrupt a
+migration, and the next boot will refuse to start until you resolve it (the log
+names the migration and the command). Wait until Docker reports the container
as `healthy`, then open [http://localhost:3000](http://localhost:3000).
Registration starts open; the loopback-only port binding above keeps it local.
If you expose the app to a network, anyone who can reach it can register until
@@ -37,6 +40,21 @@ Then recreate the container with `AUTH_ALLOW_NEW_ACCOUNTS=false` to close
registration. Keep the same `autogpt-data` volume so accounts, agents, memory,
and generated application secrets survive container replacement.
+## Stopping
+
+`docker stop` completes in well under Docker's stock 10-second timeout, so the
+appliance shuts down cleanly on a default host and needs no change to Docker's
+host-wide stop timeout. The bundled PostgreSQL, RabbitMQ, Valkey and FalkorDB
+are drained last and in full; everything stateless is stopped first, and is
+terminated rather than waited on once it has released what it holds.
+
+Agent runs that are still executing do not survive the stop. They are not
+resumed on the next boot, so re-run them.
+
+Supervisor's process names are group-qualified inside the container -- use
+`supervisorctl status` to see them (`runtime:rest`, `state:postgres`, and so
+on) rather than assuming a bare name.
+
## Configuration
`AUTOGPT_PUBLIC_URL` must exactly match the URL used in the browser. For
diff --git a/autogpt_platform/single-container/bootstrap.sh b/autogpt_platform/single-container/bootstrap.sh
index ac343d05a4..fea8d84752 100755
--- a/autogpt_platform/single-container/bootstrap.sh
+++ b/autogpt_platform/single-container/bootstrap.sh
@@ -115,6 +115,8 @@ migrate_database() {
--set=ON_ERROR_STOP=1 \
--file="${INIT_SQL}" >/dev/null
+ report_interrupted_migration
+
log "applying Prisma migrations"
(
cd "${AUTOGPT_BACKEND_DIR}"
@@ -122,6 +124,52 @@ migrate_database() {
)
}
+query_scalar() {
+ PGPASSWORD="${POSTGRES_PASSWORD}" "${POSTGRES_BINDIR}/psql" --host=127.0.0.1 --port=5432 --username=postgres --dbname=postgres --set=ON_ERROR_STOP=1 --tuples-only --no-align --command="$1"
+}
+
+# Prisma records a migration before applying it and completes the row
+# afterwards, so a container stopped during its first boot -- while
+# `prisma migrate deploy` is still running -- can leave `finished_at` NULL.
+# Every later boot then fails with Prisma's own "migration is in a failed
+# state" trace, which says nothing about how the appliance got there or what to
+# do next. Name the migration and the fix instead.
+#
+# Deliberately not resolved automatically: the interrupted migration may have
+# applied part of its DDL, and marking it rolled back would skip the remainder
+# on the next deploy, leaving a schema that matches neither state.
+report_interrupted_migration() {
+ local table_exists unfinished
+ table_exists="$(query_scalar \
+ "SELECT to_regclass('platform._prisma_migrations') IS NOT NULL")"
+ [[ "${table_exists}" == t ]] || return 0
+
+ unfinished="$(query_scalar "
+ SELECT coalesce(string_agg(migration_name, ', ' ORDER BY started_at), '')
+ FROM platform._prisma_migrations
+ WHERE finished_at IS NULL AND rolled_back_at IS NULL")"
+ [[ -n "${unfinished}" ]] || return 0
+
+ log "a previous database migration did not finish: ${unfinished}"
+ log "This usually means the container was stopped while it was still starting"
+ log "for the first time."
+ log ""
+ log "If that was the first boot, no data exists yet: delete this container's"
+ log "/data volume (on Unraid, its appdata directory) and start it again."
+ log ""
+ log "If the instance already holds data, restore it from a backup taken"
+ log "before the update."
+ log ""
+ log "The migration can also be marked resolved by hand, but note that this"
+ log "container will not stay up to run the command: it needs PostgreSQL"
+ log "running against ${PGDATA:-/data/postgres} some other way. Determine"
+ log "whether that migration's changes reached the database first, because the"
+ log "two answers are not interchangeable:"
+ log " prisma migrate resolve --rolled-back ${unfinished} # changes absent"
+ log " prisma migrate resolve --applied ${unfinished} # changes present"
+ fatal "refusing to migrate over an interrupted migration"
+}
+
configure_frontend_database_role() {
log "configuring least-privilege frontend database role"
PGPASSWORD="${POSTGRES_PASSWORD}" \
diff --git a/autogpt_platform/single-container/entrypoint.sh b/autogpt_platform/single-container/entrypoint.sh
index 11260d5492..3f48fc7be3 100755
--- a/autogpt_platform/single-container/entrypoint.sh
+++ b/autogpt_platform/single-container/entrypoint.sh
@@ -135,6 +135,15 @@ configure_environment() {
# The appliance bundles no antivirus daemon. Force the scanner off so uploads
# short-circuit as clean instead of failing on an unreachable ClamAV service.
export CLAMAV_SERVICE_ENABLED=false
+ # mem0 and graphiti-core ship their own PostHog keys and report anonymous
+ # usage to their vendors by default. A self-hosted appliance must not phone
+ # home to third parties the operator never chose, so opt both out.
+ #
+ # This suppresses the events, not the client. mem0 constructs its PostHog
+ # client at import and only sets .disabled afterwards, so its atexit join
+ # still costs seconds on every service shutdown regardless of this flag.
+ export MEM0_TELEMETRY=false
+ export GRAPHITI_TELEMETRY_ENABLED=false
export PYRO_HOST=127.0.0.1
export AGENTSERVER_HOST=127.0.0.1 SCHEDULER_HOST=127.0.0.1
diff --git a/autogpt_platform/single-container/healthcheck.sh b/autogpt_platform/single-container/healthcheck.sh
index 62d9c4d5e9..f79ec19233 100755
--- a/autogpt_platform/single-container/healthcheck.sh
+++ b/autogpt_platform/single-container/healthcheck.sh
@@ -18,11 +18,18 @@ main() {
}
check_supervised_processes() {
+ # supervisord.conf assigns every program to an explicit stop tier, so status
+ # lines are group-qualified. Keep this list in sync with those groups.
+ # fatal-exit stays bare: supervisor puts each event listener in a group
+ # named after itself, so it reports as `fatal-exit`, not `fatal-exit:...`.
local programs=(
- fatal-exit postgres valkey-0 valkey-1 valkey-2 rabbitmq falkordb
- database-manager scheduler batch-executor notification executor
- copilot-executor copilot-bot platform-linking-manager websocket rest next nginx
- watchdog
+ fatal-exit
+ state:postgres state:valkey-0 state:valkey-1 state:valkey-2
+ state:rabbitmq state:falkordb
+ runtime:database-manager runtime:scheduler runtime:batch-executor
+ runtime:notification runtime:executor runtime:copilot-executor
+ runtime:copilot-bot runtime:platform-linking-manager runtime:websocket
+ runtime:rest runtime:next runtime:nginx runtime:watchdog
)
local program
local statuses
diff --git a/autogpt_platform/single-container/supervisor/supervisord.conf b/autogpt_platform/single-container/supervisor/supervisord.conf
index c7a402073b..2c50cc764f 100644
--- a/autogpt_platform/single-container/supervisor/supervisord.conf
+++ b/autogpt_platform/single-container/supervisor/supervisord.conf
@@ -17,12 +17,65 @@ supervisor.rpcinterface_factory=supervisor.rpcinterface:make_main_rpcinterface
[supervisorctl]
serverurl=unix:///run/autogpt/supervisor.sock
+# Supervisor shuts process groups down one at a time, in descending priority
+# order, and waits for each group to stop completely before signalling the
+# next. The default layout puts every program in a group of its own, which
+# serialises one stop phase per program, so a single slow process strands every
+# program behind it -- including the databases, which stop last by design.
+# Declaring explicit groups collapses
+# that into three: everything stateless is signalled at once, then the data
+# stores, then the event listener supervisor always groups on its own. Keep
+# these lists in sync with the [program:*] sections below and with
+# healthcheck.sh.
+#
+# The two phases add up, and so does supervisor's own cost: each phase needs at
+# least one more poll iteration of `runforever()` (which polls with timeout=1)
+# to reap what it stopped. Measured against supervisor 4.2.5 with every program
+# ignoring SIGTERM, wall time is sum(stopwaitsecs) + ~1.4s, so the budget below
+# has to leave room for that inside Docker's stock 10s stop timeout -- summing
+# to 8s measured 9.5s, which is not a margin.
+#
+# The split is deliberate, and weighted hard toward the data stores. Measured
+# with a 516MB database under a live write load, PostgreSQL's shutdown
+# checkpoint alone took 3.2s -- of which 2.4s was fsync, the part that scales
+# with the disk rather than the CPU. Everything the stateless tier would do with
+# more time is stateless, so the budget goes where durability is. The shipped
+# budget is 1s runtime + 5s state + 1s listener = 7s, measuring ~8.4s of wall
+# time against Docker's stock 10s. The stateless tier gets a firm
+# cap: its services finish their own cleanup() in milliseconds and then sit in
+# third-party telemetry teardown (a PostHog consumer join inside mem0) for
+# several seconds, so no affordable cap lets them exit on their own and waiting
+# longer only spends budget the data stores need.
+#
+# That cap is not free for all of them. The executors consume run messages with
+# auto_ack=False and release their cluster locks at the end of a cleanup() that
+# polls on a longer interval than this cap allows, so a stop with work in flight
+# leaves the cluster lock held until it expires. On the next boot the new
+# executor has a different id, sees a foreign lock owner and rejects the
+# message without requeue -- and that queue has no dead-letter exchange, so the
+# message is dropped and its execution row is left RUNNING. Not a regression
+# (Docker's SIGKILL had the same outcome), and bounding the stop is still the
+# right trade, but the cost lands on in-flight runs, not on nothing.
+#
+# Program `priority` no longer orders anything within a tier -- ProcessGroup
+# .stop_all() signals every member in one pass -- so nginx is signalled with the
+# services behind it rather than draining first. Restoring that would cost a
+# fourth phase, which measured 10.3s worst case and does not fit.
+[group:runtime]
+programs=bootstrap,database-manager,scheduler,batch-executor,notification,executor,copilot-executor,copilot-bot,platform-linking-manager,websocket,rest,next,nginx,watchdog
+priority=30
+
+[group:state]
+programs=postgres,valkey-0,valkey-1,valkey-2,rabbitmq,falkordb
+priority=10
+
[eventlistener:fatal-exit]
command=/usr/bin/env -i PATH=/usr/bin:/bin /app/autogpt_platform/backend/.venv/bin/python /opt/autogpt/single-container/fatal_listener.py
user=root
events=PROCESS_STATE_FATAL,PROCESS_STATE_EXITED
buffer_size=10
priority=1
+stopwaitsecs=1
startsecs=1
startretries=3
autorestart=true
@@ -35,11 +88,12 @@ stderr_logfile_maxbytes=0
command=/usr/bin/env -i PATH=/usr/lib/postgresql/15/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin LANG=C.UTF-8 PGDATA=/data/postgres /opt/autogpt/single-container/run-service.sh postgres
user=postgres
priority=10
+stopsignal=INT
startsecs=5
autorestart=unexpected
-stopasgroup=true
+stopasgroup=false
killasgroup=true
-stopwaitsecs=90
+stopwaitsecs=5
redirect_stderr=true
stdout_logfile=/dev/fd/1
stdout_logfile_maxbytes=0
@@ -48,6 +102,7 @@ stdout_logfile_maxbytes=0
command=/usr/bin/env -i PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin LANG=C.UTF-8 /opt/autogpt/single-container/run-service.sh valkey-0
user=autogpt-valkey
priority=10
+stopwaitsecs=5
startsecs=3
autorestart=unexpected
stopasgroup=true
@@ -60,6 +115,7 @@ stdout_logfile_maxbytes=0
command=/usr/bin/env -i PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin LANG=C.UTF-8 /opt/autogpt/single-container/run-service.sh valkey-1
user=autogpt-valkey
priority=10
+stopwaitsecs=5
startsecs=3
autorestart=unexpected
stopasgroup=true
@@ -72,6 +128,7 @@ stdout_logfile_maxbytes=0
command=/usr/bin/env -i PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin LANG=C.UTF-8 /opt/autogpt/single-container/run-service.sh valkey-2
user=autogpt-valkey
priority=10
+stopwaitsecs=5
startsecs=3
autorestart=unexpected
stopasgroup=true
@@ -88,7 +145,7 @@ startsecs=10
autorestart=unexpected
stopasgroup=true
killasgroup=true
-stopwaitsecs=120
+stopwaitsecs=5
redirect_stderr=true
stdout_logfile=/dev/fd/1
stdout_logfile_maxbytes=0
@@ -101,7 +158,7 @@ startsecs=5
autorestart=true
stopasgroup=true
killasgroup=true
-stopwaitsecs=60
+stopwaitsecs=5
redirect_stderr=true
stdout_logfile=/dev/fd/1
stdout_logfile_maxbytes=0
@@ -118,7 +175,7 @@ startretries=10
autorestart=unexpected
stopasgroup=true
killasgroup=true
-stopwaitsecs=30
+stopwaitsecs=1
redirect_stderr=true
stdout_logfile=/dev/fd/1
stdout_logfile_maxbytes=0
@@ -132,7 +189,7 @@ startsecs=10
autorestart=unexpected
stopasgroup=true
killasgroup=true
-stopwaitsecs=60
+stopwaitsecs=1
redirect_stderr=true
stdout_logfile=/dev/fd/1
stdout_logfile_maxbytes=0
@@ -146,7 +203,7 @@ startsecs=10
autorestart=unexpected
stopasgroup=true
killasgroup=true
-stopwaitsecs=60
+stopwaitsecs=1
redirect_stderr=true
stdout_logfile=/dev/fd/1
stdout_logfile_maxbytes=0
@@ -160,7 +217,7 @@ startsecs=10
autorestart=unexpected
stopasgroup=true
killasgroup=true
-stopwaitsecs=120
+stopwaitsecs=1
redirect_stderr=true
stdout_logfile=/dev/fd/1
stdout_logfile_maxbytes=0
@@ -174,7 +231,7 @@ startsecs=10
autorestart=unexpected
stopasgroup=true
killasgroup=true
-stopwaitsecs=60
+stopwaitsecs=1
redirect_stderr=true
stdout_logfile=/dev/fd/1
stdout_logfile_maxbytes=0
@@ -188,7 +245,7 @@ startsecs=10
autorestart=unexpected
stopasgroup=true
killasgroup=true
-stopwaitsecs=120
+stopwaitsecs=1
redirect_stderr=true
stdout_logfile=/dev/fd/1
stdout_logfile_maxbytes=0
@@ -202,7 +259,7 @@ startsecs=10
autorestart=unexpected
stopasgroup=true
killasgroup=true
-stopwaitsecs=120
+stopwaitsecs=1
redirect_stderr=true
stdout_logfile=/dev/fd/1
stdout_logfile_maxbytes=0
@@ -216,7 +273,7 @@ startsecs=10
autorestart=unexpected
stopasgroup=true
killasgroup=true
-stopwaitsecs=60
+stopwaitsecs=1
redirect_stderr=true
stdout_logfile=/dev/fd/1
stdout_logfile_maxbytes=0
@@ -230,7 +287,7 @@ startsecs=10
autorestart=unexpected
stopasgroup=true
killasgroup=true
-stopwaitsecs=60
+stopwaitsecs=1
redirect_stderr=true
stdout_logfile=/dev/fd/1
stdout_logfile_maxbytes=0
@@ -244,7 +301,7 @@ startsecs=10
autorestart=unexpected
stopasgroup=true
killasgroup=true
-stopwaitsecs=60
+stopwaitsecs=1
redirect_stderr=true
stdout_logfile=/dev/fd/1
stdout_logfile_maxbytes=0
@@ -258,7 +315,7 @@ startsecs=10
autorestart=unexpected
stopasgroup=true
killasgroup=true
-stopwaitsecs=60
+stopwaitsecs=1
redirect_stderr=true
stdout_logfile=/dev/fd/1
stdout_logfile_maxbytes=0
@@ -272,7 +329,7 @@ startsecs=10
autorestart=unexpected
stopasgroup=true
killasgroup=true
-stopwaitsecs=60
+stopwaitsecs=1
redirect_stderr=true
stdout_logfile=/dev/fd/1
stdout_logfile_maxbytes=0
@@ -281,11 +338,14 @@ stdout_logfile_maxbytes=0
command=/usr/bin/env -i PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin AUTOGPT_HOME=/run/autogpt/nginx/home AUTOGPT_CACHE_DIR=/run/autogpt/nginx/cache AUTOGPT_RUNTIME_DIR=/run/autogpt /opt/autogpt/single-container/run-app.sh nginx nginx -c /opt/autogpt/single-container/nginx/nginx.conf -g "daemon off;"
user=autogpt_proxy
priority=40
+# nginx reads TERM as fast shutdown, which resets connections mid-response.
+# QUIT is its graceful drain, and costs nothing against the cap.
+stopsignal=QUIT
startsecs=5
autorestart=unexpected
stopasgroup=true
killasgroup=true
-stopwaitsecs=30
+stopwaitsecs=1
redirect_stderr=true
stdout_logfile=/dev/fd/1
stdout_logfile_maxbytes=0
@@ -299,7 +359,7 @@ startretries=3
autorestart=unexpected
stopasgroup=true
killasgroup=true
-stopwaitsecs=10
+stopwaitsecs=1
redirect_stderr=true
stdout_logfile=/dev/fd/1
stdout_logfile_maxbytes=0
diff --git a/autogpt_platform/single-container/tests/test_entrypoint.py b/autogpt_platform/single-container/tests/test_entrypoint.py
index 42890e4c0d..fbc991cb97 100644
--- a/autogpt_platform/single-container/tests/test_entrypoint.py
+++ b/autogpt_platform/single-container/tests/test_entrypoint.py
@@ -255,5 +255,16 @@ class ProxyIsolationTest(unittest.TestCase):
self.assertNotIn("user=autogpt\n", nginx_program)
+class ThirdPartyTelemetryTest(unittest.TestCase):
+ def test_entrypoint_exports_the_vendor_telemetry_opt_outs(self) -> None:
+ entrypoint = ENTRYPOINT_PATH.read_text(encoding="utf-8")
+
+ # mem0 and graphiti-core embed their own PostHog write keys and report
+ # to their vendors unless these are set. A self-hosted appliance must
+ # not send anything to a third party the operator never chose.
+ self.assertIn("export MEM0_TELEMETRY=false", entrypoint)
+ self.assertIn("export GRAPHITI_TELEMETRY_ENABLED=false", entrypoint)
+
+
if __name__ == "__main__":
unittest.main()
diff --git a/autogpt_platform/single-container/tests/test_runtime_config.py b/autogpt_platform/single-container/tests/test_runtime_config.py
index 4738be2d75..9e24d25cc9 100644
--- a/autogpt_platform/single-container/tests/test_runtime_config.py
+++ b/autogpt_platform/single-container/tests/test_runtime_config.py
@@ -295,7 +295,7 @@ class PublicUrlTest(unittest.TestCase):
class FatalListenerTest(unittest.TestCase):
def test_fatal_event_acknowledges_before_terminating_supervisor(self) -> None:
- payload = "processname:rest groupname:rest from_state:BACKOFF"
+ payload = "processname:rest groupname:runtime from_state:BACKOFF"
header = f"eventname:PROCESS_STATE_FATAL len:{len(payload)}\n"
input_stream = io.StringIO(payload)
output_stream = io.StringIO()
@@ -313,7 +313,7 @@ class FatalListenerTest(unittest.TestCase):
self.assertEqual(calls, ["RESULT 2\nOK"])
def test_fatal_event_does_not_echo_untrusted_payload(self) -> None:
- payload = "processname:rest\nsecret-value groupname:rest"
+ payload = "processname:rest\nsecret-value groupname:runtime"
header = f"eventname:PROCESS_STATE_FATAL len:{len(payload)}\n"
with mock.patch.object(fatal_listener.sys, "stderr", io.StringIO()) as stderr:
fatal_listener.handle_event(
@@ -327,7 +327,7 @@ class FatalListenerTest(unittest.TestCase):
self.assertIn("unknown", stderr.getvalue())
def test_unexpected_bootstrap_exit_terminates_supervisor(self) -> None:
- payload = "processname:bootstrap groupname:bootstrap expected:0"
+ payload = "processname:bootstrap groupname:runtime expected:0"
output_stream = io.StringIO()
calls: list[str] = []
diff --git a/autogpt_platform/single-container/tests/test_supervisor_config.py b/autogpt_platform/single-container/tests/test_supervisor_config.py
new file mode 100644
index 0000000000..73008c674d
--- /dev/null
+++ b/autogpt_platform/single-container/tests/test_supervisor_config.py
@@ -0,0 +1,205 @@
+from __future__ import annotations
+
+import configparser
+import re
+import unittest
+from pathlib import Path
+
+ASSET_DIR = Path(__file__).resolve().parents[1]
+SUPERVISOR_PATH = ASSET_DIR / "supervisor" / "supervisord.conf"
+HEALTHCHECK_PATH = ASSET_DIR / "healthcheck.sh"
+
+# Unraid ships Docker's stock stop timeout and operators must not have to raise
+# it host-wide to run this appliance, so the whole supervised shutdown has to
+# finish well inside this budget or Docker SIGKILLs the container (exit 137)
+# with the data stores still running.
+DOCKER_STOP_TIMEOUT_SECONDS = 10
+# Supervisor stops one group per phase, and each phase costs more than its
+# stopwaitsecs: `runforever()` polls with timeout=1 and needs at least one more
+# iteration to reap what `ordered_stop_groups_phase_1` just stopped. Measured
+# against supervisor 4.2.5 with every program ignoring SIGTERM, wall time came
+# out at sum(stopwaitsecs) + ~1.4s across 2-, 3- and 4-phase layouts, so charge
+# each phase for that rather than assuming stopwaitsecs is the whole cost.
+# Measured flat, not per phase: 2 phases cost +1.30s, 3 cost +1.38s, 4 cost
+# +1.27s. Charging per phase happens to fit at three and under-charges at two.
+SUPERVISOR_SHUTDOWN_OVERHEAD_SECONDS = 1.5
+SHUTDOWN_MARGIN_SECONDS = 1
+
+# Programs that hold no durable state are signalled together, then the data
+# stores. `fatal-exit` is an event listener; supervisor always places those in
+# their own group, and its priority keeps it alive until everything it reports
+# on has stopped.
+RUNTIME_PROGRAMS = {
+ "bootstrap",
+ "database-manager",
+ "scheduler",
+ "batch-executor",
+ "notification",
+ "executor",
+ "copilot-executor",
+ "copilot-bot",
+ "platform-linking-manager",
+ "websocket",
+ "rest",
+ "next",
+ "nginx",
+ "watchdog",
+}
+STATE_PROGRAMS = {
+ "postgres",
+ "valkey-0",
+ "valkey-1",
+ "valkey-2",
+ "rabbitmq",
+ "falkordb",
+}
+GROUPS = {"runtime": RUNTIME_PROGRAMS, "state": STATE_PROGRAMS}
+EVENT_LISTENERS = {"fatal-exit"}
+
+# One-shot; it has normally already exited, so the healthcheck does not require
+# it to be RUNNING.
+ONE_SHOT_PROGRAMS = {"bootstrap"}
+
+
+def load_supervisor_config() -> configparser.ConfigParser:
+ config = configparser.ConfigParser(interpolation=None)
+ with SUPERVISOR_PATH.open(encoding="utf-8") as config_file:
+ config.read_file(config_file)
+ return config
+
+
+def section_names(config: configparser.ConfigParser, prefix: str) -> set[str]:
+ return {
+ section.removeprefix(f"{prefix}:")
+ for section in config.sections()
+ if section.startswith(f"{prefix}:")
+ }
+
+
+class SupervisorShutdownTierTest(unittest.TestCase):
+ def test_every_program_belongs_to_exactly_one_group(self) -> None:
+ config = load_supervisor_config()
+
+ self.assertEqual(section_names(config, "group"), set(GROUPS))
+ self.assertEqual(
+ section_names(config, "program"),
+ RUNTIME_PROGRAMS | STATE_PROGRAMS,
+ "a program outside both groups becomes its own stop tier",
+ )
+ for group, expected in GROUPS.items():
+ with self.subTest(group=group):
+ declared = {
+ program.strip()
+ for program in config[f"group:{group}"]["programs"].split(",")
+ if program.strip()
+ }
+ self.assertEqual(declared, expected)
+
+ def test_event_listeners_are_accounted_for(self) -> None:
+ config = load_supervisor_config()
+
+ # Supervisor groups each event listener on its own, so an undeclared one
+ # is an extra stop phase the budget never charged for.
+ self.assertEqual(section_names(config, "eventlistener"), EVENT_LISTENERS)
+
+ def test_state_services_stop_after_everything_that_uses_them(self) -> None:
+ config = load_supervisor_config()
+
+ runtime = config["group:runtime"].getint("priority")
+ state = config["group:state"].getint("priority")
+ listener = config["eventlistener:fatal-exit"].getint("priority")
+
+ # Supervisor stops the highest priority group first.
+ self.assertGreater(runtime, state)
+ self.assertGreater(state, listener)
+
+ def test_worst_case_shutdown_fits_inside_the_docker_stop_timeout(self) -> None:
+ config = load_supervisor_config()
+
+ def stop_wait(program: str, section: str) -> int:
+ # Supervisor's built-in default is 10s, which alone exhausts the
+ # budget, so every program must set this explicitly.
+ self.assertIn(
+ "stopwaitsecs",
+ config[f"{section}:{program}"],
+ f"{program} inherits supervisor's 10s default stopwaitsecs",
+ )
+ return config[f"{section}:{program}"].getint("stopwaitsecs")
+
+ # Groups stop one after another, and a group is only done once its
+ # slowest member has stopped, so the tiers add up.
+ waits = [
+ max(stop_wait(program, "program") for program in programs)
+ for programs in GROUPS.values()
+ ]
+ waits += [stop_wait(listener, "eventlistener") for listener in EVENT_LISTENERS]
+ budget = sum(waits) + SUPERVISOR_SHUTDOWN_OVERHEAD_SECONDS
+
+ self.assertLessEqual(
+ budget,
+ DOCKER_STOP_TIMEOUT_SECONDS - SHUTDOWN_MARGIN_SECONDS,
+ f"worst-case supervised shutdown is {sum(waits)}s of stopwaitsecs "
+ f"plus {SUPERVISOR_SHUTDOWN_OVERHEAD_SECONDS}s of supervisor "
+ f"overhead = {budget}s; Docker SIGKILLs the container at "
+ f"{DOCKER_STOP_TIMEOUT_SECONDS}s",
+ )
+
+ def test_state_tier_holds_the_larger_share_of_the_budget(self) -> None:
+ config = load_supervisor_config()
+
+ def wait(program: str) -> int:
+ return config[f"program:{program}"].getint("stopwaitsecs")
+
+ # The sum alone would let the tiers be inverted. PostgreSQL's shutdown
+ # checkpoint measured 3.2s on a seeded database, so the drainable tier
+ # has to keep the larger cap.
+ self.assertGreater(
+ min(wait(program) for program in STATE_PROGRAMS),
+ max(wait(program) for program in RUNTIME_PROGRAMS),
+ )
+
+ def test_postgres_uses_fast_shutdown(self) -> None:
+ config = load_supervisor_config()
+
+ # SIGTERM is PostgreSQL's *smart* shutdown: it waits for every client to
+ # disconnect and so never completes on a deadline. SIGINT is the fast
+ # shutdown - roll back open transactions, checkpoint, exit.
+ self.assertEqual(config["program:postgres"]["stopsignal"], "INT")
+ # run-service.sh execs the postmaster, so supervisor's child *is* the
+ # postmaster. killpg'ing SIGINT would also hit backends, where INT means
+ # cancel-query, racing the postmaster's own orchestrated shutdown.
+ self.assertEqual(config["program:postgres"]["stopasgroup"], "false")
+
+ def test_supervisor_activity_log_reaches_container_logs_once(self) -> None:
+ config = load_supervisor_config()
+
+ # Supervisor's activity log names the program that stalled a shutdown,
+ # so it has to reach `docker logs`. Under nodaemon it already mirrors
+ # that log to stdout, so pointing `logfile` at stdout as well installs a
+ # second handler on the same descriptor and prints every line twice.
+ self.assertTrue(config["supervisord"].getboolean("nodaemon"))
+ self.assertEqual(config["supervisord"]["logfile"], "/dev/null")
+
+
+class HealthcheckSupervisorNamesTest(unittest.TestCase):
+ def test_healthcheck_matches_grouped_program_names(self) -> None:
+ healthcheck = HEALTHCHECK_PATH.read_text(encoding="utf-8")
+ match = re.search(
+ r"local programs=\(\n(?P<programs>.*?)\n \)", healthcheck, re.DOTALL
+ )
+ self.assertIsNotNone(match)
+ assert match is not None
+
+ checked = set(match.group("programs").split())
+ expected = set(EVENT_LISTENERS) | {
+ f"{group}:{program}"
+ for group, programs in GROUPS.items()
+ for program in programs - ONE_SHOT_PROGRAMS
+ }
+ # Grouping renames status lines to `group:program`; an un-updated list
+ # would silently match nothing and pass every program.
+ self.assertEqual(checked, expected)
+
+
+if __name__ == "__main__":
+ unittest.main()