Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

fix(backend): make the test suite runnable — real Redis cluster and bounded teardowns (#14751)

Bentlyc9ee6bfb79d6
diff --git a/autogpt_platform/backend/backend/data/db.py b/autogpt_platform/backend/backend/data/db.py
index 562688baf3..0b75ab5717 100644
--- a/autogpt_platform/backend/backend/data/db.py
+++ b/autogpt_platform/backend/backend/data/db.py
@@ -78,12 +78,22 @@ async def connect():
# raise ConnectionError("Failed to connect to Prisma.") from e
+# Prisma shuts the query engine down by sending it SIGINT and then calling
+# ``subprocess.wait(timeout=...)``. Passing no timeout makes that wait
+# unbounded, so an engine that does not act on the SIGINT wedges the caller
+# forever — and because the wait is a blocking call inside an async function,
+# it takes the event loop with it. The visible symptom is a test session that
+# runs every test and then never prints its summary line. Bound the wait so
+# an unresponsive engine is SIGKILLed instead of hung on.
+DISCONNECT_TIMEOUT = timedelta(seconds=10)
+
+
@conn_retry("Prisma", "Releasing connection")
async def disconnect():
if not prisma.is_connected():
return
- await prisma.disconnect()
+ await prisma.disconnect(DISCONNECT_TIMEOUT)
if prisma.is_connected():
raise ConnectionError("Failed to disconnect from Prisma.")
diff --git a/autogpt_platform/backend/backend/data/db_test.py b/autogpt_platform/backend/backend/data/db_test.py
new file mode 100644
index 0000000000..3fd1419fa0
--- /dev/null
+++ b/autogpt_platform/backend/backend/data/db_test.py
@@ -0,0 +1,47 @@
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+
+from backend.data import db as db_module
+from backend.data.db import DISCONNECT_TIMEOUT, disconnect
+
+
+# The root conftest spins a full test server for every test via an autouse
+# session fixture. These two tests are pure unit tests over a mocked client,
+# so shadow it here the way `backend/util/conftest.py` does for its directory
+# -- narrowed to this module so the rest of `backend/data` is unaffected.
[email protected](scope="session")
+def server():
+ yield None
+
+
[email protected](scope="session", autouse=True)
+def graph_cleanup():
+ yield
+
+
+async def test_disconnect_bounds_the_engine_shutdown(monkeypatch):
+ """Prisma's shutdown wait is unbounded unless a timeout is handed to it.
+
+ The blocking wait sits inside an async function, so an engine that ignores
+ the SIGINT takes the event loop down with it.
+ """
+ prisma = MagicMock()
+ prisma.is_connected.side_effect = [True, False]
+ prisma.disconnect = AsyncMock()
+ monkeypatch.setattr(db_module, "prisma", prisma)
+
+ await disconnect()
+
+ prisma.disconnect.assert_awaited_once_with(DISCONNECT_TIMEOUT)
+
+
+async def test_disconnect_is_a_noop_when_not_connected(monkeypatch):
+ prisma = MagicMock()
+ prisma.is_connected.return_value = False
+ prisma.disconnect = AsyncMock()
+ monkeypatch.setattr(db_module, "prisma", prisma)
+
+ await disconnect()
+
+ prisma.disconnect.assert_not_awaited()
diff --git a/autogpt_platform/backend/backend/util/process.py b/autogpt_platform/backend/backend/util/process.py
index 84ecd13347..1c00249e5e 100644
--- a/autogpt_platform/backend/backend/util/process.py
+++ b/autogpt_platform/backend/backend/util/process.py
@@ -14,6 +14,11 @@ from backend.util.settings import set_service_name
logger = logging.getLogger(__name__)
+# How long ``AppProcess.stop`` waits for a child to act on a signal before
+# escalating. Long enough for an orderly shutdown, short enough that a wedged
+# child cannot hold the parent open indefinitely.
+STOP_TIMEOUT_SECONDS = 30
+
class AppProcess(ABC):
"""
@@ -146,12 +151,26 @@ class AppProcess(ABC):
def stop(self):
"""
Stop the background process.
+
+ Both joins are bounded. An unbounded ``join()`` here wedges whoever is
+ shutting the service down when the child does not act on the SIGTERM —
+ in a test session that is the session fixture's teardown, so the run
+ finishes every test and then never prints its summary line.
"""
if not self.process:
return
+ pid = self.process.pid
self.process.terminate()
- self.process.join()
+ self.process.join(timeout=STOP_TIMEOUT_SECONDS)
+
+ if self.process.is_alive():
+ logger.warning(
+ f"[{self.service_name}] with PID {pid} ignored SIGTERM "
+ f"for {STOP_TIMEOUT_SECONDS}s; killing it"
+ )
+ self.process.kill()
+ self.process.join(timeout=STOP_TIMEOUT_SECONDS)
- logger.info(f"[{self.service_name}] with PID {self.process.pid} stopped")
+ logger.info(f"[{self.service_name}] with PID {pid} stopped")
self.process = None
diff --git a/autogpt_platform/backend/backend/util/process_test.py b/autogpt_platform/backend/backend/util/process_test.py
new file mode 100644
index 0000000000..6f0b710d6f
--- /dev/null
+++ b/autogpt_platform/backend/backend/util/process_test.py
@@ -0,0 +1,123 @@
+import logging
+import multiprocessing
+import signal
+import threading
+import time
+
+from backend.util import process as process_module
+from backend.util.process import STOP_TIMEOUT_SECONDS, AppProcess
+
+
+def _ignore_sigterm_forever(ready):
+ """A child that takes the SIGTERM and does nothing about it."""
+ signal.signal(signal.SIGTERM, signal.SIG_IGN)
+ ready.set()
+ while True:
+ time.sleep(0.05)
+
+
+class _NoopProcess(AppProcess):
+ def run(self):
+ pass
+
+
+class _FakeProcess:
+ """Stands in for ``multiprocessing.Process`` so the joins can be inspected.
+
+ ``alive_after_terminate`` says whether the child acts on the SIGTERM. One
+ that does not must be escalated rather than waited on again.
+ """
+
+ def __init__(self, alive_after_terminate: bool):
+ self.pid = 4242
+ self.joins: list[float | None] = []
+ self.terminated = False
+ self.killed = False
+ self._alive = True
+ self._alive_after_terminate = alive_after_terminate
+
+ def terminate(self):
+ self.terminated = True
+ self._alive = self._alive_after_terminate
+
+ def kill(self):
+ self.killed = True
+ self._alive = False
+
+ def is_alive(self):
+ return self._alive
+
+ def join(self, timeout=None):
+ self.joins.append(timeout)
+
+
+def test_stop_bounds_the_join_on_a_responsive_child():
+ service = _NoopProcess()
+ child = _FakeProcess(alive_after_terminate=False)
+ service.process = child # type: ignore[assignment]
+
+ service.stop()
+
+ assert child.terminated
+ # The timeout is the whole point: an unbounded join here wedges whoever is
+ # shutting the service down.
+ assert child.joins == [STOP_TIMEOUT_SECONDS]
+ assert not child.killed
+ assert service.process is None
+
+
+def test_stop_escalates_to_kill_when_the_child_ignores_the_signal(caplog):
+ service = _NoopProcess()
+ child = _FakeProcess(alive_after_terminate=True)
+ service.process = child # type: ignore[assignment]
+
+ with caplog.at_level(logging.WARNING, logger=process_module.__name__):
+ service.stop()
+
+ assert child.killed
+ assert child.joins == [STOP_TIMEOUT_SECONDS, STOP_TIMEOUT_SECONDS]
+ assert "ignored SIGTERM" in caplog.text
+ assert service.process is None
+
+
+def test_stop_returns_even_when_a_real_child_ignores_sigterm(monkeypatch):
+ """The same guarantee, against a real process and a real signal."""
+ monkeypatch.setattr(process_module, "STOP_TIMEOUT_SECONDS", 2)
+
+ service = _NoopProcess()
+ ready = multiprocessing.Event()
+ child = multiprocessing.Process(
+ target=_ignore_sigterm_forever, args=(ready,), daemon=True
+ )
+ child.start()
+ # Without this the child may simply have died on startup, and every
+ # assertion below would pass for the wrong reason.
+ assert ready.wait(timeout=30), "the child never came up ignoring SIGTERM"
+ service.process = child
+
+ returned = threading.Event()
+
+ def _stop():
+ service.stop()
+ returned.set()
+
+ # `stop` runs on a side thread so an unbounded join fails the assertion
+ # below instead of hanging the whole session -- which is exactly how this
+ # bug presented in the first place.
+ threading.Thread(target=_stop, daemon=True).start()
+
+ try:
+ assert returned.wait(timeout=30), "stop() never returned: the join is unbounded"
+ assert not child.is_alive()
+ assert service.process is None
+ finally:
+ if child.is_alive():
+ child.kill()
+ child.join(timeout=10)
+
+
+def test_stop_is_a_noop_without_a_process():
+ service = _NoopProcess()
+ assert service.process is None
+ service.stop()
+ assert service.process is None
diff --git a/autogpt_platform/backend/backend/util/service_test.py b/autogpt_platform/backend/backend/util/service_test.py
index 7c5509364b..899cdf9009 100644
--- a/autogpt_platform/backend/backend/util/service_test.py
+++ b/autogpt_platform/backend/backend/util/service_test.py
@@ -826,12 +826,16 @@ async def test_service():
service = TestGracefulShutdownService()
service.start(background=True)
- base_url = f"http://localhost:{service.get_port()}"
-
- await wait_until_service_ready(base_url)
- yield service, base_url
-
- service.stop()
+ # `wait_until_service_ready` raises when the port never answers. Without
+ # the `finally`, the already-started child is orphaned: nothing stops it,
+ # and `multiprocessing`'s atexit handler then joins it forever — so the
+ # session prints its summary line and the process still never exits.
+ try:
+ base_url = f"http://localhost:{service.get_port()}"
+ await wait_until_service_ready(base_url)
+ yield service, base_url
+ finally:
+ service.stop()
async def wait_until_service_ready(base_url: str, timeout: float = 10):
diff --git a/autogpt_platform/backend/docker-compose.test.yaml b/autogpt_platform/backend/docker-compose.test.yaml
index de4ffa63bb..b9590ab376 100644
--- a/autogpt_platform/backend/docker-compose.test.yaml
+++ b/autogpt_platform/backend/docker-compose.test.yaml
@@ -36,17 +36,41 @@ services:
file: ../db/docker/docker-compose.yml
service: vector
- redis:
+ # The backend's Redis client is cluster-only (`RedisClient = RedisCluster`),
+ # and every other environment — `backend/.env.default`, the dev
+ # `docker-compose.yml` and backend CI — serves a 3-shard cluster on
+ # 17000/17001/17002. Reuse the dev stack's shards rather than a single-node
+ # `redis:latest` on 6379: no supported client can talk to that, and no
+ # config in the repo points at it.
+ redis-0:
<<: *agpt-services
- image: redis:latest
- command: redis-server --requirepass password
- ports:
- - "6379:6379"
- healthcheck:
- test: ["CMD", "redis-cli", "ping"]
- interval: 10s
- timeout: 5s
- retries: 5
+ extends:
+ file: ../docker-compose.platform.yml
+ service: redis-0
+
+ redis-1:
+ <<: *agpt-services
+ extends:
+ file: ../docker-compose.platform.yml
+ service: redis-1
+
+ redis-2:
+ <<: *agpt-services
+ extends:
+ file: ../docker-compose.platform.yml
+ service: redis-2
+
+ # `extends` deliberately drops `depends_on`, so the shard ordering the dev
+ # stack relies on has to be restated here.
+ redis-init:
+ <<: *agpt-services
+ extends:
+ file: ../docker-compose.platform.yml
+ service: redis-init
+ depends_on:
+ - redis-0
+ - redis-1
+ - redis-2
rabbitmq:
<<: *agpt-services
diff --git a/autogpt_platform/backend/scripts/run_tests.py b/autogpt_platform/backend/scripts/run_tests.py
index 56b7b6b51c..479b37d6e2 100644
--- a/autogpt_platform/backend/scripts/run_tests.py
+++ b/autogpt_platform/backend/scripts/run_tests.py
@@ -3,6 +3,18 @@ import subprocess
import sys
import time
+# How long a single readiness probe may take before it is abandoned.
+# ``max_retries`` bounds the number of attempts, not their duration: without a
+# timeout a wedged ``docker compose exec`` never returns, so the loop never
+# advances and the runner hangs before it can retry or tear the stack down.
+# ``docker compose exec`` has to start the compose CLI, parse the compose file
+# and resolve the container before the probe itself runs, which on a loaded CI
+# box is a few seconds on its own -- so ten seconds sits comfortably above a
+# healthy round trip while staying well inside the budget of either retry loop
+# (30 x 2s for Redis, 5 x 5s for Postgres). A single wedged probe then costs
+# one attempt instead of the whole run.
+PROBE_TIMEOUT_SECONDS = 10
+
def wait_for_postgres(max_retries=5, delay=5):
for _ in range(max_retries):
@@ -26,17 +38,61 @@ def wait_for_postgres(max_retries=5, delay=5):
check=True,
capture_output=True,
text=True,
+ timeout=PROBE_TIMEOUT_SECONDS,
)
if "accepting connections" in result.stdout:
print("PostgreSQL is ready.")
return True
- except subprocess.CalledProcessError:
+ except (subprocess.CalledProcessError, subprocess.TimeoutExpired):
print(f"PostgreSQL is not ready yet. Retrying in {delay} seconds...")
time.sleep(delay)
print("Failed to connect to PostgreSQL.")
return False
+def wait_for_redis_cluster(max_retries=30, delay=2):
+ """Block until the 3-shard cluster has finished forming.
+
+ ``redis-init`` creates the cluster asynchronously after the shards come
+ up. Until ``cluster_state`` is ``ok`` the backend's ``RedisCluster``
+ client cannot resolve slots, and its connection retry backs off for tens
+ of minutes rather than failing — so a test session started too early
+ looks like a hang, not like a race.
+ """
+ for _ in range(max_retries):
+ try:
+ result = subprocess.run(
+ [
+ "docker",
+ "compose",
+ "-f",
+ "docker-compose.test.yaml",
+ "--env-file",
+ "../.env",
+ "exec",
+ "redis-0",
+ "redis-cli",
+ "-p",
+ "17000",
+ "cluster",
+ "info",
+ ],
+ check=False, # readiness is read from stdout, not from the exit code
+ capture_output=True,
+ text=True,
+ timeout=PROBE_TIMEOUT_SECONDS,
+ )
+ if "cluster_state:ok" in result.stdout:
+ print("Redis cluster is ready.")
+ return True
+ except subprocess.TimeoutExpired:
+ print(f"Redis cluster probe timed out after {PROBE_TIMEOUT_SECONDS}s.")
+ print(f"Redis cluster is not ready yet. Retrying in {delay} seconds...")
+ time.sleep(delay)
+ print("Failed to form the Redis cluster.")
+ return False
+
+
def run_command(command, check=True):
try:
subprocess.run(command, check=check)
@@ -60,7 +116,7 @@ def test():
]
)
- if not wait_for_postgres():
+ if not wait_for_postgres() or not wait_for_redis_cluster():
run_command(["docker", "compose", "-f", "docker-compose.test.yaml", "down"])
sys.exit(1)