Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

dx(backend): stop parallel test runs clobbering each other's docker stack (#14923)

Nicholas Tindle4383422a885b
diff --git a/autogpt_platform/backend/scripts/run_tests.py b/autogpt_platform/backend/scripts/run_tests.py
index 479b37d6e2..2ac0e58aff 100644
--- a/autogpt_platform/backend/scripts/run_tests.py
+++ b/autogpt_platform/backend/scripts/run_tests.py
@@ -11,12 +11,29 @@ import time
# and resolve the container before the probe itself runs, which on a loaded CI
# box is a few seconds on its own -- so ten seconds sits comfortably above a
# healthy round trip while staying well inside the budget of either retry loop
-# (30 x 2s for Redis, 5 x 5s for Postgres). A single wedged probe then costs
+# (30 x 2s for Redis, 36 x 5s for Postgres). A single wedged probe then costs
# one attempt instead of the whole run.
PROBE_TIMEOUT_SECONDS = 10
+# docker compose names the project after this directory, so every checkout's
+# test stack is the same "backend" project with the same container names.
+# An `up` from another worktree recreates the containers under a test session
+# that is using them, and its `down` removes them. A run waits for another
+# worktree's stack to go away, and only takes down a stack that is its own.
+COMPOSE_PROJECT = "backend"
+BACKEND_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+STACK_WAIT_SECONDS = 30 * 60
-def wait_for_postgres(max_retries=5, delay=5):
+
+def wait_for_postgres(max_retries=36, delay=5):
+ """Block until the `postgres` role can run a query.
+
+ pg_isready isn't enough: on a fresh data directory the Supabase image runs
+ its init scripts against a temporary server that already accepts
+ connections, before the `postgres` role exists. Starting on that, or having
+ the container recreated mid-init, leaves a half-built database that fails
+ every later run with `role "postgres" does not exist`.
+ """
for _ in range(max_retries):
try:
result = subprocess.run(
@@ -28,25 +45,36 @@ def wait_for_postgres(max_retries=5, delay=5):
"--env-file",
"../.env",
"exec",
+ "-T",
"db",
- "pg_isready",
+ "psql",
"-U",
"postgres",
"-d",
"postgres",
+ "-tAc",
+ "select 1",
],
check=True,
capture_output=True,
text=True,
timeout=PROBE_TIMEOUT_SECONDS,
)
- if "accepting connections" in result.stdout:
+ if result.stdout.strip() == "1":
print("PostgreSQL is ready.")
return True
except (subprocess.CalledProcessError, subprocess.TimeoutExpired):
- print(f"PostgreSQL is not ready yet. Retrying in {delay} seconds...")
- time.sleep(delay)
- print("Failed to connect to PostgreSQL.")
+ pass
+ print(f"PostgreSQL is not ready yet. Retrying in {delay} seconds...")
+ time.sleep(delay)
+ print(
+ "Failed to connect to PostgreSQL. If `docker compose -f "
+ "docker-compose.test.yaml logs db` says the `postgres` role does not "
+ "exist, the database's first start was interrupted. Its data "
+ "directory, ../db/docker/volumes/db/data, is also your local dev "
+ "Supabase database: back up anything you need from it, then delete it "
+ "and run the tests again."
+ )
return False
@@ -101,7 +129,84 @@ def run_command(command, check=True):
sys.exit(1)
+def wait_for_stack_to_be_free(max_wait=STACK_WAIT_SECONDS, delay=10):
+ """Wait until no other checkout's test run is using the test stack."""
+ deadline = time.monotonic() + max_wait
+ reported = set()
+ while owners := other_stack_owners():
+ in_use_by = ", ".join(sorted(owners))
+ if time.monotonic() >= deadline:
+ print(
+ f"The test stack is still in use by {in_use_by}. If no test run "
+ "is going there, take its stack down with `docker compose -f "
+ "docker-compose.test.yaml down` from that directory."
+ )
+ return False
+ if owners != reported:
+ print(f"The test stack is in use by {in_use_by}; waiting for that run.")
+ reported = owners
+ time.sleep(delay)
+ return True
+
+
+def tear_down_stack():
+ """Take the test stack down, unless another checkout has taken it over."""
+ if owners := other_stack_owners():
+ print(f"Leaving the test stack up for {', '.join(sorted(owners))}.")
+ return
+ run_command(["docker", "compose", "-f", "docker-compose.test.yaml", "down"])
+
+
+def other_stack_owners():
+ """Directories other than this one whose `docker compose` created the
+ running containers of the test stack."""
+ try:
+ result = subprocess.run(
+ [
+ "docker",
+ "ps",
+ "--filter",
+ f"label=com.docker.compose.project={COMPOSE_PROJECT}",
+ "--format",
+ '{{.Label "com.docker.compose.project.working_dir"}}\t'
+ '{{.Label "com.docker.compose.project.config_files"}}',
+ ],
+ check=False,
+ capture_output=True,
+ text=True,
+ timeout=PROBE_TIMEOUT_SECONDS,
+ )
+ except subprocess.TimeoutExpired:
+ # Docker isn't answering; the compose command that follows will say so.
+ print(f"`docker ps` timed out after {PROBE_TIMEOUT_SECONDS}s.")
+ return set()
+ owners = {
+ _normalize_path(working_dir)
+ for working_dir, _, config_files in (
+ line.partition("\t") for line in result.stdout.splitlines()
+ )
+ if working_dir and _is_test_stack(config_files)
+ }
+ return owners - {_normalize_path(BACKEND_DIR)}
+
+
+def _is_test_stack(config_files):
+ """Any project started from a directory named `backend` is a `backend`
+ project; only one started from docker-compose.test.yaml is a test run."""
+ return any(
+ os.path.basename(path.strip()) == "docker-compose.test.yaml"
+ for path in config_files.split(",")
+ )
+
+
+def _normalize_path(path):
+ return os.path.normcase(os.path.normpath(path.strip()))
+
+
def test():
+ if not wait_for_stack_to_be_free():
+ sys.exit(1)
+
# Start PostgreSQL with Docker Compose
run_command(
[
@@ -117,7 +222,7 @@ def test():
)
if not wait_for_postgres() or not wait_for_redis_cluster():
- run_command(["docker", "compose", "-f", "docker-compose.test.yaml", "down"])
+ tear_down_stack()
sys.exit(1)
# IMPORTANT: Set test database environment variables to prevent accidentally
@@ -207,6 +312,6 @@ def test():
# not any database that might be configured in the developer's environment
result = subprocess.run(["pytest"] + sys.argv[1:], env=test_env, check=False)
- run_command(["docker", "compose", "-f", "docker-compose.test.yaml", "down"])
+ tear_down_stack()
sys.exit(result.returncode)
diff --git a/autogpt_platform/backend/scripts/run_tests_test.py b/autogpt_platform/backend/scripts/run_tests_test.py
new file mode 100644
index 0000000000..f4f09b7e66
--- /dev/null
+++ b/autogpt_platform/backend/scripts/run_tests_test.py
@@ -0,0 +1,110 @@
+"""Tests for how the local test runner shares the docker test stack.
+
+Every checkout's stack is the same compose project, so a run in one worktree
+must not recreate or remove the containers another worktree's run is using.
+"""
+
+import os
+import subprocess
+from unittest.mock import patch
+
+from scripts import run_tests
+
+OTHER_CHECKOUT = os.path.join(os.sep, "code", "other-worktree", "backend")
+
+
+def _completed(stdout: str) -> subprocess.CompletedProcess:
+ return subprocess.CompletedProcess(args=[], returncode=0, stdout=stdout)
+
+
+def _container(working_dir: str, compose_file: str = "docker-compose.test.yaml"):
+ """A `docker ps` line: the container's compose working dir and config files."""
+ return f"{working_dir}\t{os.path.join(working_dir, compose_file)}\n"
+
+
+def test_other_stack_owners_leaves_out_this_checkout():
+ listing = (
+ _container(run_tests.BACKEND_DIR)
+ + _container(OTHER_CHECKOUT)
+ + _container(OTHER_CHECKOUT)
+ )
+ with patch("scripts.run_tests.subprocess.run", return_value=_completed(listing)):
+ owners = run_tests.other_stack_owners()
+
+ assert owners == {os.path.normcase(os.path.normpath(OTHER_CHECKOUT))}
+
+
+def test_other_stack_owners_ignores_other_projects_named_backend():
+ """Compose names a project after its directory, so an unrelated app
+ started from some other `backend` directory is a `backend` project too.
+ It isn't a test run, so it must neither be waited for nor block teardown."""
+ other_app = os.path.join(os.sep, "code", "some-other-app", "backend")
+ listing = _container(other_app, compose_file="docker-compose.yml")
+ with patch("scripts.run_tests.subprocess.run", return_value=_completed(listing)):
+ assert run_tests.other_stack_owners() == set()
+
+
+def test_a_hung_docker_ps_counts_as_no_other_owner():
+ """The compose command that runs next reports a Docker outage better than
+ a traceback from the ownership check would."""
+ with patch(
+ "scripts.run_tests.subprocess.run",
+ side_effect=subprocess.TimeoutExpired("docker", 10),
+ ):
+ assert run_tests.other_stack_owners() == set()
+
+
+def test_waits_until_another_checkouts_run_has_finished():
+ owners = iter([{OTHER_CHECKOUT}, {OTHER_CHECKOUT}, set()])
+ with patch(
+ "scripts.run_tests.other_stack_owners", side_effect=lambda: next(owners)
+ ), patch("scripts.run_tests.time.sleep") as sleep:
+ assert run_tests.wait_for_stack_to_be_free(max_wait=60) is True
+
+ assert sleep.call_count == 2
+
+
+def test_gives_up_while_another_checkout_still_holds_the_stack():
+ with patch(
+ "scripts.run_tests.other_stack_owners", return_value={OTHER_CHECKOUT}
+ ), patch("scripts.run_tests.time.sleep"):
+ assert run_tests.wait_for_stack_to_be_free(max_wait=0) is False
+
+
+def test_tear_down_leaves_a_stack_another_checkout_has_taken_over():
+ with patch(
+ "scripts.run_tests.other_stack_owners", return_value={OTHER_CHECKOUT}
+ ), patch("scripts.run_tests.run_command") as run_command:
+ run_tests.tear_down_stack()
+
+ run_command.assert_not_called()
+
+
+def test_tear_down_takes_its_own_stack_down():
+ with patch("scripts.run_tests.other_stack_owners", return_value=set()), patch(
+ "scripts.run_tests.run_command"
+ ) as run_command:
+ run_tests.tear_down_stack()
+
+ run_command.assert_called_once_with(
+ ["docker", "compose", "-f", "docker-compose.test.yaml", "down"]
+ )
+
+
+def test_postgres_is_ready_only_once_the_postgres_role_answers():
+ """During a fresh data directory's init, the server already accepts
+ connections (pg_isready passes) before the `postgres` role exists."""
+ role_missing = subprocess.CalledProcessError(
+ 2, "psql", stderr='FATAL: role "postgres" does not exist'
+ )
+ with patch(
+ "scripts.run_tests.subprocess.run",
+ side_effect=[role_missing, _completed("1\n")],
+ ) as run, patch("scripts.run_tests.time.sleep"):
+ assert run_tests.wait_for_postgres() is True
+
+ assert run.call_count == 2
+ probe = run.call_args.args[0]
+ assert "psql" in probe
+ # No TTY, so nothing but psql's own output is compared with "1".
+ assert "-T" in probe