Skip to content

armgit / AiSOC

Public Imported from GitHub
0

Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.

fix(api): retry transient asyncpg connect failures in the migration runner (#372)

Beenu Arorad3813b92d74e
diff --git a/services/api/app/scripts/run_migrations.py b/services/api/app/scripts/run_migrations.py
index 4d3771001..8244a99bf 100644
--- a/services/api/app/scripts/run_migrations.py
+++ b/services/api/app/scripts/run_migrations.py
@@ -86,8 +86,47 @@ def _asyncpg_dsn(url: str) -> tuple[str, dict]:
async def _connect() -> asyncpg.Connection:
+ """Open a fresh asyncpg connection, retrying on transient connect errors.
+
+ On Fly, the migration ``release_command`` runs the moment a new app VM
+ boots. The Postgres app it talks to may have just woken from
+ autostop / be mid-failover / be reachable but still completing its
+ own boot. asyncpg surfaces these as ``ConnectionDoesNotExistError``,
+ ``ConnectionResetError``, or ``OSError`` raised *inside the initial
+ handshake* — i.e. before the connection is ever usable. A single
+ failed connect would crash the entire deploy, which made the
+ "Deploy API to Fly" workflow look like a code regression when it
+ was really just a cold-start race.
+
+ We retry on the small set of connect-time exceptions that are
+ actually transient. Anything authentication- or schema-related
+ (``InvalidPasswordError``, ``InvalidCatalogNameError``, …) is
+ intentionally **not** retried — those need human attention, not
+ more attempts.
+ """
dsn, kwargs = _asyncpg_dsn(str(settings.DATABASE_URL))
- return await asyncpg.connect(dsn, **kwargs)
+ last_exc: Exception | None = None
+ for attempt in range(1, 7): # ~30 s total at the back-off schedule below
+ try:
+ return await asyncpg.connect(dsn, **kwargs)
+ except (
+ asyncpg.exceptions.ConnectionDoesNotExistError,
+ asyncpg.exceptions.CannotConnectNowError,
+ ConnectionResetError,
+ OSError,
+ ) as exc:
+ last_exc = exc
+ delay = min(2 ** (attempt - 1), 8)
+ logger.warning(
+ "asyncpg.connect failed on attempt %d (%s: %s); retrying in %ds",
+ attempt,
+ type(exc).__name__,
+ exc,
+ delay,
+ )
+ await asyncio.sleep(delay)
+ assert last_exc is not None # for type narrowing
+ raise last_exc
async def _applied(conn: asyncpg.Connection) -> set[str]:
diff --git a/services/api/tests/test_run_migrations_retry.py b/services/api/tests/test_run_migrations_retry.py
new file mode 100644
index 000000000..a3ad8d524
--- /dev/null
+++ b/services/api/tests/test_run_migrations_retry.py
@@ -0,0 +1,91 @@
+"""Tests for ``app.scripts.run_migrations._connect`` retry semantics.
+
+The migration runner is invoked by Fly's ``release_command`` the moment a new
+API VM boots, which races against Postgres becoming reachable (autostop wake,
+mid-failover, etc.). A single failed ``asyncpg.connect`` would crash the whole
+deploy — these tests pin the retry contract that makes the runner resilient
+to those transient connect errors without masking real auth/config failures.
+"""
+
+from __future__ import annotations
+
+from unittest.mock import AsyncMock
+
+import asyncpg
+import pytest
+from app.scripts import run_migrations
+
+
+async def test_connect_retries_until_success(monkeypatch):
+ """Transient connect failures are retried and eventually succeed."""
+ fake_conn = object()
+ sleeps: list[float] = []
+ attempts = [
+ asyncpg.exceptions.ConnectionDoesNotExistError("connection was closed in the middle of operation"),
+ OSError("Connection refused"),
+ fake_conn,
+ ]
+
+ async def fake_connect(*_args, **_kwargs):
+ outcome = attempts.pop(0)
+ if isinstance(outcome, BaseException):
+ raise outcome
+ return outcome
+
+ async def fake_sleep(seconds):
+ sleeps.append(seconds)
+
+ monkeypatch.setattr(run_migrations.asyncpg, "connect", fake_connect)
+ monkeypatch.setattr(run_migrations.asyncio, "sleep", fake_sleep)
+
+ result = await run_migrations._connect()
+
+ assert result is fake_conn
+ assert sleeps == [1, 2], f"expected exponential back-off (1s, 2s) before the third (successful) attempt, got {sleeps}"
+
+
+async def test_connect_gives_up_after_exhausting_retries(monkeypatch):
+ """A persistent transient failure ultimately propagates the last exception."""
+ final_exc = asyncpg.exceptions.ConnectionDoesNotExistError("still down")
+
+ async def always_fail(*_args, **_kwargs):
+ raise final_exc
+
+ async def fake_sleep(_seconds):
+ pass
+
+ monkeypatch.setattr(run_migrations.asyncpg, "connect", always_fail)
+ monkeypatch.setattr(run_migrations.asyncio, "sleep", fake_sleep)
+
+ with pytest.raises(asyncpg.exceptions.ConnectionDoesNotExistError):
+ await run_migrations._connect()
+
+
+async def test_connect_does_not_retry_auth_failures(monkeypatch):
+ """Auth/config errors are surfaced immediately — retrying them is pointless
+ and would mask a real misconfiguration as a deploy hang."""
+
+ async def auth_fail(*_args, **_kwargs):
+ raise asyncpg.exceptions.InvalidPasswordError("wrong password for user")
+
+ call_count = {"connect": 0}
+
+ async def counting_connect(*args, **kwargs):
+ call_count["connect"] += 1
+ await auth_fail(*args, **kwargs)
+
+ monkeypatch.setattr(run_migrations.asyncpg, "connect", counting_connect)
+ # Make sure asyncio.sleep is never called — auth failures shouldn't retry.
+ monkeypatch.setattr(
+ run_migrations.asyncio,
+ "sleep",
+ AsyncMock(side_effect=AssertionError("auth failure must not retry")),
+ )
+
+ with pytest.raises(asyncpg.exceptions.InvalidPasswordError):
+ await run_migrations._connect()
+
+ assert call_count["connect"] == 1, "auth failures must propagate after the first attempt, no retries"