Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.
Python agentic-aiai-securityalert-triageblue-teamcybersecuritydetection-engineeringincident-responselanggraphllm-agentsmcpmitre-attackmodel-context-protocolpurple-teamsecurity-operationsself-hostedsiemsoarsocthreat-detectionthreat-intelligence
fix(api): retry transient asyncpg connect failures in the migration runner (#372)
Beenu Arorad3813b92d74e
diff --git a/services/api/app/scripts/run_migrations.py b/services/api/app/scripts/run_migrations.pyindex 4d3771001..8244a99bf 100644--- a/services/api/app/scripts/run_migrations.py+++ b/services/api/app/scripts/run_migrations.py@@ -86,8 +86,47 @@ def _asyncpg_dsn(url: str) -> tuple[str, dict]:async def _connect() -> asyncpg.Connection:+ """Open a fresh asyncpg connection, retrying on transient connect errors.++ On Fly, the migration ``release_command`` runs the moment a new app VM+ boots. The Postgres app it talks to may have just woken from+ autostop / be mid-failover / be reachable but still completing its+ own boot. asyncpg surfaces these as ``ConnectionDoesNotExistError``,+ ``ConnectionResetError``, or ``OSError`` raised *inside the initial+ handshake* — i.e. before the connection is ever usable. A single+ failed connect would crash the entire deploy, which made the+ "Deploy API to Fly" workflow look like a code regression when it+ was really just a cold-start race.++ We retry on the small set of connect-time exceptions that are+ actually transient. Anything authentication- or schema-related+ (``InvalidPasswordError``, ``InvalidCatalogNameError``, …) is+ intentionally **not** retried — those need human attention, not+ more attempts.+ """dsn, kwargs = _asyncpg_dsn(str(settings.DATABASE_URL))- return await asyncpg.connect(dsn, **kwargs)+ last_exc: Exception | None = None+ for attempt in range(1, 7): # ~30 s total at the back-off schedule below+ try:+ return await asyncpg.connect(dsn, **kwargs)+ except (+ asyncpg.exceptions.ConnectionDoesNotExistError,+ asyncpg.exceptions.CannotConnectNowError,+ ConnectionResetError,+ OSError,+ ) as exc:+ last_exc = exc+ delay = min(2 ** (attempt - 1), 8)+ logger.warning(+ "asyncpg.connect failed on attempt %d (%s: %s); retrying in %ds",+ attempt,+ type(exc).__name__,+ exc,+ delay,+ )+ await asyncio.sleep(delay)+ assert last_exc is not None # for type narrowing+ raise last_excasync def _applied(conn: asyncpg.Connection) -> set[str]:diff --git a/services/api/tests/test_run_migrations_retry.py b/services/api/tests/test_run_migrations_retry.pynew file mode 100644index 000000000..a3ad8d524--- /dev/null+++ b/services/api/tests/test_run_migrations_retry.py@@ -0,0 +1,91 @@+"""Tests for ``app.scripts.run_migrations._connect`` retry semantics.++The migration runner is invoked by Fly's ``release_command`` the moment a new+API VM boots, which races against Postgres becoming reachable (autostop wake,+mid-failover, etc.). A single failed ``asyncpg.connect`` would crash the whole+deploy — these tests pin the retry contract that makes the runner resilient+to those transient connect errors without masking real auth/config failures.+"""++from __future__ import annotations++from unittest.mock import AsyncMock++import asyncpg+import pytest+from app.scripts import run_migrations+++async def test_connect_retries_until_success(monkeypatch):+ """Transient connect failures are retried and eventually succeed."""+ fake_conn = object()+ sleeps: list[float] = []+ attempts = [+ asyncpg.exceptions.ConnectionDoesNotExistError("connection was closed in the middle of operation"),+ OSError("Connection refused"),+ fake_conn,+ ]++ async def fake_connect(*_args, **_kwargs):+ outcome = attempts.pop(0)+ if isinstance(outcome, BaseException):+ raise outcome+ return outcome++ async def fake_sleep(seconds):+ sleeps.append(seconds)++ monkeypatch.setattr(run_migrations.asyncpg, "connect", fake_connect)+ monkeypatch.setattr(run_migrations.asyncio, "sleep", fake_sleep)++ result = await run_migrations._connect()++ assert result is fake_conn+ assert sleeps == [1, 2], f"expected exponential back-off (1s, 2s) before the third (successful) attempt, got {sleeps}"+++async def test_connect_gives_up_after_exhausting_retries(monkeypatch):+ """A persistent transient failure ultimately propagates the last exception."""+ final_exc = asyncpg.exceptions.ConnectionDoesNotExistError("still down")++ async def always_fail(*_args, **_kwargs):+ raise final_exc++ async def fake_sleep(_seconds):+ pass++ monkeypatch.setattr(run_migrations.asyncpg, "connect", always_fail)+ monkeypatch.setattr(run_migrations.asyncio, "sleep", fake_sleep)++ with pytest.raises(asyncpg.exceptions.ConnectionDoesNotExistError):+ await run_migrations._connect()+++async def test_connect_does_not_retry_auth_failures(monkeypatch):+ """Auth/config errors are surfaced immediately — retrying them is pointless+ and would mask a real misconfiguration as a deploy hang."""++ async def auth_fail(*_args, **_kwargs):+ raise asyncpg.exceptions.InvalidPasswordError("wrong password for user")++ call_count = {"connect": 0}++ async def counting_connect(*args, **kwargs):+ call_count["connect"] += 1+ await auth_fail(*args, **kwargs)++ monkeypatch.setattr(run_migrations.asyncpg, "connect", counting_connect)+ # Make sure asyncio.sleep is never called — auth failures shouldn't retry.+ monkeypatch.setattr(+ run_migrations.asyncio,+ "sleep",+ AsyncMock(side_effect=AssertionError("auth failure must not retry")),+ )++ with pytest.raises(asyncpg.exceptions.InvalidPasswordError):+ await run_migrations._connect()++ assert call_count["connect"] == 1, "auth failures must propagate after the first attempt, no retries"