Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.
Python agentic-aiai-securityalert-triageblue-teamcybersecuritydetection-engineeringincident-responselanggraphllm-agentsmcpmitre-attackmodel-context-protocolpurple-teamsecurity-operationsself-hostedsiemsoarsocthreat-detectionthreat-intelligence
fix(api): isolate migration runner on dedicated asyncpg connection
Beenu Arora0b0f9caa59b9
diff --git a/services/api/app/scripts/run_migrations.py b/services/api/app/scripts/run_migrations.pyindex b340aeff4..09a514be9 100644--- a/services/api/app/scripts/run_migrations.py+++ b/services/api/app/scripts/run_migrations.py@@ -1,12 +1,27 @@"""Apply raw SQL migrations under ``services/api/migrations``.This is a lightweight, idempotent migration runner. Each migration is executed-inside a transaction and tracked in an ``aisoc_schema_migrations`` table so it-will not be re-applied on subsequent runs. Migrations themselves are written-defensively (``CREATE TABLE IF NOT EXISTS``, ``ALTER TABLE … ADD COLUMN IF NOT-EXISTS``) so partial re-applies are safe.--Run via:+in its own asyncpg connection and tracked in an ``aisoc_schema_migrations``+table so it will not be re-applied on subsequent runs. Migrations themselves+are written defensively (``CREATE TABLE IF NOT EXISTS``, ``ALTER TABLE … ADD+COLUMN IF NOT EXISTS``) so partial re-applies are safe.++Why a dedicated asyncpg connection (not SQLAlchemy)?+----------------------------------------------------+The migration files frequently contain multi-statement SQL scripts+(``BEGIN; … COMMIT;``, multiple ``CREATE TABLE`` statements, etc.). SQLAlchemy++ asyncpg routes everything through asyncpg's *prepared statement* protocol,+which raises ``cannot insert multiple commands into a prepared statement``.+We previously worked around this by reaching for the underlying asyncpg+connection inside ``engine.begin()``. That fixed the multi-statement issue,+but mixing raw ``Connection.execute()`` with SQLAlchemy's transaction+management left pool connections in an inconsistent state — every later+request that landed on a poisoned connection failed with ``cannot use+Connection.transaction() in a manually started transaction``. Using a+fresh, short-lived asyncpg connection (closed before the API starts serving+traffic) avoids both problems.++Run standalone via:python -m app.scripts.run_migrations"""@@ -16,16 +31,21 @@ from __future__ import annotationsimport asyncioimport loggingfrom pathlib import Path+from urllib.parse import parse_qsl, urlsplit, urlunsplit-from sqlalchemy import text+import asyncpg-from app.db.database import engine+from app.core.config import settingslogger = logging.getLogger(__name__)logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")MIGRATIONS_DIR = Path(__file__).resolve().parents[2] / "migrations"+# libpq sslmode → asyncpg ssl kwarg. Mirrors the mapping in+# ``app.db.database._normalize_async_pg_url`` so this script can be run+# standalone without depending on SQLAlchemy's connection plumbing.+_SSLMODE_PASSTHROUGH = {"disable", "allow", "prefer", "require", "verify-ca", "verify-full"}CREATE_MIGRATIONS_TABLE = """CREATE TABLE IF NOT EXISTS aisoc_schema_migrations (@@ -35,44 +55,71 @@ CREATE TABLE IF NOT EXISTS aisoc_schema_migrations ("""-async def _raw_execute(conn, sql: str) -> None:- """Execute SQL on the underlying asyncpg connection.+def _asyncpg_dsn(url: str) -> tuple[str, dict]:+ """Strip SQLAlchemy/libpq adornments and return (DSN, asyncpg connect kwargs).- SQLAlchemy's ``text()`` path uses asyncpg's prepared-statement protocol,- which cannot handle multi-statement SQL (``cannot insert multiple commands- into a prepared statement``). The raw asyncpg ``execute()`` uses the simple- query protocol and accepts multi-statement scripts, which is what every- file under ``services/api/migrations`` ships.+ asyncpg accepts only the bare ``postgres://`` / ``postgresql://`` scheme,+ no ``+asyncpg`` suffix. It also rejects libpq-only query params like+ ``sslmode`` and ``channel_binding``."""- raw = await conn.get_raw_connection()- asyncpg_conn = raw.driver_connection- await asyncpg_conn.execute(sql)+ if url.startswith("postgresql+asyncpg://"):+ url = "postgresql://" + url[len("postgresql+asyncpg://") :]+ elif url.startswith("postgres+asyncpg://"):+ url = "postgres://" + url[len("postgres+asyncpg://") :]++ parts = urlsplit(url)+ pairs = parse_qsl(parts.query, keep_blank_values=True)+ kwargs: dict = {}+ remaining: list[tuple[str, str]] = []+ for key, value in pairs:+ if key == "sslmode":+ mode = value.lower().strip()+ if mode in _SSLMODE_PASSTHROUGH:+ kwargs["ssl"] = False if mode == "disable" else mode+ continue+ if key == "channel_binding":+ continue+ remaining.append((key, value))++ new_url = urlunsplit(+ (parts.scheme, parts.netloc, parts.path, "&".join(f"{k}={v}" for k, v in remaining), parts.fragment)+ )+ return new_url, kwargs-async def _applied(conn) -> set[str]:- res = await conn.execute(text("SELECT name FROM aisoc_schema_migrations"))- return {row[0] for row in res}+async def _connect() -> asyncpg.Connection:+ dsn, kwargs = _asyncpg_dsn(str(settings.DATABASE_URL))+ return await asyncpg.connect(dsn, **kwargs)-async def _record(conn, name: str) -> None:- await conn.execute(- text("INSERT INTO aisoc_schema_migrations(name) VALUES (:n) ON CONFLICT DO NOTHING").bindparams(n=name)- )+async def _applied(conn: asyncpg.Connection) -> set[str]:+ rows = await conn.fetch("SELECT name FROM aisoc_schema_migrations")+ return {row["name"] for row in rows}-async def _apply_one(name: str, sql: str) -> tuple[str, bool, str | None]:- """Apply a single migration in its own transaction.+async def _apply_one(conn: asyncpg.Connection, name: str, sql: str) -> tuple[str, bool, str | None]:+ """Apply a single migration.- Returns (name, ok, error). Failures are recorded but don't crash the runner- so later migrations (which may not depend on the failing one) can still- apply. The deploy log will surface which migrations failed.+ The migration files manage their own transactions (most start with+ ``BEGIN;`` and end with ``COMMIT;``). Where they don't, asyncpg+ auto-commits after each statement. Either way, after the script returns+ the connection is in a clean state so subsequent migrations don't+ interfere."""try:- async with engine.begin() as conn:- await _raw_execute(conn, sql)- await _record(conn, name)+ await conn.execute(sql)+ await conn.execute(+ "INSERT INTO aisoc_schema_migrations(name) VALUES ($1) ON CONFLICT DO NOTHING",+ name,+ )return name, True, Noneexcept Exception as exc: # noqa: BLE001 — we intentionally continue on failure+ # Make sure the connection isn't left mid-transaction after a failure+ # (e.g. ``BEGIN`` succeeded but a subsequent statement raised).+ try:+ await conn.execute("ROLLBACK")+ except Exception:+ passreturn name, False, str(exc)@@ -84,27 +131,27 @@ async def main() -> None:files = sorted(p for p in MIGRATIONS_DIR.iterdir() if p.suffix == ".sql")logger.info("Found %d migration files", len(files))- async with engine.begin() as conn:- await _raw_execute(conn, CREATE_MIGRATIONS_TABLE)-- async with engine.connect() as conn:+ conn = await _connect()+ try:+ await conn.execute(CREATE_MIGRATIONS_TABLE)already = await _applied(conn)-- pending = [p for p in files if p.name not in already]- logger.info("%d migrations already applied; %d pending", len(already), len(pending))-- failures: list[tuple[str, str]] = []- for path in pending:- sql = path.read_text(encoding="utf-8")- name, ok, err = await _apply_one(path.name, sql)- if ok:- logger.info("✓ applied %s", name)- else:- logger.error("✗ failed %s: %s", name, err)- failures.append((name, err or ""))-- if failures:- logger.warning("%d migrations failed; see logs above", len(failures))+ pending = [p for p in files if p.name not in already]+ logger.info("%d migrations already applied; %d pending", len(already), len(pending))++ failures: list[tuple[str, str]] = []+ for path in pending:+ sql = path.read_text(encoding="utf-8")+ name, ok, err = await _apply_one(conn, path.name, sql)+ if ok:+ logger.info("✓ applied %s", name)+ else:+ logger.error("✗ failed %s: %s", name, err)+ failures.append((name, err or ""))++ if failures:+ logger.warning("%d migrations failed; see logs above", len(failures))+ finally:+ await conn.close()if __name__ == "__main__":