ARES - authorized red-team engagement automation with dashboard, campaign scope, module orchestration, OPSEC controls, encrypted vault, and reporting.
Python active-directorycloud-securitydefensive-validationfastapigood-first-issuemitre-attackopsecreactred-teamreportingsecurity
chore(db): expose PostgreSQL startup diagnostics
Mafifrizi808c7717a92f
diff --git a/ares/db/postgres.py b/ares/db/postgres.pyindex c1e54c5..6107202 100644--- a/ares/db/postgres.py+++ b/ares/db/postgres.py@@ -321,6 +321,254 @@ CREATE INDEX IF NOT EXISTS idx_ws_tickets_api_keyON websocket_tickets(api_key_id);"""+_POSTGRES_FALLBACK_DDL_CODES = (+ "campaigns-table",+ "module-runs-table",+ "module-runs-campaign-index",+ "module-runs-completed-index",+ "findings-table",+ "findings-campaign-index",+ "findings-severity-index",+ "findings-false-positive-index",+ "hosts-table",+ "hosts-campaign-index",+ "hosts-ip-index",+ "credentials-table",+ "credentials-campaign-index",+ "loot-table",+ "audit-log-table",+ "audit-log-campaign-index",+ "users-table",+ "users-username-index",+ "api-keys-table",+ "api-keys-user-index",+ "api-keys-prefix-index",+ "refresh-tokens-table",+ "refresh-tokens-user-index",+ "refresh-tokens-expiry-index",+ "revoked-access-tokens-table",+ "revoked-access-tokens-expiry-index",+ "websocket-tickets-table",+ "websocket-tickets-expiry-index",+ "websocket-tickets-user-index",+ "websocket-tickets-campaign-index",+ "websocket-tickets-api-key-index",+)+++def _postgres_fallback_statement_spans() -> tuple[+ tuple[str, int, int], ...+]:+ spans: list[tuple[str, int, int]] = []+ start = 0+ for code in _POSTGRES_FALLBACK_DDL_CODES:+ end = _PG_CREATE_TABLES.find(";", start)+ if end < 0:+ raise RuntimeError("Invalid PostgreSQL fallback DDL contract")+ spans.append((code, start + 1, end + 1))+ start = end + 1+ if _PG_CREATE_TABLES[start:].strip():+ raise RuntimeError("Invalid PostgreSQL fallback DDL contract")+ return tuple(spans)+++_POSTGRES_FALLBACK_STATEMENT_SPANS = _postgres_fallback_statement_spans()+_POSTGRES_STARTUP_STAGES = frozenset(+ {+ "ownership",+ "fallback-ddl",+ "fallback-validation",+ "managed-validation",+ "unclassified",+ }+)+_POSTGRES_STARTUP_OPERATIONAL_CATEGORIES = frozenset(+ {"none", "timeout", "connection", "database", "runtime", "other"}+)+_POSTGRES_TICKET_INVARIANTS = frozenset(+ {+ "ticket-relation-present",+ "ticket-relation-metadata",+ "ticket-columns",+ "ticket-constraint-inventory",+ "ticket-canonical-opclasses",+ "ticket-index-identity",+ "ticket-index-inventory",+ "ticket-constraint-definition-primary",+ "ticket-constraint-definition-campaign",+ "ticket-constraint-definition-user",+ "ticket-constraint-definition-api-key",+ "ticket-reference-inventory",+ "ticket-constraint-table-binding",+ "ticket-foreign-key-campaign",+ "ticket-foreign-key-user",+ "ticket-foreign-key-api-key",+ "ticket-primary-key-binding",+ "ticket-check-hash",+ "ticket-check-kind",+ "ticket-check-created-at",+ "ticket-check-expires-at",+ "ticket-check-consumed-at",+ "ticket-check-bearer-expires-finite",+ "ticket-check-created-finite",+ "ticket-check-expires-finite",+ "ticket-check-consumed-finite",+ "ticket-check-time-order",+ "ticket-check-source-shape",+ "ticket-validation-unclassified",+ }+)+_POSTGRES_OWNERSHIP_INVARIANTS = frozenset(+ {+ "ownership-relation-query",+ "ownership-relation-metadata",+ "ownership-revision-query",+ "ownership-revision-cardinality",+ "ownership-revision-value",+ "ownership-known-older-revision",+ "ownership-unclassified",+ }+)+_POSTGRES_STARTUP_INVARIANTS = (+ frozenset(_POSTGRES_FALLBACK_DDL_CODES)+ | _POSTGRES_TICKET_INVARIANTS+ | _POSTGRES_OWNERSHIP_INVARIANTS+ | frozenset({"fallback-ddl-unclassified", "managed-validation-unclassified"})+)+_POSTGRES_STARTUP_TRACE_CODES = frozenset(+ {"fallback-entered", "fallback-ddl-complete", "startup-ready"}+)+++def _postgres_operational_category(error: BaseException) -> str:+ if isinstance(error, TimeoutError):+ return "timeout"+ if isinstance(error, ConnectionError):+ return "connection"+ error_type_names = {+ base.__name__+ for base in type(error).__mro__+ }+ if error_type_names & {+ "PostgresError",+ "InterfaceError",+ "ConnectionDoesNotExistError",+ "CannotConnectNowError",+ }:+ return "database"+ if isinstance(error, RuntimeError):+ return "runtime"+ return "other"+++class _PostgresStartupDiagnosticError(RuntimeError):+ def __init__(+ self,+ public_message: str,+ *,+ diagnostic_stage: str,+ diagnostic_invariant: str,+ operational_category: str = "none",+ ) -> None:+ super().__init__(public_message)+ self.diagnostic_stage = (+ diagnostic_stage+ if diagnostic_stage in _POSTGRES_STARTUP_STAGES+ else "unclassified"+ )+ self.diagnostic_invariant = (+ diagnostic_invariant+ if diagnostic_invariant in _POSTGRES_STARTUP_INVARIANTS+ else "ownership-unclassified"+ )+ self.operational_category = (+ operational_category+ if operational_category+ in _POSTGRES_STARTUP_OPERATIONAL_CATEGORIES+ else "other"+ )+++def _postgres_startup_diagnostic_label(error: BaseException) -> str:+ if not isinstance(error, _PostgresStartupDiagnosticError):+ return "unclassified"+ if error.diagnostic_invariant in {+ "ownership-unclassified",+ "fallback-ddl-unclassified",+ "managed-validation-unclassified",+ "ticket-validation-unclassified",+ }:+ return "unclassified"+ if (+ error.diagnostic_stage not in _POSTGRES_STARTUP_STAGES+ or error.diagnostic_invariant not in _POSTGRES_STARTUP_INVARIANTS+ or error.operational_category+ not in _POSTGRES_STARTUP_OPERATIONAL_CATEGORIES+ ):+ return "unclassified"+ return (+ f"{error.diagnostic_stage}:"+ f"{error.diagnostic_invariant}:"+ f"{error.operational_category}"+ )+++def _postgres_startup_error(+ public_message: str,+ *,+ stage: str,+ invariant: str,+ cause: BaseException | None = None,+) -> _PostgresStartupDiagnosticError:+ category = (+ "none"+ if cause is None+ else _postgres_operational_category(cause)+ )+ return _PostgresStartupDiagnosticError(+ public_message,+ diagnostic_stage=stage,+ diagnostic_invariant=invariant,+ operational_category=category,+ )+++def _postgres_fallback_failure_invariant(error: BaseException) -> str:+ raw_position = getattr(error, "position", None)+ if not isinstance(raw_position, (int, str)):+ return "fallback-ddl-unclassified"+ try:+ position = int(raw_position)+ except (TypeError, ValueError):+ return "fallback-ddl-unclassified"+ for code, start, end in _POSTGRES_FALLBACK_STATEMENT_SPANS:+ if start <= position <= end:+ return code+ return "fallback-ddl-unclassified"+++def _postgres_restage_startup_error(+ error: _PostgresStartupDiagnosticError,+ stage: str,+) -> _PostgresStartupDiagnosticError:+ return _PostgresStartupDiagnosticError(+ str(error),+ diagnostic_stage=stage,+ diagnostic_invariant=error.diagnostic_invariant,+ operational_category=error.operational_category,+ )+++def _postgres_ticket_schema_error(+ invariant: str,+) -> _PostgresStartupDiagnosticError:+ return _postgres_startup_error(+ "Incompatible WebSocket ticket schema",+ stage="managed-validation",+ invariant=invariant,+ )++_POSTGRES_MANAGED_REVISION = "0008"_POSTGRES_OLDER_REVISIONS = frozenset({"0001", "0002", "0003", "0004", "0005", "0006", "0007"}@@ -330,11 +578,11 @@ _POSTGRES_MANAGED_SCHEMA_ERROR = "Incompatible managed PostgreSQL schema"_POSTGRES_SCHEMA_VALIDATION_ERROR = "PostgreSQL schema validation failed"-class _PostgresMigrationRequiredError(RuntimeError):+class _PostgresMigrationRequiredError(_PostgresStartupDiagnosticError):pass-class _PostgresManagedSchemaError(RuntimeError):+class _PostgresManagedSchemaError(_PostgresStartupDiagnosticError):pass@@ -561,16 +809,32 @@ def _postgres_index_opclass(table: str, column: str) -> str:def _classify_postgres_revision(values: tuple[object, ...]) -> str:- if len(values) != 1 or not isinstance(values[0], str):- raise _PostgresManagedSchemaError(_POSTGRES_MANAGED_SCHEMA_ERROR)+ if len(values) != 1:+ raise _PostgresManagedSchemaError(+ _POSTGRES_MANAGED_SCHEMA_ERROR,+ diagnostic_stage="ownership",+ diagnostic_invariant="ownership-revision-cardinality",+ )+ if not isinstance(values[0], str):+ raise _PostgresManagedSchemaError(+ _POSTGRES_MANAGED_SCHEMA_ERROR,+ diagnostic_stage="ownership",+ diagnostic_invariant="ownership-revision-value",+ )revision = values[0]if revision == _POSTGRES_MANAGED_REVISION:return revisionif revision in _POSTGRES_OLDER_REVISIONS:raise _PostgresMigrationRequiredError(- _POSTGRES_MIGRATION_REQUIRED_ERROR+ _POSTGRES_MIGRATION_REQUIRED_ERROR,+ diagnostic_stage="ownership",+ diagnostic_invariant="ownership-known-older-revision",)- raise _PostgresManagedSchemaError(_POSTGRES_MANAGED_SCHEMA_ERROR)+ raise _PostgresManagedSchemaError(+ _POSTGRES_MANAGED_SCHEMA_ERROR,+ diagnostic_stage="ownership",+ diagnostic_invariant="ownership-revision-value",+ )class PostgresDatabase:@@ -603,6 +867,7 @@ class PostgresDatabase:self._pool_min = pool_minself._pool_max = pool_maxself._pool: Any = None # asyncpg.Pool+ self._startup_trace: list[str] = []# ── Lifecycle ──────────────────────────────────────────────────────────────@@ -629,6 +894,7 @@ class PostgresDatabase:except asyncio.CancelledError as primary:await self._close_failed_startup_pool(pool, primary)raise+ self._record_startup_trace("startup-ready")logger.info("pg_db_ready")return self@@ -640,14 +906,20 @@ class PostgresDatabase:try:await pool.close()except Exception as cleanup_error:+ cleanup_category = _postgres_operational_category(cleanup_error)primary.add_note("PostgreSQL startup cleanup failure "- f"[pool-close: {type(cleanup_error).__name__}]"+ f"[pool-close: {cleanup_category}]")finally:if self._pool is pool:self._pool = None+ def _record_startup_trace(self, code: str) -> None:+ if code not in _POSTGRES_STARTUP_TRACE_CODES:+ raise RuntimeError("Invalid PostgreSQL startup trace identifier")+ self._startup_trace.append(code)+async def __aenter__(self) -> "PostgresDatabase":return await self.connect()@@ -669,9 +941,10 @@ class PostgresDatabase:@staticmethodasync def _managed_schema_revision(connection: Any) -> str | None:- relation_rows = await connection.fetch(- """- SELECT rel.oid::bigint AS table_oid,+ try:+ relation_rows = await connection.fetch(+ """+ SELECT rel.oid::bigint AS table_oid,nsp.nspname AS schema_name,relation_type.oid::bigint AS type_oid,type_nsp.nspname AS type_schema,@@ -715,14 +988,27 @@ class PostgresDatabase:ON relation_type.oid=rel.reltypeLEFT JOIN pg_namespace AS type_nspON type_nsp.oid=relation_type.typnamespace- WHERE nsp.nspname=current_schema()- AND rel.relname='alembic_version'- """- )+ WHERE nsp.nspname=current_schema()+ AND rel.relname='alembic_version'+ """+ )+ except asyncio.CancelledError:+ raise+ except Exception as exc:+ raise _postgres_startup_error(+ _POSTGRES_SCHEMA_VALIDATION_ERROR,+ stage="ownership",+ invariant="ownership-relation-query",+ cause=exc,+ ) from Noneif not relation_rows:return Noneif len(relation_rows) != 1:- raise RuntimeError(_POSTGRES_MANAGED_SCHEMA_ERROR)+ raise _PostgresManagedSchemaError(+ _POSTGRES_MANAGED_SCHEMA_ERROR,+ diagnostic_stage="ownership",+ diagnostic_invariant="ownership-relation-metadata",+ )relation = relation_rows[0]try:table_oid = int(relation["table_oid"])@@ -747,7 +1033,11 @@ class PostgresDatabase:int(relation["user_rule_count"]),)except (KeyError, TypeError, ValueError):- raise RuntimeError(_POSTGRES_MANAGED_SCHEMA_ERROR) from None+ raise _PostgresManagedSchemaError(+ _POSTGRES_MANAGED_SCHEMA_ERROR,+ diagnostic_stage="ownership",+ diagnostic_invariant="ownership-relation-metadata",+ ) from Noneif (not isinstance(schema_name, str)or not schema_name@@ -757,7 +1047,11 @@ class PostgresDatabase:or relation_contract!= ("r", "p", False, False, False, 0, 0, 0, 0, 0)):- raise RuntimeError(_POSTGRES_MANAGED_SCHEMA_ERROR)+ raise _PostgresManagedSchemaError(+ _POSTGRES_MANAGED_SCHEMA_ERROR,+ diagnostic_stage="ownership",+ diagnostic_invariant="ownership-relation-metadata",+ )column_rows = await connection.fetch("""@@ -810,7 +1104,11 @@ class PostgresDatabase:for row in column_rows)except (KeyError, TypeError, ValueError):- raise RuntimeError(_POSTGRES_MANAGED_SCHEMA_ERROR) from None+ raise _PostgresManagedSchemaError(+ _POSTGRES_MANAGED_SCHEMA_ERROR,+ diagnostic_stage="ownership",+ diagnostic_invariant="ownership-relation-metadata",+ ) from Noneif columns != ((1,@@ -828,7 +1126,11 @@ class PostgresDatabase:None,),):- raise RuntimeError(_POSTGRES_MANAGED_SCHEMA_ERROR)+ raise _PostgresManagedSchemaError(+ _POSTGRES_MANAGED_SCHEMA_ERROR,+ diagnostic_stage="ownership",+ diagnostic_invariant="ownership-relation-metadata",+ )primary_rows = await connection.fetch("""@@ -865,7 +1167,11 @@ class PostgresDatabase:for row in primary_rows)except (KeyError, TypeError, ValueError):- raise RuntimeError(_POSTGRES_MANAGED_SCHEMA_ERROR) from None+ raise _PostgresManagedSchemaError(+ _POSTGRES_MANAGED_SCHEMA_ERROR,+ diagnostic_stage="ownership",+ diagnostic_invariant="ownership-relation-metadata",+ ) from Noneif primary_contract != (("alembic_version_pkc",@@ -876,7 +1182,11 @@ class PostgresDatabase:("version_num",),),):- raise RuntimeError(_POSTGRES_MANAGED_SCHEMA_ERROR)+ raise _PostgresManagedSchemaError(+ _POSTGRES_MANAGED_SCHEMA_ERROR,+ diagnostic_stage="ownership",+ diagnostic_invariant="ownership-relation-metadata",+ )version_index_rows = await connection.fetch("""@@ -983,7 +1293,11 @@ class PostgresDatabase:for row in version_index_rows)except (KeyError, TypeError, ValueError):- raise RuntimeError(_POSTGRES_MANAGED_SCHEMA_ERROR) from None+ raise _PostgresManagedSchemaError(+ _POSTGRES_MANAGED_SCHEMA_ERROR,+ diagnostic_stage="ownership",+ diagnostic_invariant="ownership-relation-metadata",+ ) from Noneif version_indexes != (("alembic_version_pkc",@@ -1008,16 +1322,34 @@ class PostgresDatabase:None,),):- raise RuntimeError(_POSTGRES_MANAGED_SCHEMA_ERROR)+ raise _PostgresManagedSchemaError(+ _POSTGRES_MANAGED_SCHEMA_ERROR,+ diagnostic_stage="ownership",+ diagnostic_invariant="ownership-relation-metadata",+ )quoted_schema = '"' + schema_name.replace('"', '""') + '"'- revision_rows = await connection.fetch(- f'SELECT version_num FROM {quoted_schema}."alembic_version"' # noqa: S608- )+ try:+ revision_rows = await connection.fetch(+ f'SELECT version_num FROM {quoted_schema}."alembic_version"' # noqa: S608+ )+ except asyncio.CancelledError:+ raise+ except Exception as exc:+ raise _postgres_startup_error(+ _POSTGRES_SCHEMA_VALIDATION_ERROR,+ stage="ownership",+ invariant="ownership-revision-query",+ cause=exc,+ ) from Nonetry:values = tuple(row["version_num"] for row in revision_rows)except (KeyError, TypeError):- raise RuntimeError(_POSTGRES_MANAGED_SCHEMA_ERROR) from None+ raise _PostgresManagedSchemaError(+ _POSTGRES_MANAGED_SCHEMA_ERROR,+ diagnostic_stage="ownership",+ diagnostic_invariant="ownership-revision-cardinality",+ ) from Nonereturn _classify_postgres_revision(values)@staticmethod@@ -1946,61 +2278,117 @@ class PostgresDatabase:except (_PostgresManagedSchemaError,_PostgresMigrationRequiredError,+ _PostgresStartupDiagnosticError,):raise- except RuntimeError as exc:- if str(exc) == _POSTGRES_MANAGED_SCHEMA_ERROR:- raise _PostgresManagedSchemaError(- _POSTGRES_MANAGED_SCHEMA_ERROR- ) from None- raise RuntimeError(- _POSTGRES_SCHEMA_VALIDATION_ERROR- ) from None- except Exception:- raise RuntimeError(- _POSTGRES_SCHEMA_VALIDATION_ERROR+ except asyncio.CancelledError:+ raise+ except Exception as exc:+ raise _postgres_startup_error(+ _POSTGRES_SCHEMA_VALIDATION_ERROR,+ stage="ownership",+ invariant="ownership-relation-metadata",+ cause=exc,) from Noneif revision is not None:try:await self._validate_managed_schema(conn)await self._validate_websocket_ticket_schema(conn)- except _PostgresManagedSchemaError:+ except _PostgresMigrationRequiredError:+ raise+ except _PostgresStartupDiagnosticError as exc:+ raise _postgres_restage_startup_error(+ exc,+ "managed-validation",+ ) from None+ except asyncio.CancelledError:raiseexcept RuntimeError as exc:if str(exc) == _POSTGRES_MANAGED_SCHEMA_ERROR:raise _PostgresManagedSchemaError(- _POSTGRES_MANAGED_SCHEMA_ERROR+ _POSTGRES_MANAGED_SCHEMA_ERROR,+ diagnostic_stage="managed-validation",+ diagnostic_invariant=(+ "managed-validation-unclassified"+ ),) from None- if str(exc) == (- "Incompatible WebSocket ticket schema"- ):- raise- raise RuntimeError(- _POSTGRES_SCHEMA_VALIDATION_ERROR+ raise _postgres_startup_error(+ _POSTGRES_SCHEMA_VALIDATION_ERROR,+ stage="managed-validation",+ invariant="managed-validation-unclassified",+ cause=exc,) from None- except Exception:- raise RuntimeError(- _POSTGRES_SCHEMA_VALIDATION_ERROR+ except Exception as exc:+ raise _postgres_startup_error(+ _POSTGRES_SCHEMA_VALIDATION_ERROR,+ stage="managed-validation",+ invariant="managed-validation-unclassified",+ cause=exc,) from Noneelse:- ticket_table_exists = bool(- await conn.fetchval(- """- SELECT EXISTS(- SELECT 1- FROM pg_class AS rel- JOIN pg_namespace AS nsp- ON nsp.oid=rel.relnamespace- WHERE nsp.nspname=current_schema()- AND rel.relname='websocket_tickets'+ self._record_startup_trace("fallback-entered")+ try:+ ticket_table_exists = bool(+ await conn.fetchval(+ """+ SELECT EXISTS(+ SELECT 1+ FROM pg_class AS rel+ JOIN pg_namespace AS nsp+ ON nsp.oid=rel.relnamespace+ WHERE nsp.nspname=current_schema()+ AND rel.relname='websocket_tickets'+ )+ """)- """)- )+ except asyncio.CancelledError:+ raise+ except Exception as exc:+ raise _postgres_startup_error(+ _POSTGRES_SCHEMA_VALIDATION_ERROR,+ stage="fallback-validation",+ invariant="ticket-relation-present",+ cause=exc,+ ) from Noneif ticket_table_exists:+ try:+ await self._validate_websocket_ticket_schema(conn)+ except _PostgresStartupDiagnosticError as exc:+ raise _postgres_restage_startup_error(+ exc,+ "fallback-validation",+ ) from None+ try:+ await conn.execute(_PG_CREATE_TABLES)+ except asyncio.CancelledError:+ raise+ except Exception as exc:+ raise _postgres_startup_error(+ _POSTGRES_SCHEMA_VALIDATION_ERROR,+ stage="fallback-ddl",+ invariant=_postgres_fallback_failure_invariant(+ exc+ ),+ cause=exc,+ ) from None+ self._record_startup_trace("fallback-ddl-complete")+ try:await self._validate_websocket_ticket_schema(conn)- await conn.execute(_PG_CREATE_TABLES)- await self._validate_websocket_ticket_schema(conn)+ except _PostgresStartupDiagnosticError as exc:+ raise _postgres_restage_startup_error(+ exc,+ "fallback-validation",+ ) from None+ except asyncio.CancelledError:+ raise+ except Exception as exc:+ raise _postgres_startup_error(+ _POSTGRES_SCHEMA_VALIDATION_ERROR,+ stage="fallback-validation",+ invariant="ticket-validation-unclassified",+ cause=exc,+ ) from Nonelogger.info("pg_schema_ready")@staticmethod@@ -2049,7 +2437,9 @@ class PostgresDatabase:""")if relation is None:- raise RuntimeError("Incompatible WebSocket ticket schema")+ raise _postgres_ticket_schema_error(+ "ticket-relation-present"+ )table_oid = int(relation["table_oid"])schema_oid = int(relation["schema_oid"])schema_name = str(relation["schema_name"])@@ -2077,7 +2467,9 @@ class PostgresDatabase:0,0,):- raise RuntimeError("Incompatible WebSocket ticket schema")+ raise _postgres_ticket_schema_error(+ "ticket-relation-metadata"+ )rows = await connection.fetch("""@@ -2158,7 +2550,7 @@ class PostgresDatabase:),]if actual_columns != expected_columns:- raise RuntimeError("Incompatible WebSocket ticket schema")+ raise _postgres_ticket_schema_error("ticket-columns")constraint_rows = await connection.fetch("""@@ -2219,7 +2611,9 @@ class PostgresDatabase:"websocket_tickets_pkey",}if set(actual_constraints) != expected_constraint_names:- raise RuntimeError("Incompatible WebSocket ticket schema")+ raise _postgres_ticket_schema_error(+ "ticket-constraint-inventory"+ )index_rows = await connection.fetch("""@@ -2316,7 +2710,9 @@ class PostgresDatabase:for row in canonical_opclass_rows}if set(canonical_opclasses) != {"text_ops", "timestamptz_ops"}:- raise RuntimeError("Incompatible WebSocket ticket schema")+ raise _postgres_ticket_schema_error(+ "ticket-canonical-opclasses"+ )index_oids: set[int] = set()for row in index_rows:@@ -2348,7 +2744,9 @@ class PostgresDatabase:and index_oid not in index_oids)if not identity_is_exact:- raise RuntimeError("Incompatible WebSocket ticket schema")+ raise _postgres_ticket_schema_error(+ "ticket-index-identity"+ )index_oids.add(index_oid)actual_indexes = {@@ -2397,7 +2795,9 @@ class PostgresDatabase:),}if actual_indexes != expected_indexes:- raise RuntimeError("Incompatible WebSocket ticket schema")+ raise _postgres_ticket_schema_error(+ "ticket-index-inventory"+ )expected_constraint_fragments = {"websocket_tickets_pkey": ("p", "PRIMARY KEY (ticket_hash)"),@@ -2414,6 +2814,20 @@ class PostgresDatabase:"FOREIGN KEY (api_key_id) REFERENCES api_keys(id) ON DELETE CASCADE",),}+ constraint_fragment_invariants = {+ "websocket_tickets_pkey": (+ "ticket-constraint-definition-primary"+ ),+ "fk_ws_ticket_campaign": (+ "ticket-constraint-definition-campaign"+ ),+ "fk_ws_ticket_user": (+ "ticket-constraint-definition-user"+ ),+ "fk_ws_ticket_api_key": (+ "ticket-constraint-definition-api-key"+ ),+ }for name, definition in expected_constraint_fragments.items():row = actual_constraints[name]if (@@ -2423,7 +2837,9 @@ class PostgresDatabase:or bool(row["condeferred"])or " ".join(str(row["definition"]).split()) != definition[1]):- raise RuntimeError("Incompatible WebSocket ticket schema")+ raise _postgres_ticket_schema_error(+ constraint_fragment_invariants[name]+ )referenced_schema = str(await connection.fetchval("SELECT current_schema()"))expected_foreign_keys = {@@ -2452,14 +2868,23 @@ class PostgresDatabase:for row in referenced_rows}if set(referenced_oids) != {"api_keys", "campaigns", "users"}:- raise RuntimeError("Incompatible WebSocket ticket schema")+ raise _postgres_ticket_schema_error(+ "ticket-reference-inventory"+ )if any(int(row["table_oid"]) != table_oidfor row in actual_constraints.values()):- raise RuntimeError("Incompatible WebSocket ticket schema")+ raise _postgres_ticket_schema_error(+ "ticket-constraint-table-binding"+ )+ foreign_key_invariants = {+ "fk_ws_ticket_campaign": "ticket-foreign-key-campaign",+ "fk_ws_ticket_user": "ticket-foreign-key-user",+ "fk_ws_ticket_api_key": "ticket-foreign-key-api-key",+ }for name, expected in expected_foreign_keys.items():row = actual_constraints[name]actual = (@@ -2477,7 +2902,9 @@ class PostgresDatabase:== referenced_oids[expected[2]])if actual != expected or not reference_identity_is_exact:- raise RuntimeError("Incompatible WebSocket ticket schema")+ raise _postgres_ticket_schema_error(+ foreign_key_invariants[name]+ )primary = actual_constraints["websocket_tickets_pkey"]if (@@ -2493,7 +2920,9 @@ class PostgresDatabase:if str(row["index_name"]) == "websocket_tickets_pkey")):- raise RuntimeError("Incompatible WebSocket ticket schema")+ raise _postgres_ticket_schema_error(+ "ticket-primary-key-binding"+ )expected_check_definitions = {"ck_ws_ticket_hash": (@@ -2542,6 +2971,27 @@ class PostgresDatabase:"AND required_scope = 'read'::text)"),}+ check_invariants = {+ "ck_ws_ticket_hash": "ticket-check-hash",+ "ck_ws_ticket_kind": "ticket-check-kind",+ "ck_ws_ticket_created_at": "ticket-check-created-at",+ "ck_ws_ticket_expires_at": "ticket-check-expires-at",+ "ck_ws_ticket_consumed_at": "ticket-check-consumed-at",+ "ck_ws_ticket_bearer_expires_finite": (+ "ticket-check-bearer-expires-finite"+ ),+ "ck_ws_ticket_created_finite": (+ "ticket-check-created-finite"+ ),+ "ck_ws_ticket_expires_finite": (+ "ticket-check-expires-finite"+ ),+ "ck_ws_ticket_consumed_finite": (+ "ticket-check-consumed-finite"+ ),+ "ck_ws_ticket_time_order": "ticket-check-time-order",+ "ck_ws_ticket_source_shape": "ticket-check-source-shape",+ }for name, expected_definition in expected_check_definitions.items():row = actual_constraints[name]if (@@ -2552,7 +3002,9 @@ class PostgresDatabase:or " ".join(str(row["definition"]).split())!= expected_definition):- raise RuntimeError("Incompatible WebSocket ticket schema")+ raise _postgres_ticket_schema_error(+ check_invariants[name]+ )# ── Helpers ────────────────────────────────────────────────────────────────diff --git a/tests/integration/test_postgres_auth_token_lifecycle.py b/tests/integration/test_postgres_auth_token_lifecycle.pyindex 14fe743..960073a 100644--- a/tests/integration/test_postgres_auth_token_lifecycle.py+++ b/tests/integration/test_postgres_auth_token_lifecycle.py@@ -80,8 +80,19 @@ def _runtime_dsn(config: dict[str, str | int], database: str) -> str:def _sanitized_setup_failure(action: str, exc: Exception) -> RuntimeError:+ from ares.db.postgres import _postgres_operational_category++ if action == "runtime-initialize":+ from ares.db.postgres import _postgres_startup_diagnostic_label++ diagnostic = _postgres_startup_diagnostic_label(exc)+ return RuntimeError(+ "PostgreSQL token-test setup failed "+ f"[runtime-initialize:{diagnostic}]"+ )+ category = _postgres_operational_category(exc)return RuntimeError(- f"PostgreSQL token-test setup failed [{action}: {type(exc).__name__}]"+ f"PostgreSQL token-test setup failed [{action}:{category}]")diff --git a/tests/integration/test_postgres_runtime_smoke.py b/tests/integration/test_postgres_runtime_smoke.pyindex a6b9dab..1b1fd7c 100644--- a/tests/integration/test_postgres_runtime_smoke.py+++ b/tests/integration/test_postgres_runtime_smoke.py@@ -24,6 +24,15 @@ class _RollbackProbe(RuntimeError):"""Intentional exception used to verify a real transaction rollback."""+class PostgresError(RuntimeError):+ """Synthetic database-category error for diagnostic-only tests."""+++def _require_fixed(condition: bool, message: str) -> None:+ if not condition:+ pytest.fail(message, pytrace=False)++def _postgres_test_config() -> dict[str, str | int]:values = {name: os.environ.get(name) for name in _POSTGRES_ENV}configured = [name for name, value in values.items() if value]@@ -64,7 +73,20 @@ def _runtime_dsn(config: dict[str, str | int], database: str) -> str:def _sanitized_setup_failure(action: str, exc: Exception) -> RuntimeError:- return RuntimeError(f"PostgreSQL smoke setup failed [{action}: {type(exc).__name__}]")+ from ares.db.postgres import _postgres_operational_category++ if action == "runtime-initialize":+ from ares.db.postgres import _postgres_startup_diagnostic_label++ diagnostic = _postgres_startup_diagnostic_label(exc)+ return RuntimeError(+ "PostgreSQL smoke setup failed "+ f"[runtime-initialize:{diagnostic}]"+ )+ category = _postgres_operational_category(exc)+ return RuntimeError(+ f"PostgreSQL smoke setup failed [{action}:{category}]"+ )async def _attempt_cleanup(@@ -78,6 +100,186 @@ async def _attempt_cleanup(failures.append(f"{action}: {type(exc).__name__}")+def test_postgres_startup_diagnostic_statement_contract() -> None:+ from ares.db.postgres import (+ _POSTGRES_FALLBACK_DDL_CODES,+ _POSTGRES_FALLBACK_STATEMENT_SPANS,+ _postgres_fallback_failure_invariant,+ )++ codes_are_unique = (+ len(_POSTGRES_FALLBACK_DDL_CODES)+ == len(set(_POSTGRES_FALLBACK_DDL_CODES))+ == len(_POSTGRES_FALLBACK_STATEMENT_SPANS)+ )+ _require_fixed(+ codes_are_unique,+ "Fallback statements require unique diagnostic identifiers",+ )+ every_statement_maps = True+ spans_are_ordered = True+ previous_end = 0+ for code, start, end in _POSTGRES_FALLBACK_STATEMENT_SPANS:+ failure = PostgresError("diagnostic canary")+ failure.position = start + ((end - start) // 2)+ every_statement_maps = (+ every_statement_maps+ and _postgres_fallback_failure_invariant(failure) == code+ )+ spans_are_ordered = (+ spans_are_ordered+ and start > previous_end+ and end >= start+ )+ previous_end = end+ _require_fixed(+ every_statement_maps,+ "Every fallback statement must map to its fixed identifier",+ )+ _require_fixed(+ spans_are_ordered,+ "Fallback diagnostic statement spans must be ordered",+ )+++async def test_postgres_startup_diagnostics_are_distinct_and_sanitized() -> None:+ from ares.db.postgres import (+ PostgresDatabase,+ _postgres_restage_startup_error,+ _postgres_startup_diagnostic_label,+ _postgres_startup_error,+ _postgres_ticket_schema_error,+ )++ canary = "diagnostic-canary-not-for-output"++ class _OwnershipFailureConnection:+ async def fetch(self, *_args: object) -> object:+ raise PostgresError(canary)++ try:+ await PostgresDatabase._managed_schema_revision(+ _OwnershipFailureConnection()+ )+ except Exception as exc:+ ownership_failure: BaseException = exc+ else:+ pytest.fail("Ownership failure must be diagnosed", pytrace=False)++ ddl_cause = PostgresError(canary)+ ddl_failure = _postgres_startup_error(+ "PostgreSQL schema validation failed",+ stage="fallback-ddl",+ invariant="campaigns-table",+ cause=ddl_cause,+ )+ validation_failure = _postgres_restage_startup_error(+ _postgres_ticket_schema_error("ticket-columns"),+ "fallback-validation",+ )+ typed_unknown_failure = _postgres_startup_error(+ "PostgreSQL schema validation failed",+ stage="fallback-validation",+ invariant="ticket-validation-unclassified",+ cause=RuntimeError(canary),+ )+ unknown_failure = RuntimeError(canary)++ labels = (+ _postgres_startup_diagnostic_label(ownership_failure),+ _postgres_startup_diagnostic_label(ddl_failure),+ _postgres_startup_diagnostic_label(validation_failure),+ _postgres_startup_diagnostic_label(typed_unknown_failure),+ _postgres_startup_diagnostic_label(unknown_failure),+ )+ labels_are_exact = labels == (+ "ownership:ownership-relation-query:database",+ "fallback-ddl:campaigns-table:database",+ "fallback-validation:ticket-columns:none",+ "unclassified",+ "unclassified",+ )+ _require_fixed(+ labels_are_exact,+ "Startup diagnostic stages must remain distinct and fixed",+ )+ public_messages_are_generic = (+ str(ownership_failure) == "PostgreSQL schema validation failed"+ and str(ddl_failure) == "PostgreSQL schema validation failed"+ and str(validation_failure)+ == "Incompatible WebSocket ticket schema"+ )+ _require_fixed(+ public_messages_are_generic,+ "Startup diagnostics must preserve generic public messages",+ )+ rendered = " ".join(+ (+ *labels,+ str(ownership_failure),+ repr(ownership_failure),+ str(ddl_failure),+ repr(ddl_failure),+ str(validation_failure),+ repr(validation_failure),+ )+ )+ _require_fixed(+ canary not in rendered,+ "Startup diagnostics must not expose source content",+ )+++async def test_postgres_startup_cleanup_preserves_primary_diagnostic() -> None:+ from ares.db.postgres import (+ PostgresDatabase,+ _postgres_startup_diagnostic_label,+ _postgres_startup_error,+ )++ canary = "cleanup-canary-not-for-output"++ class _FailingPool:+ async def close(self) -> None:+ raise RuntimeError(canary)++ database = PostgresDatabase("synthetic")+ pool = _FailingPool()+ database._pool = pool+ primary = _postgres_startup_error(+ "PostgreSQL schema validation failed",+ stage="fallback-validation",+ invariant="ticket-index-inventory",+ )+ await database._close_failed_startup_pool(pool, primary)++ primary_is_preserved = (+ _postgres_startup_diagnostic_label(primary)+ == "fallback-validation:ticket-index-inventory:none"+ )+ _require_fixed(+ primary_is_preserved,+ "Cleanup failure must not replace the primary diagnostic",+ )+ notes = tuple(getattr(primary, "__notes__", ()))+ notes_are_sanitized = (+ len(notes) == 1+ and notes[0]+ == "PostgreSQL startup cleanup failure [pool-close: runtime]"+ and canary not in " ".join(notes)+ )+ _require_fixed(+ notes_are_sanitized,+ "Cleanup diagnostics must remain fixed and sanitized",+ )+ _require_fixed(+ database._pool is None,+ "Failed startup cleanup must clear the candidate pool",+ )++@pytest.mark.asyncioasync def test_postgres_runtime_schema_and_transaction_smoke() -> None:config = _postgres_test_config()@@ -126,6 +328,61 @@ async def test_postgres_runtime_schema_and_transaction_smoke() -> None:except Exception as exc:raise _sanitized_setup_failure("database-create", exc) from None+ target_probe = None+ try:+ target_probe = await asyncpg.connect(+ host=config["host"],+ port=config["port"],+ user=config["user"],+ database=test_database,+ )+ initial_application_count = int(+ await target_probe.fetchval(+ """+ SELECT COUNT(*)+ FROM information_schema.tables+ WHERE table_schema=current_schema()+ """+ )+ )+ initial_version_relation = bool(+ await target_probe.fetchval(+ """+ SELECT EXISTS(+ SELECT 1+ FROM pg_class AS rel+ JOIN pg_namespace AS nsp+ ON nsp.oid=rel.relnamespace+ WHERE nsp.nspname=current_schema()+ AND rel.relname='alembic_version'+ )+ """+ )+ )+ except Exception as exc:+ raise _sanitized_setup_failure(+ "empty-catalog-probe",+ exc,+ ) from None+ finally:+ if target_probe is not None:+ try:+ await target_probe.close()+ except Exception as exc:+ raise _sanitized_setup_failure(+ "empty-catalog-probe-close",+ exc,+ ) from None++ _require_fixed(+ initial_application_count == 0,+ "Runtime smoke database must begin without application tables",+ )+ _require_fixed(+ not initial_version_relation,+ "Runtime smoke database must begin without Alembic ownership",+ )+database = PostgresDatabase(_runtime_dsn(config, test_database),pool_min=1,@@ -136,8 +393,35 @@ async def test_postgres_runtime_schema_and_transaction_smoke() -> None:except Exception as exc:raise _sanitized_setup_failure("runtime-initialize", exc) from Noneruntime_connected = True+ initial_trace_is_exact = tuple(database._startup_trace) == (+ "fallback-entered",+ "fallback-ddl-complete",+ "startup-ready",+ )+ _require_fixed(+ initial_trace_is_exact,+ "Unversioned runtime startup must complete the fallback path",+ )async with database._pool.acquire() as connection:+ version_relation_exists = bool(+ await connection.fetchval(+ """+ SELECT EXISTS(+ SELECT 1+ FROM pg_class AS rel+ JOIN pg_namespace AS nsp+ ON nsp.oid=rel.relnamespace+ WHERE nsp.nspname=current_schema()+ AND rel.relname='alembic_version'+ )+ """+ )+ )+ _require_fixed(+ not version_relation_exists,+ "Runtime fallback must not stamp Alembic ownership",+ )table_rows = await connection.fetch("""SELECT table_name@@ -252,6 +536,39 @@ async def test_postgres_runtime_schema_and_transaction_smoke() -> None:async with database._pool.acquire() as reusable_connection:assert await reusable_connection.fetchval("SELECT 1") == 1++ await database.close()+ runtime_connected = False+ database = PostgresDatabase(+ _runtime_dsn(config, test_database),+ pool_min=1,+ pool_max=2,+ )+ try:+ await database.connect()+ except Exception as exc:+ raise _sanitized_setup_failure(+ "runtime-reconnect",+ exc,+ ) from None+ runtime_connected = True+ reconnect_trace_is_exact = tuple(database._startup_trace) == (+ "fallback-entered",+ "fallback-ddl-complete",+ "startup-ready",+ )+ _require_fixed(+ reconnect_trace_is_exact,+ "Unversioned runtime reconnect must complete the fallback path",+ )+ async with database._pool.acquire() as reusable_connection:+ reusable_after_reconnect = (+ await reusable_connection.fetchval("SELECT 1") == 1+ )+ _require_fixed(+ reusable_after_reconnect,+ "Runtime pool must remain reusable after reconnect",+ )finally:primary_failure = sys.exception()