Skip to content

armgit / ARES

Public Imported from GitHub
0

ARES - authorized red-team engagement automation with dashboard, campaign scope, module orchestration, OPSEC controls, encrypted vault, and reporting.

chore(db): expose PostgreSQL startup diagnostics

Mafifrizi808c7717a92f
diff --git a/ares/db/postgres.py b/ares/db/postgres.py
index c1e54c5..6107202 100644
--- a/ares/db/postgres.py
+++ b/ares/db/postgres.py
@@ -321,6 +321,254 @@ CREATE INDEX IF NOT EXISTS idx_ws_tickets_api_key
ON websocket_tickets(api_key_id);
"""
+_POSTGRES_FALLBACK_DDL_CODES = (
+ "campaigns-table",
+ "module-runs-table",
+ "module-runs-campaign-index",
+ "module-runs-completed-index",
+ "findings-table",
+ "findings-campaign-index",
+ "findings-severity-index",
+ "findings-false-positive-index",
+ "hosts-table",
+ "hosts-campaign-index",
+ "hosts-ip-index",
+ "credentials-table",
+ "credentials-campaign-index",
+ "loot-table",
+ "audit-log-table",
+ "audit-log-campaign-index",
+ "users-table",
+ "users-username-index",
+ "api-keys-table",
+ "api-keys-user-index",
+ "api-keys-prefix-index",
+ "refresh-tokens-table",
+ "refresh-tokens-user-index",
+ "refresh-tokens-expiry-index",
+ "revoked-access-tokens-table",
+ "revoked-access-tokens-expiry-index",
+ "websocket-tickets-table",
+ "websocket-tickets-expiry-index",
+ "websocket-tickets-user-index",
+ "websocket-tickets-campaign-index",
+ "websocket-tickets-api-key-index",
+)
+
+
+def _postgres_fallback_statement_spans() -> tuple[
+ tuple[str, int, int], ...
+]:
+ spans: list[tuple[str, int, int]] = []
+ start = 0
+ for code in _POSTGRES_FALLBACK_DDL_CODES:
+ end = _PG_CREATE_TABLES.find(";", start)
+ if end < 0:
+ raise RuntimeError("Invalid PostgreSQL fallback DDL contract")
+ spans.append((code, start + 1, end + 1))
+ start = end + 1
+ if _PG_CREATE_TABLES[start:].strip():
+ raise RuntimeError("Invalid PostgreSQL fallback DDL contract")
+ return tuple(spans)
+
+
+_POSTGRES_FALLBACK_STATEMENT_SPANS = _postgres_fallback_statement_spans()
+_POSTGRES_STARTUP_STAGES = frozenset(
+ {
+ "ownership",
+ "fallback-ddl",
+ "fallback-validation",
+ "managed-validation",
+ "unclassified",
+ }
+)
+_POSTGRES_STARTUP_OPERATIONAL_CATEGORIES = frozenset(
+ {"none", "timeout", "connection", "database", "runtime", "other"}
+)
+_POSTGRES_TICKET_INVARIANTS = frozenset(
+ {
+ "ticket-relation-present",
+ "ticket-relation-metadata",
+ "ticket-columns",
+ "ticket-constraint-inventory",
+ "ticket-canonical-opclasses",
+ "ticket-index-identity",
+ "ticket-index-inventory",
+ "ticket-constraint-definition-primary",
+ "ticket-constraint-definition-campaign",
+ "ticket-constraint-definition-user",
+ "ticket-constraint-definition-api-key",
+ "ticket-reference-inventory",
+ "ticket-constraint-table-binding",
+ "ticket-foreign-key-campaign",
+ "ticket-foreign-key-user",
+ "ticket-foreign-key-api-key",
+ "ticket-primary-key-binding",
+ "ticket-check-hash",
+ "ticket-check-kind",
+ "ticket-check-created-at",
+ "ticket-check-expires-at",
+ "ticket-check-consumed-at",
+ "ticket-check-bearer-expires-finite",
+ "ticket-check-created-finite",
+ "ticket-check-expires-finite",
+ "ticket-check-consumed-finite",
+ "ticket-check-time-order",
+ "ticket-check-source-shape",
+ "ticket-validation-unclassified",
+ }
+)
+_POSTGRES_OWNERSHIP_INVARIANTS = frozenset(
+ {
+ "ownership-relation-query",
+ "ownership-relation-metadata",
+ "ownership-revision-query",
+ "ownership-revision-cardinality",
+ "ownership-revision-value",
+ "ownership-known-older-revision",
+ "ownership-unclassified",
+ }
+)
+_POSTGRES_STARTUP_INVARIANTS = (
+ frozenset(_POSTGRES_FALLBACK_DDL_CODES)
+ | _POSTGRES_TICKET_INVARIANTS
+ | _POSTGRES_OWNERSHIP_INVARIANTS
+ | frozenset({"fallback-ddl-unclassified", "managed-validation-unclassified"})
+)
+_POSTGRES_STARTUP_TRACE_CODES = frozenset(
+ {"fallback-entered", "fallback-ddl-complete", "startup-ready"}
+)
+
+
+def _postgres_operational_category(error: BaseException) -> str:
+ if isinstance(error, TimeoutError):
+ return "timeout"
+ if isinstance(error, ConnectionError):
+ return "connection"
+ error_type_names = {
+ base.__name__
+ for base in type(error).__mro__
+ }
+ if error_type_names & {
+ "PostgresError",
+ "InterfaceError",
+ "ConnectionDoesNotExistError",
+ "CannotConnectNowError",
+ }:
+ return "database"
+ if isinstance(error, RuntimeError):
+ return "runtime"
+ return "other"
+
+
+class _PostgresStartupDiagnosticError(RuntimeError):
+ def __init__(
+ self,
+ public_message: str,
+ *,
+ diagnostic_stage: str,
+ diagnostic_invariant: str,
+ operational_category: str = "none",
+ ) -> None:
+ super().__init__(public_message)
+ self.diagnostic_stage = (
+ diagnostic_stage
+ if diagnostic_stage in _POSTGRES_STARTUP_STAGES
+ else "unclassified"
+ )
+ self.diagnostic_invariant = (
+ diagnostic_invariant
+ if diagnostic_invariant in _POSTGRES_STARTUP_INVARIANTS
+ else "ownership-unclassified"
+ )
+ self.operational_category = (
+ operational_category
+ if operational_category
+ in _POSTGRES_STARTUP_OPERATIONAL_CATEGORIES
+ else "other"
+ )
+
+
+def _postgres_startup_diagnostic_label(error: BaseException) -> str:
+ if not isinstance(error, _PostgresStartupDiagnosticError):
+ return "unclassified"
+ if error.diagnostic_invariant in {
+ "ownership-unclassified",
+ "fallback-ddl-unclassified",
+ "managed-validation-unclassified",
+ "ticket-validation-unclassified",
+ }:
+ return "unclassified"
+ if (
+ error.diagnostic_stage not in _POSTGRES_STARTUP_STAGES
+ or error.diagnostic_invariant not in _POSTGRES_STARTUP_INVARIANTS
+ or error.operational_category
+ not in _POSTGRES_STARTUP_OPERATIONAL_CATEGORIES
+ ):
+ return "unclassified"
+ return (
+ f"{error.diagnostic_stage}:"
+ f"{error.diagnostic_invariant}:"
+ f"{error.operational_category}"
+ )
+
+
+def _postgres_startup_error(
+ public_message: str,
+ *,
+ stage: str,
+ invariant: str,
+ cause: BaseException | None = None,
+) -> _PostgresStartupDiagnosticError:
+ category = (
+ "none"
+ if cause is None
+ else _postgres_operational_category(cause)
+ )
+ return _PostgresStartupDiagnosticError(
+ public_message,
+ diagnostic_stage=stage,
+ diagnostic_invariant=invariant,
+ operational_category=category,
+ )
+
+
+def _postgres_fallback_failure_invariant(error: BaseException) -> str:
+ raw_position = getattr(error, "position", None)
+ if not isinstance(raw_position, (int, str)):
+ return "fallback-ddl-unclassified"
+ try:
+ position = int(raw_position)
+ except (TypeError, ValueError):
+ return "fallback-ddl-unclassified"
+ for code, start, end in _POSTGRES_FALLBACK_STATEMENT_SPANS:
+ if start <= position <= end:
+ return code
+ return "fallback-ddl-unclassified"
+
+
+def _postgres_restage_startup_error(
+ error: _PostgresStartupDiagnosticError,
+ stage: str,
+) -> _PostgresStartupDiagnosticError:
+ return _PostgresStartupDiagnosticError(
+ str(error),
+ diagnostic_stage=stage,
+ diagnostic_invariant=error.diagnostic_invariant,
+ operational_category=error.operational_category,
+ )
+
+
+def _postgres_ticket_schema_error(
+ invariant: str,
+) -> _PostgresStartupDiagnosticError:
+ return _postgres_startup_error(
+ "Incompatible WebSocket ticket schema",
+ stage="managed-validation",
+ invariant=invariant,
+ )
+
+
_POSTGRES_MANAGED_REVISION = "0008"
_POSTGRES_OLDER_REVISIONS = frozenset(
{"0001", "0002", "0003", "0004", "0005", "0006", "0007"}
@@ -330,11 +578,11 @@ _POSTGRES_MANAGED_SCHEMA_ERROR = "Incompatible managed PostgreSQL schema"
_POSTGRES_SCHEMA_VALIDATION_ERROR = "PostgreSQL schema validation failed"
-class _PostgresMigrationRequiredError(RuntimeError):
+class _PostgresMigrationRequiredError(_PostgresStartupDiagnosticError):
pass
-class _PostgresManagedSchemaError(RuntimeError):
+class _PostgresManagedSchemaError(_PostgresStartupDiagnosticError):
pass
@@ -561,16 +809,32 @@ def _postgres_index_opclass(table: str, column: str) -> str:
def _classify_postgres_revision(values: tuple[object, ...]) -> str:
- if len(values) != 1 or not isinstance(values[0], str):
- raise _PostgresManagedSchemaError(_POSTGRES_MANAGED_SCHEMA_ERROR)
+ if len(values) != 1:
+ raise _PostgresManagedSchemaError(
+ _POSTGRES_MANAGED_SCHEMA_ERROR,
+ diagnostic_stage="ownership",
+ diagnostic_invariant="ownership-revision-cardinality",
+ )
+ if not isinstance(values[0], str):
+ raise _PostgresManagedSchemaError(
+ _POSTGRES_MANAGED_SCHEMA_ERROR,
+ diagnostic_stage="ownership",
+ diagnostic_invariant="ownership-revision-value",
+ )
revision = values[0]
if revision == _POSTGRES_MANAGED_REVISION:
return revision
if revision in _POSTGRES_OLDER_REVISIONS:
raise _PostgresMigrationRequiredError(
- _POSTGRES_MIGRATION_REQUIRED_ERROR
+ _POSTGRES_MIGRATION_REQUIRED_ERROR,
+ diagnostic_stage="ownership",
+ diagnostic_invariant="ownership-known-older-revision",
)
- raise _PostgresManagedSchemaError(_POSTGRES_MANAGED_SCHEMA_ERROR)
+ raise _PostgresManagedSchemaError(
+ _POSTGRES_MANAGED_SCHEMA_ERROR,
+ diagnostic_stage="ownership",
+ diagnostic_invariant="ownership-revision-value",
+ )
class PostgresDatabase:
@@ -603,6 +867,7 @@ class PostgresDatabase:
self._pool_min = pool_min
self._pool_max = pool_max
self._pool: Any = None # asyncpg.Pool
+ self._startup_trace: list[str] = []
# ── Lifecycle ──────────────────────────────────────────────────────────────
@@ -629,6 +894,7 @@ class PostgresDatabase:
except asyncio.CancelledError as primary:
await self._close_failed_startup_pool(pool, primary)
raise
+ self._record_startup_trace("startup-ready")
logger.info("pg_db_ready")
return self
@@ -640,14 +906,20 @@ class PostgresDatabase:
try:
await pool.close()
except Exception as cleanup_error:
+ cleanup_category = _postgres_operational_category(cleanup_error)
primary.add_note(
"PostgreSQL startup cleanup failure "
- f"[pool-close: {type(cleanup_error).__name__}]"
+ f"[pool-close: {cleanup_category}]"
)
finally:
if self._pool is pool:
self._pool = None
+ def _record_startup_trace(self, code: str) -> None:
+ if code not in _POSTGRES_STARTUP_TRACE_CODES:
+ raise RuntimeError("Invalid PostgreSQL startup trace identifier")
+ self._startup_trace.append(code)
+
async def __aenter__(self) -> "PostgresDatabase":
return await self.connect()
@@ -669,9 +941,10 @@ class PostgresDatabase:
@staticmethod
async def _managed_schema_revision(connection: Any) -> str | None:
- relation_rows = await connection.fetch(
- """
- SELECT rel.oid::bigint AS table_oid,
+ try:
+ relation_rows = await connection.fetch(
+ """
+ SELECT rel.oid::bigint AS table_oid,
nsp.nspname AS schema_name,
relation_type.oid::bigint AS type_oid,
type_nsp.nspname AS type_schema,
@@ -715,14 +988,27 @@ class PostgresDatabase:
ON relation_type.oid=rel.reltype
LEFT JOIN pg_namespace AS type_nsp
ON type_nsp.oid=relation_type.typnamespace
- WHERE nsp.nspname=current_schema()
- AND rel.relname='alembic_version'
- """
- )
+ WHERE nsp.nspname=current_schema()
+ AND rel.relname='alembic_version'
+ """
+ )
+ except asyncio.CancelledError:
+ raise
+ except Exception as exc:
+ raise _postgres_startup_error(
+ _POSTGRES_SCHEMA_VALIDATION_ERROR,
+ stage="ownership",
+ invariant="ownership-relation-query",
+ cause=exc,
+ ) from None
if not relation_rows:
return None
if len(relation_rows) != 1:
- raise RuntimeError(_POSTGRES_MANAGED_SCHEMA_ERROR)
+ raise _PostgresManagedSchemaError(
+ _POSTGRES_MANAGED_SCHEMA_ERROR,
+ diagnostic_stage="ownership",
+ diagnostic_invariant="ownership-relation-metadata",
+ )
relation = relation_rows[0]
try:
table_oid = int(relation["table_oid"])
@@ -747,7 +1033,11 @@ class PostgresDatabase:
int(relation["user_rule_count"]),
)
except (KeyError, TypeError, ValueError):
- raise RuntimeError(_POSTGRES_MANAGED_SCHEMA_ERROR) from None
+ raise _PostgresManagedSchemaError(
+ _POSTGRES_MANAGED_SCHEMA_ERROR,
+ diagnostic_stage="ownership",
+ diagnostic_invariant="ownership-relation-metadata",
+ ) from None
if (
not isinstance(schema_name, str)
or not schema_name
@@ -757,7 +1047,11 @@ class PostgresDatabase:
or relation_contract
!= ("r", "p", False, False, False, 0, 0, 0, 0, 0)
):
- raise RuntimeError(_POSTGRES_MANAGED_SCHEMA_ERROR)
+ raise _PostgresManagedSchemaError(
+ _POSTGRES_MANAGED_SCHEMA_ERROR,
+ diagnostic_stage="ownership",
+ diagnostic_invariant="ownership-relation-metadata",
+ )
column_rows = await connection.fetch(
"""
@@ -810,7 +1104,11 @@ class PostgresDatabase:
for row in column_rows
)
except (KeyError, TypeError, ValueError):
- raise RuntimeError(_POSTGRES_MANAGED_SCHEMA_ERROR) from None
+ raise _PostgresManagedSchemaError(
+ _POSTGRES_MANAGED_SCHEMA_ERROR,
+ diagnostic_stage="ownership",
+ diagnostic_invariant="ownership-relation-metadata",
+ ) from None
if columns != (
(
1,
@@ -828,7 +1126,11 @@ class PostgresDatabase:
None,
),
):
- raise RuntimeError(_POSTGRES_MANAGED_SCHEMA_ERROR)
+ raise _PostgresManagedSchemaError(
+ _POSTGRES_MANAGED_SCHEMA_ERROR,
+ diagnostic_stage="ownership",
+ diagnostic_invariant="ownership-relation-metadata",
+ )
primary_rows = await connection.fetch(
"""
@@ -865,7 +1167,11 @@ class PostgresDatabase:
for row in primary_rows
)
except (KeyError, TypeError, ValueError):
- raise RuntimeError(_POSTGRES_MANAGED_SCHEMA_ERROR) from None
+ raise _PostgresManagedSchemaError(
+ _POSTGRES_MANAGED_SCHEMA_ERROR,
+ diagnostic_stage="ownership",
+ diagnostic_invariant="ownership-relation-metadata",
+ ) from None
if primary_contract != (
(
"alembic_version_pkc",
@@ -876,7 +1182,11 @@ class PostgresDatabase:
("version_num",),
),
):
- raise RuntimeError(_POSTGRES_MANAGED_SCHEMA_ERROR)
+ raise _PostgresManagedSchemaError(
+ _POSTGRES_MANAGED_SCHEMA_ERROR,
+ diagnostic_stage="ownership",
+ diagnostic_invariant="ownership-relation-metadata",
+ )
version_index_rows = await connection.fetch(
"""
@@ -983,7 +1293,11 @@ class PostgresDatabase:
for row in version_index_rows
)
except (KeyError, TypeError, ValueError):
- raise RuntimeError(_POSTGRES_MANAGED_SCHEMA_ERROR) from None
+ raise _PostgresManagedSchemaError(
+ _POSTGRES_MANAGED_SCHEMA_ERROR,
+ diagnostic_stage="ownership",
+ diagnostic_invariant="ownership-relation-metadata",
+ ) from None
if version_indexes != (
(
"alembic_version_pkc",
@@ -1008,16 +1322,34 @@ class PostgresDatabase:
None,
),
):
- raise RuntimeError(_POSTGRES_MANAGED_SCHEMA_ERROR)
+ raise _PostgresManagedSchemaError(
+ _POSTGRES_MANAGED_SCHEMA_ERROR,
+ diagnostic_stage="ownership",
+ diagnostic_invariant="ownership-relation-metadata",
+ )
quoted_schema = '"' + schema_name.replace('"', '""') + '"'
- revision_rows = await connection.fetch(
- f'SELECT version_num FROM {quoted_schema}."alembic_version"' # noqa: S608
- )
+ try:
+ revision_rows = await connection.fetch(
+ f'SELECT version_num FROM {quoted_schema}."alembic_version"' # noqa: S608
+ )
+ except asyncio.CancelledError:
+ raise
+ except Exception as exc:
+ raise _postgres_startup_error(
+ _POSTGRES_SCHEMA_VALIDATION_ERROR,
+ stage="ownership",
+ invariant="ownership-revision-query",
+ cause=exc,
+ ) from None
try:
values = tuple(row["version_num"] for row in revision_rows)
except (KeyError, TypeError):
- raise RuntimeError(_POSTGRES_MANAGED_SCHEMA_ERROR) from None
+ raise _PostgresManagedSchemaError(
+ _POSTGRES_MANAGED_SCHEMA_ERROR,
+ diagnostic_stage="ownership",
+ diagnostic_invariant="ownership-revision-cardinality",
+ ) from None
return _classify_postgres_revision(values)
@staticmethod
@@ -1946,61 +2278,117 @@ class PostgresDatabase:
except (
_PostgresManagedSchemaError,
_PostgresMigrationRequiredError,
+ _PostgresStartupDiagnosticError,
):
raise
- except RuntimeError as exc:
- if str(exc) == _POSTGRES_MANAGED_SCHEMA_ERROR:
- raise _PostgresManagedSchemaError(
- _POSTGRES_MANAGED_SCHEMA_ERROR
- ) from None
- raise RuntimeError(
- _POSTGRES_SCHEMA_VALIDATION_ERROR
- ) from None
- except Exception:
- raise RuntimeError(
- _POSTGRES_SCHEMA_VALIDATION_ERROR
+ except asyncio.CancelledError:
+ raise
+ except Exception as exc:
+ raise _postgres_startup_error(
+ _POSTGRES_SCHEMA_VALIDATION_ERROR,
+ stage="ownership",
+ invariant="ownership-relation-metadata",
+ cause=exc,
) from None
if revision is not None:
try:
await self._validate_managed_schema(conn)
await self._validate_websocket_ticket_schema(conn)
- except _PostgresManagedSchemaError:
+ except _PostgresMigrationRequiredError:
+ raise
+ except _PostgresStartupDiagnosticError as exc:
+ raise _postgres_restage_startup_error(
+ exc,
+ "managed-validation",
+ ) from None
+ except asyncio.CancelledError:
raise
except RuntimeError as exc:
if str(exc) == _POSTGRES_MANAGED_SCHEMA_ERROR:
raise _PostgresManagedSchemaError(
- _POSTGRES_MANAGED_SCHEMA_ERROR
+ _POSTGRES_MANAGED_SCHEMA_ERROR,
+ diagnostic_stage="managed-validation",
+ diagnostic_invariant=(
+ "managed-validation-unclassified"
+ ),
) from None
- if str(exc) == (
- "Incompatible WebSocket ticket schema"
- ):
- raise
- raise RuntimeError(
- _POSTGRES_SCHEMA_VALIDATION_ERROR
+ raise _postgres_startup_error(
+ _POSTGRES_SCHEMA_VALIDATION_ERROR,
+ stage="managed-validation",
+ invariant="managed-validation-unclassified",
+ cause=exc,
) from None
- except Exception:
- raise RuntimeError(
- _POSTGRES_SCHEMA_VALIDATION_ERROR
+ except Exception as exc:
+ raise _postgres_startup_error(
+ _POSTGRES_SCHEMA_VALIDATION_ERROR,
+ stage="managed-validation",
+ invariant="managed-validation-unclassified",
+ cause=exc,
) from None
else:
- ticket_table_exists = bool(
- await conn.fetchval(
- """
- SELECT EXISTS(
- SELECT 1
- FROM pg_class AS rel
- JOIN pg_namespace AS nsp
- ON nsp.oid=rel.relnamespace
- WHERE nsp.nspname=current_schema()
- AND rel.relname='websocket_tickets'
+ self._record_startup_trace("fallback-entered")
+ try:
+ ticket_table_exists = bool(
+ await conn.fetchval(
+ """
+ SELECT EXISTS(
+ SELECT 1
+ FROM pg_class AS rel
+ JOIN pg_namespace AS nsp
+ ON nsp.oid=rel.relnamespace
+ WHERE nsp.nspname=current_schema()
+ AND rel.relname='websocket_tickets'
+ )
+ """
)
- """
)
- )
+ except asyncio.CancelledError:
+ raise
+ except Exception as exc:
+ raise _postgres_startup_error(
+ _POSTGRES_SCHEMA_VALIDATION_ERROR,
+ stage="fallback-validation",
+ invariant="ticket-relation-present",
+ cause=exc,
+ ) from None
if ticket_table_exists:
+ try:
+ await self._validate_websocket_ticket_schema(conn)
+ except _PostgresStartupDiagnosticError as exc:
+ raise _postgres_restage_startup_error(
+ exc,
+ "fallback-validation",
+ ) from None
+ try:
+ await conn.execute(_PG_CREATE_TABLES)
+ except asyncio.CancelledError:
+ raise
+ except Exception as exc:
+ raise _postgres_startup_error(
+ _POSTGRES_SCHEMA_VALIDATION_ERROR,
+ stage="fallback-ddl",
+ invariant=_postgres_fallback_failure_invariant(
+ exc
+ ),
+ cause=exc,
+ ) from None
+ self._record_startup_trace("fallback-ddl-complete")
+ try:
await self._validate_websocket_ticket_schema(conn)
- await conn.execute(_PG_CREATE_TABLES)
- await self._validate_websocket_ticket_schema(conn)
+ except _PostgresStartupDiagnosticError as exc:
+ raise _postgres_restage_startup_error(
+ exc,
+ "fallback-validation",
+ ) from None
+ except asyncio.CancelledError:
+ raise
+ except Exception as exc:
+ raise _postgres_startup_error(
+ _POSTGRES_SCHEMA_VALIDATION_ERROR,
+ stage="fallback-validation",
+ invariant="ticket-validation-unclassified",
+ cause=exc,
+ ) from None
logger.info("pg_schema_ready")
@staticmethod
@@ -2049,7 +2437,9 @@ class PostgresDatabase:
"""
)
if relation is None:
- raise RuntimeError("Incompatible WebSocket ticket schema")
+ raise _postgres_ticket_schema_error(
+ "ticket-relation-present"
+ )
table_oid = int(relation["table_oid"])
schema_oid = int(relation["schema_oid"])
schema_name = str(relation["schema_name"])
@@ -2077,7 +2467,9 @@ class PostgresDatabase:
0,
0,
):
- raise RuntimeError("Incompatible WebSocket ticket schema")
+ raise _postgres_ticket_schema_error(
+ "ticket-relation-metadata"
+ )
rows = await connection.fetch(
"""
@@ -2158,7 +2550,7 @@ class PostgresDatabase:
),
]
if actual_columns != expected_columns:
- raise RuntimeError("Incompatible WebSocket ticket schema")
+ raise _postgres_ticket_schema_error("ticket-columns")
constraint_rows = await connection.fetch(
"""
@@ -2219,7 +2611,9 @@ class PostgresDatabase:
"websocket_tickets_pkey",
}
if set(actual_constraints) != expected_constraint_names:
- raise RuntimeError("Incompatible WebSocket ticket schema")
+ raise _postgres_ticket_schema_error(
+ "ticket-constraint-inventory"
+ )
index_rows = await connection.fetch(
"""
@@ -2316,7 +2710,9 @@ class PostgresDatabase:
for row in canonical_opclass_rows
}
if set(canonical_opclasses) != {"text_ops", "timestamptz_ops"}:
- raise RuntimeError("Incompatible WebSocket ticket schema")
+ raise _postgres_ticket_schema_error(
+ "ticket-canonical-opclasses"
+ )
index_oids: set[int] = set()
for row in index_rows:
@@ -2348,7 +2744,9 @@ class PostgresDatabase:
and index_oid not in index_oids
)
if not identity_is_exact:
- raise RuntimeError("Incompatible WebSocket ticket schema")
+ raise _postgres_ticket_schema_error(
+ "ticket-index-identity"
+ )
index_oids.add(index_oid)
actual_indexes = {
@@ -2397,7 +2795,9 @@ class PostgresDatabase:
),
}
if actual_indexes != expected_indexes:
- raise RuntimeError("Incompatible WebSocket ticket schema")
+ raise _postgres_ticket_schema_error(
+ "ticket-index-inventory"
+ )
expected_constraint_fragments = {
"websocket_tickets_pkey": ("p", "PRIMARY KEY (ticket_hash)"),
@@ -2414,6 +2814,20 @@ class PostgresDatabase:
"FOREIGN KEY (api_key_id) REFERENCES api_keys(id) ON DELETE CASCADE",
),
}
+ constraint_fragment_invariants = {
+ "websocket_tickets_pkey": (
+ "ticket-constraint-definition-primary"
+ ),
+ "fk_ws_ticket_campaign": (
+ "ticket-constraint-definition-campaign"
+ ),
+ "fk_ws_ticket_user": (
+ "ticket-constraint-definition-user"
+ ),
+ "fk_ws_ticket_api_key": (
+ "ticket-constraint-definition-api-key"
+ ),
+ }
for name, definition in expected_constraint_fragments.items():
row = actual_constraints[name]
if (
@@ -2423,7 +2837,9 @@ class PostgresDatabase:
or bool(row["condeferred"])
or " ".join(str(row["definition"]).split()) != definition[1]
):
- raise RuntimeError("Incompatible WebSocket ticket schema")
+ raise _postgres_ticket_schema_error(
+ constraint_fragment_invariants[name]
+ )
referenced_schema = str(await connection.fetchval("SELECT current_schema()"))
expected_foreign_keys = {
@@ -2452,14 +2868,23 @@ class PostgresDatabase:
for row in referenced_rows
}
if set(referenced_oids) != {"api_keys", "campaigns", "users"}:
- raise RuntimeError("Incompatible WebSocket ticket schema")
+ raise _postgres_ticket_schema_error(
+ "ticket-reference-inventory"
+ )
if any(
int(row["table_oid"]) != table_oid
for row in actual_constraints.values()
):
- raise RuntimeError("Incompatible WebSocket ticket schema")
+ raise _postgres_ticket_schema_error(
+ "ticket-constraint-table-binding"
+ )
+ foreign_key_invariants = {
+ "fk_ws_ticket_campaign": "ticket-foreign-key-campaign",
+ "fk_ws_ticket_user": "ticket-foreign-key-user",
+ "fk_ws_ticket_api_key": "ticket-foreign-key-api-key",
+ }
for name, expected in expected_foreign_keys.items():
row = actual_constraints[name]
actual = (
@@ -2477,7 +2902,9 @@ class PostgresDatabase:
== referenced_oids[expected[2]]
)
if actual != expected or not reference_identity_is_exact:
- raise RuntimeError("Incompatible WebSocket ticket schema")
+ raise _postgres_ticket_schema_error(
+ foreign_key_invariants[name]
+ )
primary = actual_constraints["websocket_tickets_pkey"]
if (
@@ -2493,7 +2920,9 @@ class PostgresDatabase:
if str(row["index_name"]) == "websocket_tickets_pkey"
)
):
- raise RuntimeError("Incompatible WebSocket ticket schema")
+ raise _postgres_ticket_schema_error(
+ "ticket-primary-key-binding"
+ )
expected_check_definitions = {
"ck_ws_ticket_hash": (
@@ -2542,6 +2971,27 @@ class PostgresDatabase:
"AND required_scope = 'read'::text)"
),
}
+ check_invariants = {
+ "ck_ws_ticket_hash": "ticket-check-hash",
+ "ck_ws_ticket_kind": "ticket-check-kind",
+ "ck_ws_ticket_created_at": "ticket-check-created-at",
+ "ck_ws_ticket_expires_at": "ticket-check-expires-at",
+ "ck_ws_ticket_consumed_at": "ticket-check-consumed-at",
+ "ck_ws_ticket_bearer_expires_finite": (
+ "ticket-check-bearer-expires-finite"
+ ),
+ "ck_ws_ticket_created_finite": (
+ "ticket-check-created-finite"
+ ),
+ "ck_ws_ticket_expires_finite": (
+ "ticket-check-expires-finite"
+ ),
+ "ck_ws_ticket_consumed_finite": (
+ "ticket-check-consumed-finite"
+ ),
+ "ck_ws_ticket_time_order": "ticket-check-time-order",
+ "ck_ws_ticket_source_shape": "ticket-check-source-shape",
+ }
for name, expected_definition in expected_check_definitions.items():
row = actual_constraints[name]
if (
@@ -2552,7 +3002,9 @@ class PostgresDatabase:
or " ".join(str(row["definition"]).split())
!= expected_definition
):
- raise RuntimeError("Incompatible WebSocket ticket schema")
+ raise _postgres_ticket_schema_error(
+ check_invariants[name]
+ )
# ── Helpers ────────────────────────────────────────────────────────────────
diff --git a/tests/integration/test_postgres_auth_token_lifecycle.py b/tests/integration/test_postgres_auth_token_lifecycle.py
index 14fe743..960073a 100644
--- a/tests/integration/test_postgres_auth_token_lifecycle.py
+++ b/tests/integration/test_postgres_auth_token_lifecycle.py
@@ -80,8 +80,19 @@ def _runtime_dsn(config: dict[str, str | int], database: str) -> str:
def _sanitized_setup_failure(action: str, exc: Exception) -> RuntimeError:
+ from ares.db.postgres import _postgres_operational_category
+
+ if action == "runtime-initialize":
+ from ares.db.postgres import _postgres_startup_diagnostic_label
+
+ diagnostic = _postgres_startup_diagnostic_label(exc)
+ return RuntimeError(
+ "PostgreSQL token-test setup failed "
+ f"[runtime-initialize:{diagnostic}]"
+ )
+ category = _postgres_operational_category(exc)
return RuntimeError(
- f"PostgreSQL token-test setup failed [{action}: {type(exc).__name__}]"
+ f"PostgreSQL token-test setup failed [{action}:{category}]"
)
diff --git a/tests/integration/test_postgres_runtime_smoke.py b/tests/integration/test_postgres_runtime_smoke.py
index a6b9dab..1b1fd7c 100644
--- a/tests/integration/test_postgres_runtime_smoke.py
+++ b/tests/integration/test_postgres_runtime_smoke.py
@@ -24,6 +24,15 @@ class _RollbackProbe(RuntimeError):
"""Intentional exception used to verify a real transaction rollback."""
+class PostgresError(RuntimeError):
+ """Synthetic database-category error for diagnostic-only tests."""
+
+
+def _require_fixed(condition: bool, message: str) -> None:
+ if not condition:
+ pytest.fail(message, pytrace=False)
+
+
def _postgres_test_config() -> dict[str, str | int]:
values = {name: os.environ.get(name) for name in _POSTGRES_ENV}
configured = [name for name, value in values.items() if value]
@@ -64,7 +73,20 @@ def _runtime_dsn(config: dict[str, str | int], database: str) -> str:
def _sanitized_setup_failure(action: str, exc: Exception) -> RuntimeError:
- return RuntimeError(f"PostgreSQL smoke setup failed [{action}: {type(exc).__name__}]")
+ from ares.db.postgres import _postgres_operational_category
+
+ if action == "runtime-initialize":
+ from ares.db.postgres import _postgres_startup_diagnostic_label
+
+ diagnostic = _postgres_startup_diagnostic_label(exc)
+ return RuntimeError(
+ "PostgreSQL smoke setup failed "
+ f"[runtime-initialize:{diagnostic}]"
+ )
+ category = _postgres_operational_category(exc)
+ return RuntimeError(
+ f"PostgreSQL smoke setup failed [{action}:{category}]"
+ )
async def _attempt_cleanup(
@@ -78,6 +100,186 @@ async def _attempt_cleanup(
failures.append(f"{action}: {type(exc).__name__}")
+def test_postgres_startup_diagnostic_statement_contract() -> None:
+ from ares.db.postgres import (
+ _POSTGRES_FALLBACK_DDL_CODES,
+ _POSTGRES_FALLBACK_STATEMENT_SPANS,
+ _postgres_fallback_failure_invariant,
+ )
+
+ codes_are_unique = (
+ len(_POSTGRES_FALLBACK_DDL_CODES)
+ == len(set(_POSTGRES_FALLBACK_DDL_CODES))
+ == len(_POSTGRES_FALLBACK_STATEMENT_SPANS)
+ )
+ _require_fixed(
+ codes_are_unique,
+ "Fallback statements require unique diagnostic identifiers",
+ )
+ every_statement_maps = True
+ spans_are_ordered = True
+ previous_end = 0
+ for code, start, end in _POSTGRES_FALLBACK_STATEMENT_SPANS:
+ failure = PostgresError("diagnostic canary")
+ failure.position = start + ((end - start) // 2)
+ every_statement_maps = (
+ every_statement_maps
+ and _postgres_fallback_failure_invariant(failure) == code
+ )
+ spans_are_ordered = (
+ spans_are_ordered
+ and start > previous_end
+ and end >= start
+ )
+ previous_end = end
+ _require_fixed(
+ every_statement_maps,
+ "Every fallback statement must map to its fixed identifier",
+ )
+ _require_fixed(
+ spans_are_ordered,
+ "Fallback diagnostic statement spans must be ordered",
+ )
+
+
+async def test_postgres_startup_diagnostics_are_distinct_and_sanitized() -> None:
+ from ares.db.postgres import (
+ PostgresDatabase,
+ _postgres_restage_startup_error,
+ _postgres_startup_diagnostic_label,
+ _postgres_startup_error,
+ _postgres_ticket_schema_error,
+ )
+
+ canary = "diagnostic-canary-not-for-output"
+
+ class _OwnershipFailureConnection:
+ async def fetch(self, *_args: object) -> object:
+ raise PostgresError(canary)
+
+ try:
+ await PostgresDatabase._managed_schema_revision(
+ _OwnershipFailureConnection()
+ )
+ except Exception as exc:
+ ownership_failure: BaseException = exc
+ else:
+ pytest.fail("Ownership failure must be diagnosed", pytrace=False)
+
+ ddl_cause = PostgresError(canary)
+ ddl_failure = _postgres_startup_error(
+ "PostgreSQL schema validation failed",
+ stage="fallback-ddl",
+ invariant="campaigns-table",
+ cause=ddl_cause,
+ )
+ validation_failure = _postgres_restage_startup_error(
+ _postgres_ticket_schema_error("ticket-columns"),
+ "fallback-validation",
+ )
+ typed_unknown_failure = _postgres_startup_error(
+ "PostgreSQL schema validation failed",
+ stage="fallback-validation",
+ invariant="ticket-validation-unclassified",
+ cause=RuntimeError(canary),
+ )
+ unknown_failure = RuntimeError(canary)
+
+ labels = (
+ _postgres_startup_diagnostic_label(ownership_failure),
+ _postgres_startup_diagnostic_label(ddl_failure),
+ _postgres_startup_diagnostic_label(validation_failure),
+ _postgres_startup_diagnostic_label(typed_unknown_failure),
+ _postgres_startup_diagnostic_label(unknown_failure),
+ )
+ labels_are_exact = labels == (
+ "ownership:ownership-relation-query:database",
+ "fallback-ddl:campaigns-table:database",
+ "fallback-validation:ticket-columns:none",
+ "unclassified",
+ "unclassified",
+ )
+ _require_fixed(
+ labels_are_exact,
+ "Startup diagnostic stages must remain distinct and fixed",
+ )
+ public_messages_are_generic = (
+ str(ownership_failure) == "PostgreSQL schema validation failed"
+ and str(ddl_failure) == "PostgreSQL schema validation failed"
+ and str(validation_failure)
+ == "Incompatible WebSocket ticket schema"
+ )
+ _require_fixed(
+ public_messages_are_generic,
+ "Startup diagnostics must preserve generic public messages",
+ )
+ rendered = " ".join(
+ (
+ *labels,
+ str(ownership_failure),
+ repr(ownership_failure),
+ str(ddl_failure),
+ repr(ddl_failure),
+ str(validation_failure),
+ repr(validation_failure),
+ )
+ )
+ _require_fixed(
+ canary not in rendered,
+ "Startup diagnostics must not expose source content",
+ )
+
+
+async def test_postgres_startup_cleanup_preserves_primary_diagnostic() -> None:
+ from ares.db.postgres import (
+ PostgresDatabase,
+ _postgres_startup_diagnostic_label,
+ _postgres_startup_error,
+ )
+
+ canary = "cleanup-canary-not-for-output"
+
+ class _FailingPool:
+ async def close(self) -> None:
+ raise RuntimeError(canary)
+
+ database = PostgresDatabase("synthetic")
+ pool = _FailingPool()
+ database._pool = pool
+ primary = _postgres_startup_error(
+ "PostgreSQL schema validation failed",
+ stage="fallback-validation",
+ invariant="ticket-index-inventory",
+ )
+ await database._close_failed_startup_pool(pool, primary)
+
+ primary_is_preserved = (
+ _postgres_startup_diagnostic_label(primary)
+ == "fallback-validation:ticket-index-inventory:none"
+ )
+ _require_fixed(
+ primary_is_preserved,
+ "Cleanup failure must not replace the primary diagnostic",
+ )
+ notes = tuple(getattr(primary, "__notes__", ()))
+ notes_are_sanitized = (
+ len(notes) == 1
+ and notes[0]
+ == "PostgreSQL startup cleanup failure [pool-close: runtime]"
+ and canary not in " ".join(notes)
+ )
+ _require_fixed(
+ notes_are_sanitized,
+ "Cleanup diagnostics must remain fixed and sanitized",
+ )
+ _require_fixed(
+ database._pool is None,
+ "Failed startup cleanup must clear the candidate pool",
+ )
+
+
@pytest.mark.asyncio
async def test_postgres_runtime_schema_and_transaction_smoke() -> None:
config = _postgres_test_config()
@@ -126,6 +328,61 @@ async def test_postgres_runtime_schema_and_transaction_smoke() -> None:
except Exception as exc:
raise _sanitized_setup_failure("database-create", exc) from None
+ target_probe = None
+ try:
+ target_probe = await asyncpg.connect(
+ host=config["host"],
+ port=config["port"],
+ user=config["user"],
+ database=test_database,
+ )
+ initial_application_count = int(
+ await target_probe.fetchval(
+ """
+ SELECT COUNT(*)
+ FROM information_schema.tables
+ WHERE table_schema=current_schema()
+ """
+ )
+ )
+ initial_version_relation = bool(
+ await target_probe.fetchval(
+ """
+ SELECT EXISTS(
+ SELECT 1
+ FROM pg_class AS rel
+ JOIN pg_namespace AS nsp
+ ON nsp.oid=rel.relnamespace
+ WHERE nsp.nspname=current_schema()
+ AND rel.relname='alembic_version'
+ )
+ """
+ )
+ )
+ except Exception as exc:
+ raise _sanitized_setup_failure(
+ "empty-catalog-probe",
+ exc,
+ ) from None
+ finally:
+ if target_probe is not None:
+ try:
+ await target_probe.close()
+ except Exception as exc:
+ raise _sanitized_setup_failure(
+ "empty-catalog-probe-close",
+ exc,
+ ) from None
+
+ _require_fixed(
+ initial_application_count == 0,
+ "Runtime smoke database must begin without application tables",
+ )
+ _require_fixed(
+ not initial_version_relation,
+ "Runtime smoke database must begin without Alembic ownership",
+ )
+
database = PostgresDatabase(
_runtime_dsn(config, test_database),
pool_min=1,
@@ -136,8 +393,35 @@ async def test_postgres_runtime_schema_and_transaction_smoke() -> None:
except Exception as exc:
raise _sanitized_setup_failure("runtime-initialize", exc) from None
runtime_connected = True
+ initial_trace_is_exact = tuple(database._startup_trace) == (
+ "fallback-entered",
+ "fallback-ddl-complete",
+ "startup-ready",
+ )
+ _require_fixed(
+ initial_trace_is_exact,
+ "Unversioned runtime startup must complete the fallback path",
+ )
async with database._pool.acquire() as connection:
+ version_relation_exists = bool(
+ await connection.fetchval(
+ """
+ SELECT EXISTS(
+ SELECT 1
+ FROM pg_class AS rel
+ JOIN pg_namespace AS nsp
+ ON nsp.oid=rel.relnamespace
+ WHERE nsp.nspname=current_schema()
+ AND rel.relname='alembic_version'
+ )
+ """
+ )
+ )
+ _require_fixed(
+ not version_relation_exists,
+ "Runtime fallback must not stamp Alembic ownership",
+ )
table_rows = await connection.fetch(
"""
SELECT table_name
@@ -252,6 +536,39 @@ async def test_postgres_runtime_schema_and_transaction_smoke() -> None:
async with database._pool.acquire() as reusable_connection:
assert await reusable_connection.fetchval("SELECT 1") == 1
+
+ await database.close()
+ runtime_connected = False
+ database = PostgresDatabase(
+ _runtime_dsn(config, test_database),
+ pool_min=1,
+ pool_max=2,
+ )
+ try:
+ await database.connect()
+ except Exception as exc:
+ raise _sanitized_setup_failure(
+ "runtime-reconnect",
+ exc,
+ ) from None
+ runtime_connected = True
+ reconnect_trace_is_exact = tuple(database._startup_trace) == (
+ "fallback-entered",
+ "fallback-ddl-complete",
+ "startup-ready",
+ )
+ _require_fixed(
+ reconnect_trace_is_exact,
+ "Unversioned runtime reconnect must complete the fallback path",
+ )
+ async with database._pool.acquire() as reusable_connection:
+ reusable_after_reconnect = (
+ await reusable_connection.fetchval("SELECT 1") == 1
+ )
+ _require_fixed(
+ reusable_after_reconnect,
+ "Runtime pool must remain reusable after reconnect",
+ )
finally:
primary_failure = sys.exception()