ARES - authorized red-team engagement automation with dashboard, campaign scope, module orchestration, OPSEC controls, encrypted vault, and reporting.
Python active-directorycloud-securitydefensive-validationfastapigood-first-issuemitre-attackopsecreactred-teamreportingsecurity
chore(db): expose complete PostgreSQL ticket schema census
Mafifrizidfd65d6ddf73
diff --git a/ares/db/postgres.py b/ares/db/postgres.pyindex 77c47c3..a731e43 100644--- a/ares/db/postgres.py+++ b/ares/db/postgres.py@@ -573,6 +573,235 @@ def _is_postgres_ticket_relation_diagnostic(invariant: str) -> bool:)+_POSTGRES_TICKET_CENSUS_FIELDS = (+ ("r", (("m", 0x3FF, 3), ("a", 0x3FF, 3), ("x", 0x3FF, 3),+ ("n", 0x3FF, 3), ("g", 0x1, 1))),+ ("c", (("m", 0xFFF, 3), ("e", 0x1, 1), ("o", 0xFFF, 3),+ ("t", 0xFFF, 3), ("u", 0xFFF, 3), ("d", 0xFFF, 3),+ ("l", 0xFFF, 3), ("i", 0xFFF, 3), ("a", 0xFFF, 3),+ ("x", 0xFFF, 3), ("g", 0x1, 1))),+ ("q", (("m", 0x7FFF, 4), ("e", 0x1, 1), ("t", 0x7FFF, 4),+ ("v", 0x7FFF, 4), ("f", 0x7FFF, 4), ("d", 0x7FFF, 4),+ ("s", 0x7FFF, 4), ("b", 0x7FFF, 4), ("a", 0x7FFF, 4),+ ("x", 0x7FFF, 4), ("g", 0x1, 1))),+ ("p", (("l", 0x1, 1), ("r", 0x1, 1), ("n", 0x1, 1),+ ("i", 0x1, 1), ("a", 0x1, 1), ("x", 0x1, 1),+ ("g", 0x1, 1))),+ ("h", (("t", 0x7FF, 3), ("v", 0x7FF, 3), ("f", 0x7FF, 3),+ ("d", 0x7FF, 3), ("s", 0x7FF, 3), ("a", 0x7FF, 3),+ ("x", 0x7FF, 3), ("g", 0x1, 1))),+ ("f", (("l", 0x7, 1), ("s", 0x7, 1), ("t", 0x7, 1),+ ("r", 0x7, 1), ("u", 0x7, 1), ("d", 0x7, 1),+ ("i", 0x7, 1), ("a", 0x7, 1), ("x", 0x7, 1),+ ("g", 0x1, 1))),+ ("i", (("m", 0x1F, 2), ("e", 0x1, 1), ("o", 0x1F, 2),+ ("b", 0x1F, 2), ("u", 0x1F, 2), ("p", 0x1F, 2),+ ("v", 0x1F, 2), ("r", 0x1F, 2), ("l", 0x1F, 2),+ ("k", 0x1F, 2), ("n", 0x1F, 2), ("c", 0x1F, 2),+ ("d", 0x1F, 2), ("a", 0x1F, 2), ("j", 0x1F, 2),+ ("s", 0x1F, 2), ("t", 0x1F, 2), ("y", 0x1F, 2),+ ("h", 0x1F, 2), ("x", 0x1F, 2), ("g", 0x1, 1))),+ ("o", (("m", 0x3, 1), ("e", 0x1, 1), ("x", 0x3, 1),+ ("g", 0x1, 1))),+ ("z", (("q", 0x7F, 2), ("x", 0x7F, 2))),+)+++def _new_postgres_ticket_schema_census() -> dict[str, dict[str, int]]:+ return {+ section: {field: 0 for field, _maximum, _width in fields}+ for section, fields in _POSTGRES_TICKET_CENSUS_FIELDS+ }+++def _postgres_ticket_schema_census_invariant(+ census: dict[str, dict[str, int]],+) -> str:+ sections: list[str] = []+ for section, fields in _POSTGRES_TICKET_CENSUS_FIELDS:+ values = census.get(section)+ if not isinstance(values, dict):+ return "ticket-validation-unclassified"+ encoded: list[str] = []+ for field, maximum, width in fields:+ value = values.get(field)+ if type(value) is not int or not 0 <= value <= maximum:+ return "ticket-validation-unclassified"+ encoded.append(f"{field}{value:0{width}x}")+ sections.append(f"{section}=" + ",".join(encoded))+ return "ticket-census:" + ";".join(sections)+++def _is_postgres_ticket_schema_census(invariant: str) -> bool:+ if not invariant.startswith("ticket-census:"):+ return False+ census = _new_postgres_ticket_schema_census()+ sections = invariant.removeprefix("ticket-census:").split(";")+ if len(sections) != len(_POSTGRES_TICKET_CENSUS_FIELDS):+ return False+ for encoded_section, (section, fields) in zip(+ sections, _POSTGRES_TICKET_CENSUS_FIELDS, strict=True+ ):+ key, separator, encoded_fields = encoded_section.partition("=")+ parts = encoded_fields.split(",")+ if separator != "=" or key != section or len(parts) != len(fields):+ return False+ for part, (field, maximum, width) in zip(parts, fields, strict=True):+ encoded = part.removeprefix(field)+ if (+ not part.startswith(field)+ or len(encoded) != width+ or any(character not in "0123456789abcdef" for character in encoded)+ ):+ return False+ value = int(encoded, 16)+ if value > maximum:+ return False+ census[section][field] = value+ return _postgres_ticket_schema_census_invariant(census) == invariant+++def _postgres_internal_character(+ value: object,+) -> tuple[str | None, bool, bool]:+ if type(value) is str:+ return value, False, False+ if type(value) is bytes and len(value) == 0:+ return "", True, False+ if type(value) is bytes and len(value) == 1:+ raw = value[0]+ if raw == 0:+ return "", True, False+ if 1 <= raw <= 0x7F:+ return chr(raw), True, False+ return None, False, True+++_POSTGRES_TICKET_CENSUS_REJECTION_FIELDS = {+ "ticket-relation-present": "r.n",+ "ticket-relation-query": "z.q",+ "ticket-relation-metadata": "r.m",+ "ticket-columns": "c",+ "ticket-constraint-inventory": "q.m",+ "ticket-canonical-opclasses": "o.m",+ "ticket-index-identity": "i.y",+ "ticket-index-inventory": "i",+ "ticket-constraint-definition-primary": "q",+ "ticket-constraint-definition-campaign": "q",+ "ticket-constraint-definition-user": "q",+ "ticket-constraint-definition-api-key": "q",+ "ticket-reference-inventory": "f.i",+ "ticket-constraint-table-binding": "q.b",+ "ticket-foreign-key-campaign": "f",+ "ticket-foreign-key-user": "f",+ "ticket-foreign-key-api-key": "f",+ "ticket-primary-key-binding": "p",+ "ticket-check-hash": "h",+ "ticket-check-kind": "h",+ "ticket-check-created-at": "h",+ "ticket-check-expires-at": "h",+ "ticket-check-consumed-at": "h",+ "ticket-check-bearer-expires-finite": "h",+ "ticket-check-created-finite": "h",+ "ticket-check-expires-finite": "h",+ "ticket-check-consumed-finite": "h",+ "ticket-check-time-order": "h",+ "ticket-check-source-shape": "h",+ "ticket-validation-unclassified": "z.x",+}+++def _postgres_ticket_column_census(+ rows: object,+ expected: list[tuple[str, str, bool, str, str, bool, str | None]],+) -> dict[str, int]:+ section = _new_postgres_ticket_schema_census()["c"]+ if not isinstance(rows, (list, tuple)):+ section["x"] = 0xFFF+ return section++ expected_names = tuple(item[0] for item in expected)+ actual_names: list[str | None] = []+ by_name: dict[str, object] = {}+ for row in rows:+ try:+ raw_name = row["column_name"] # type: ignore[index]+ except (KeyError, TypeError, IndexError):+ actual_names.append(None)+ section["x"] = 0xFFF+ continue+ if type(raw_name) is not str:+ actual_names.append(None)+ section["x"] = 0xFFF+ continue+ actual_names.append(raw_name)+ if raw_name in by_name:+ section["e"] = 1+ by_name[raw_name] = row++ actual_name_set = {name for name in actual_names if name is not None}+ section["e"] |= int(bool(actual_name_set - set(expected_names)))+ if tuple(actual_names) != expected_names:+ for ordinal, expected_name in enumerate(expected_names):+ if ordinal >= len(actual_names) or actual_names[ordinal] != expected_name:+ section["o"] |= 1 << ordinal++ for ordinal, expected_item in enumerate(expected):+ bit = 1 << ordinal+ row = by_name.get(expected_item[0])+ if row is None:+ section["m"] |= bit+ continue+ try:+ raw_type = row["data_type"] # type: ignore[index]+ raw_nullability = row["attnotnull"] # type: ignore[index]+ raw_identity = row["attidentity"] # type: ignore[index]+ raw_generated = row["attgenerated"] # type: ignore[index]+ raw_collation = row["collation_is_default"] # type: ignore[index]+ raw_default = row["column_default"] # type: ignore[index]+ except (KeyError, TypeError, IndexError):+ section["x"] |= bit+ continue++ if type(raw_type) is not str:+ section["x"] |= bit+ elif raw_type != expected_item[1]:+ section["t"] |= bit+ if type(raw_nullability) is not bool:+ section["x"] |= bit+ elif raw_nullability is not expected_item[2]:+ section["u"] |= bit++ for raw_internal, expected_internal in (+ (raw_identity, expected_item[3]),+ (raw_generated, expected_item[4]),+ ):+ semantic, alternate, unexpected = _postgres_internal_character(+ raw_internal+ )+ if unexpected:+ section["x"] |= bit+ elif semantic != expected_internal:+ section["i"] |= bit+ elif alternate:+ section["a"] |= bit++ if type(raw_collation) is not bool:+ section["x"] |= bit+ elif raw_collation is not expected_item[5]:+ section["l"] |= bit+ if raw_default is None:+ actual_default = None+ elif type(raw_default) is str:+ actual_default = raw_default+ else:+ actual_default = None+ section["x"] |= bit+ if actual_default != expected_item[6]:+ section["d"] |= bit+ return section++_POSTGRES_TICKET_INVARIANTS = frozenset({"ticket-relation-present",@@ -672,6 +901,9 @@ class _PostgresStartupDiagnosticError(RuntimeError):or _is_postgres_ticket_relation_diagnostic(diagnostic_invariant)+ or _is_postgres_ticket_schema_census(+ diagnostic_invariant+ ))else "ownership-unclassified")@@ -701,6 +933,9 @@ def _postgres_startup_diagnostic_label(error: BaseException) -> str:and not _is_postgres_ticket_relation_diagnostic(error.diagnostic_invariant)+ and not _is_postgres_ticket_schema_census(+ error.diagnostic_invariant+ ))or error.operational_categorynot in _POSTGRES_STARTUP_OPERATIONAL_CATEGORIES@@ -2594,6 +2829,8 @@ class PostgresDatabase:@staticmethodasync def _validate_websocket_ticket_schema(connection: Any) -> None:"""Reject any ticket catalog that differs from the owned schema."""+ census = _new_postgres_ticket_schema_census()+ rejected = Falsetry:relation = await connection.fetchrow("""@@ -2639,30 +2876,35 @@ class PostgresDatabase:)except asyncio.CancelledError:raise- except Exception as exc:- raise _postgres_startup_error(- "Incompatible WebSocket ticket schema",- stage="managed-validation",- invariant="ticket-relation-query",- cause=exc,- ) from None+ except Exception:+ relation = None+ census["z"]["q"] |= 0x01+ rejected = Trueif relation is None:- raise _postgres_ticket_schema_error(- "ticket-relation-present"- )- table_oid = int(relation["table_oid"])- schema_oid = int(relation["schema_oid"])- schema_name = str(relation["schema_name"])- if not _postgres_ticket_relation_is_canonical(relation):- diagnostic = _postgres_ticket_relation_diagnostic_invariant(- _postgres_ticket_relation_diagnostic_masks(relation)- )- raise _postgres_ticket_schema_error(- diagnostic- )+ census["r"]["n"] = 0x3FF+ rejected = True+ table_oid = 0+ schema_oid = 0+ schema_name = ""+ else:+ table_oid = int(relation["table_oid"])+ schema_oid = int(relation["schema_oid"])+ schema_name = str(relation["schema_name"])+ if not _postgres_ticket_relation_is_canonical(relation):+ relation_masks = _postgres_ticket_relation_diagnostic_masks(+ relation+ )+ for field, value in zip(+ ("m", "a", "x", "n"), relation_masks, strict=True+ ):+ census["r"][field] = value+ if relation_masks == (0, 0, 0, 0):+ census["r"]["g"] = 1+ rejected = True- rows = await connection.fetch(- """+ try:+ rows = await connection.fetch(+ """SELECT att.attname AS column_name,pg_catalog.format_type(att.atttypid, att.atttypmod) AS data_type,att.attnotnull,@@ -2678,9 +2920,15 @@ class PostgresDatabase:AND att.attnum > 0AND NOT att.attisdroppedORDER BY att.attnum- """,- table_oid,- )+ """,+ table_oid,+ )+ except asyncio.CancelledError:+ raise+ except Exception:+ rows = []+ census["z"]["q"] |= 0x02+ rejected = Trueactual_columns = [(str(row["column_name"]),@@ -2740,10 +2988,17 @@ class PostgresDatabase:),]if actual_columns != expected_columns:- raise _postgres_ticket_schema_error("ticket-columns")+ census["c"] = _postgres_ticket_column_census(+ rows,+ expected_columns,+ )+ if not any(census["c"].values()):+ census["c"]["g"] = 1+ rejected = True- constraint_rows = await connection.fetch(- """+ try:+ constraint_rows = await connection.fetch(+ """SELECT con.conname, con.contype,con.conrelid::bigint AS table_oid,con.confrelid::bigint AS referenced_oid,@@ -2776,9 +3031,15 @@ class PostgresDatabase:LEFT JOIN pg_namespace AS ref_nsp ON ref_nsp.oid=ref_rel.relnamespaceWHERE con.conrelid=$1::oidORDER BY con.conname- """,- table_oid,- )+ """,+ table_oid,+ )+ except asyncio.CancelledError:+ raise+ except Exception:+ constraint_rows = []+ census["z"]["q"] |= 0x04+ rejected = Trueactual_constraints = {str(row["conname"]): rowfor row in constraint_rows@@ -2800,13 +3061,22 @@ class PostgresDatabase:"fk_ws_ticket_user","websocket_tickets_pkey",}+ constraint_ordinals = {+ name: ordinal+ for ordinal, name in enumerate(sorted(expected_constraint_names))+ }if set(actual_constraints) != expected_constraint_names:- raise _postgres_ticket_schema_error(- "ticket-constraint-inventory"+ for name in expected_constraint_names - set(actual_constraints):+ census["q"]["m"] |= 1 << constraint_ordinals[name]+ census["q"]["e"] = int(+ bool(set(actual_constraints) - expected_constraint_names)+ or len(actual_constraints) != len(constraint_rows))+ rejected = True- index_rows = await connection.fetch(- """+ try:+ index_rows = await connection.fetch(+ """SELECT ind.indrelid::bigint AS table_oid,ind.indexrelid::bigint AS index_oid,idx.relnamespace::bigint AS index_schema_oid,@@ -2880,11 +3150,18 @@ class PostgresDatabase:JOIN pg_am AS am ON am.oid=idx.relamWHERE ind.indrelid=$1::oidORDER BY idx.relname- """,- table_oid,- )- canonical_opclass_rows = await connection.fetch(- """+ """,+ table_oid,+ )+ except asyncio.CancelledError:+ raise+ except Exception:+ index_rows = []+ census["z"]["q"] |= 0x08+ rejected = True+ try:+ canonical_opclass_rows = await connection.fetch(+ """SELECT opc.opcname, opc.oid::bigint AS opclass_oidFROM pg_opclass AS opcJOIN pg_namespace AS nsp ON nsp.oid=opc.opcnamespace@@ -2892,19 +3169,32 @@ class PostgresDatabase:WHERE nsp.nspname='pg_catalog'AND am.amname='btree'AND opc.opcname=ANY($1::text[])- """,- ["text_ops", "timestamptz_ops"],- )+ """,+ ["text_ops", "timestamptz_ops"],+ )+ except asyncio.CancelledError:+ raise+ except Exception:+ canonical_opclass_rows = []+ census["z"]["q"] |= 0x10+ rejected = Truecanonical_opclasses = {str(row["opcname"]): int(row["opclass_oid"])for row in canonical_opclass_rows}if set(canonical_opclasses) != {"text_ops", "timestamptz_ops"}:- raise _postgres_ticket_schema_error(- "ticket-canonical-opclasses"+ census["o"]["m"] = (+ int("text_ops" not in canonical_opclasses)+ | (int("timestamptz_ops" not in canonical_opclasses) << 1))+ census["o"]["e"] = int(+ bool(set(canonical_opclasses) - {"text_ops", "timestamptz_ops"})+ )+ rejected = Trueindex_oids: set[int] = set()+ index_identity_failures: set[str] = set()+ index_identity_alternates: set[str] = set()for row in index_rows:operator_classes = tuple(str(opclass) for opclass in row["operator_classes"]@@ -2917,12 +3207,16 @@ class PostgresDatabase:for opclass in operator_classes)index_oid = int(row["index_oid"])- identity_is_exact = (+ relkind, relkind_alternate, relkind_unexpected = (+ _postgres_internal_character(row["index_relkind"])+ )+ persistence, persistence_alternate, persistence_unexpected = (+ _postgres_internal_character(row["index_relpersistence"])+ )+ identity_noncodec_is_exact = (int(row["table_oid"]) == table_oidand int(row["index_schema_oid"]) == schema_oidand str(row["index_schema"]) == schema_name- and str(row["index_relkind"]) == "i"- and str(row["index_relpersistence"]) == "p"and not bool(row["index_is_partition"])and tuple(str(namespace)@@ -2933,10 +3227,24 @@ class PostgresDatabase:and index_oid > 0and index_oid not in index_oids)+ identity_is_exact = (+ identity_noncodec_is_exact+ and type(row["index_relkind"]) is str+ and relkind == "i"+ and type(row["index_relpersistence"]) is str+ and persistence == "p"+ )if not identity_is_exact:- raise _postgres_ticket_schema_error(- "ticket-index-identity"- )+ index_identity_failures.add(str(row["index_name"]))+ if (+ not relkind_unexpected+ and not persistence_unexpected+ and identity_noncodec_is_exact+ and relkind == "i"+ and persistence == "p"+ and (relkind_alternate or persistence_alternate)+ ):+ index_identity_alternates.add(str(row["index_name"]))index_oids.add(index_oid)actual_indexes = {@@ -2985,9 +3293,51 @@ class PostgresDatabase:),}if actual_indexes != expected_indexes:- raise _postgres_ticket_schema_error(- "ticket-index-inventory"+ expected_index_names = tuple(expected_indexes)+ index_ordinals = {+ name: ordinal for ordinal, name in enumerate(expected_index_names)+ }+ actual_index_names = tuple(str(row["index_name"]) for row in index_rows)+ actual_index_name_set = set(actual_index_names)+ for name in set(expected_index_names) - actual_index_name_set:+ census["i"]["m"] |= 1 << index_ordinals[name]+ census["i"]["e"] = int(+ bool(actual_index_name_set - set(expected_index_names))+ or len(actual_indexes) != len(index_rows))+ for ordinal, name in enumerate(expected_index_names):+ bit = 1 << ordinal+ if ordinal >= len(actual_index_names) or actual_index_names[ordinal] != name:+ census["i"]["o"] |= bit+ actual = actual_indexes.get(name)+ if actual is None:+ continue+ expected = expected_indexes[name]+ for field, position in (+ ("b", 0), ("u", 1), ("p", 2), ("v", 3),+ ("r", 4), ("l", 5), ("k", 6), ("n", 7),+ ("c", 8), ("d", 9), ("a", 10), ("j", 11),+ ("s", 12), ("t", 13),+ ):+ if actual[position] != expected[position]:+ census["i"][field] |= bit+ rejected = True++ expected_index_ordinals = {+ name: ordinal for ordinal, name in enumerate(expected_indexes)+ }+ for name in index_identity_failures:+ ordinal = expected_index_ordinals.get(name)+ if ordinal is None:+ census["i"]["e"] = 1+ continue+ bit = 1 << ordinal+ if name in index_identity_alternates:+ census["i"]["h"] |= bit+ else:+ census["i"]["y"] |= bit+ if index_identity_failures:+ rejected = Trueexpected_constraint_fragments = {"websocket_tickets_pkey": ("p", "PRIMARY KEY (ticket_hash)"),@@ -3004,34 +3354,56 @@ class PostgresDatabase:"FOREIGN KEY (api_key_id) REFERENCES api_keys(id) ON DELETE CASCADE",),}- constraint_fragment_invariants = {- "websocket_tickets_pkey": (- "ticket-constraint-definition-primary"- ),- "fk_ws_ticket_campaign": (- "ticket-constraint-definition-campaign"- ),- "fk_ws_ticket_user": (- "ticket-constraint-definition-user"- ),- "fk_ws_ticket_api_key": (- "ticket-constraint-definition-api-key"- ),- }for name, definition in expected_constraint_fragments.items():- row = actual_constraints[name]- if (- str(row["contype"]) != definition[0]- or not bool(row["convalidated"])- or bool(row["condeferrable"])- or bool(row["condeferred"])- or " ".join(str(row["definition"]).split()) != definition[1]- ):- raise _postgres_ticket_schema_error(- constraint_fragment_invariants[name]- )+ row = actual_constraints.get(name)+ if row is None:+ continue+ bit = 1 << constraint_ordinals[name]+ contype, alternate, unexpected = _postgres_internal_character(+ row["contype"]+ )+ mismatch = False+ if unexpected:+ census["q"]["x"] |= bit+ mismatch = True+ elif contype != definition[0]:+ census["q"]["t"] |= bit+ mismatch = True+ elif alternate:+ census["q"]["a"] |= bit+ mismatch = True+ if not bool(row["convalidated"]):+ census["q"]["v"] |= bit+ mismatch = True+ if bool(row["condeferrable"]):+ census["q"]["f"] |= bit+ mismatch = True+ if bool(row["condeferred"]):+ census["q"]["d"] |= bit+ mismatch = True+ if " ".join(str(row["definition"]).split()) != definition[1]:+ census["q"]["s"] |= bit+ mismatch = True+ rejected = rejected or mismatch- referenced_schema = str(await connection.fetchval("SELECT current_schema()"))+ try:+ raw_referenced_schema = await connection.fetchval(+ "SELECT current_schema()"+ )+ referenced_schema = (+ raw_referenced_schema+ if type(raw_referenced_schema) is str+ else ""+ )+ if not referenced_schema:+ census["z"]["x"] |= 0x20+ rejected = True+ except asyncio.CancelledError:+ raise+ except Exception:+ referenced_schema = ""+ census["z"]["q"] |= 0x20+ rejected = Trueexpected_foreign_keys = {"fk_ws_ticket_campaign": (("campaign_id",), referenced_schema, "campaigns", ("id",), "a", "c",@@ -3043,40 +3415,45 @@ class PostgresDatabase:("api_key_id",), referenced_schema, "api_keys", ("id",), "a", "c",),}- referenced_rows = await connection.fetch(- """+ try:+ referenced_rows = await connection.fetch(+ """SELECT rel.relname, rel.oid::bigint AS relation_oidFROM pg_class AS relWHERE rel.relnamespace=$1::oidAND rel.relname=ANY($2::text[])- """,- schema_oid,- ["api_keys", "campaigns", "users"],- )+ """,+ schema_oid,+ ["api_keys", "campaigns", "users"],+ )+ except asyncio.CancelledError:+ raise+ except Exception:+ referenced_rows = []+ census["z"]["q"] |= 0x40+ rejected = Truereferenced_oids = {str(row["relname"]): int(row["relation_oid"])for row in referenced_rows}if set(referenced_oids) != {"api_keys", "campaigns", "users"}:- raise _postgres_ticket_schema_error(- "ticket-reference-inventory"- )+ census["f"]["i"] = 0x7+ rejected = True- if any(- int(row["table_oid"]) != table_oid- for row in actual_constraints.values()- ):- raise _postgres_ticket_schema_error(- "ticket-constraint-table-binding"- )+ for name, row in actual_constraints.items():+ ordinal = constraint_ordinals.get(name)+ if ordinal is not None and int(row["table_oid"]) != table_oid:+ census["q"]["b"] |= 1 << ordinal+ rejected = True- foreign_key_invariants = {- "fk_ws_ticket_campaign": "ticket-foreign-key-campaign",- "fk_ws_ticket_user": "ticket-foreign-key-user",- "fk_ws_ticket_api_key": "ticket-foreign-key-api-key",+ foreign_key_ordinals = {+ name: ordinal for ordinal, name in enumerate(expected_foreign_keys)}for name, expected in expected_foreign_keys.items():- row = actual_constraints[name]+ row = actual_constraints.get(name)+ if row is None:+ continue+ bit = 1 << foreign_key_ordinals[name]actual = (tuple(str(column) for column in row["local_columns"]),str(row["referenced_schema"]),@@ -3089,30 +3466,64 @@ class PostgresDatabase:row["referenced_schema_oid"] is not Noneand int(row["referenced_schema_oid"]) == schema_oidand int(row["referenced_oid"])- == referenced_oids[expected[2]]+ == referenced_oids.get(expected[2]))- if actual != expected or not reference_identity_is_exact:- raise _postgres_ticket_schema_error(- foreign_key_invariants[name]+ for field, actual_value, expected_value in zip(+ ("l", "s", "t", "r"), actual[:4], expected[:4], strict=True+ ):+ if actual_value != expected_value:+ census["f"][field] |= bit+ rejected = True+ for field, raw_value, expected_value in (+ ("u", row["confupdtype"], expected[4]),+ ("d", row["confdeltype"], expected[5]),+ ):+ semantic, alternate, unexpected = _postgres_internal_character(+ raw_value)-- primary = actual_constraints["websocket_tickets_pkey"]- if (- tuple(str(column) for column in primary["local_columns"])- != ("ticket_hash",)- or tuple(primary["remote_columns"])- or primary["referenced_schema"] is not None- or primary["referenced_table"] is not None- or int(primary["constraint_index_oid"])- != next(- int(row["index_oid"])- for row in index_rows+ if unexpected:+ census["f"]["x"] |= bit+ rejected = True+ elif semantic != expected_value:+ census["f"][field] |= bit+ rejected = True+ elif alternate:+ census["f"]["a"] |= bit+ rejected = True+ if not reference_identity_is_exact:+ census["f"]["i"] |= bit+ rejected = True++ primary = actual_constraints.get("websocket_tickets_pkey")+ primary_index = next(+ (+ row for row in index_rowsif str(row["index_name"]) == "websocket_tickets_pkey"- )- ):- raise _postgres_ticket_schema_error(- "ticket-primary-key-binding"- )+ ),+ None,+ )+ if primary is not None:+ if tuple(str(column) for column in primary["local_columns"]) != (+ "ticket_hash",+ ):+ census["p"]["l"] = 1+ rejected = True+ if tuple(primary["remote_columns"]):+ census["p"]["r"] = 1+ rejected = True+ if (+ primary["referenced_schema"] is not None+ or primary["referenced_table"] is not None+ ):+ census["p"]["n"] = 1+ rejected = True+ if (+ primary_index is None+ or int(primary["constraint_index_oid"])+ != int(primary_index["index_oid"])+ ):+ census["p"]["i"] = 1+ rejected = Trueexpected_check_definitions = {"ck_ws_ticket_hash": (@@ -3161,40 +3572,50 @@ class PostgresDatabase:"AND required_scope = 'read'::text)"),}- check_invariants = {- "ck_ws_ticket_hash": "ticket-check-hash",- "ck_ws_ticket_kind": "ticket-check-kind",- "ck_ws_ticket_created_at": "ticket-check-created-at",- "ck_ws_ticket_expires_at": "ticket-check-expires-at",- "ck_ws_ticket_consumed_at": "ticket-check-consumed-at",- "ck_ws_ticket_bearer_expires_finite": (- "ticket-check-bearer-expires-finite"- ),- "ck_ws_ticket_created_finite": (- "ticket-check-created-finite"- ),- "ck_ws_ticket_expires_finite": (- "ticket-check-expires-finite"- ),- "ck_ws_ticket_consumed_finite": (- "ticket-check-consumed-finite"- ),- "ck_ws_ticket_time_order": "ticket-check-time-order",- "ck_ws_ticket_source_shape": "ticket-check-source-shape",+ check_ordinals = {+ name: ordinal for ordinal, name in enumerate(expected_check_definitions)}for name, expected_definition in expected_check_definitions.items():- row = actual_constraints[name]- if (- str(row["contype"]) != "c"- or not bool(row["convalidated"])- or bool(row["condeferrable"])- or bool(row["condeferred"])- or " ".join(str(row["definition"]).split())- != expected_definition- ):- raise _postgres_ticket_schema_error(- check_invariants[name]- )+ row = actual_constraints.get(name)+ if row is None:+ continue+ bit = 1 << check_ordinals[name]+ constraint_bit = 1 << constraint_ordinals[name]+ contype, alternate, unexpected = _postgres_internal_character(+ row["contype"]+ )+ if unexpected:+ census["h"]["x"] |= bit+ census["q"]["x"] |= constraint_bit+ rejected = True+ elif contype != "c":+ census["h"]["t"] |= bit+ census["q"]["t"] |= constraint_bit+ rejected = True+ elif alternate:+ census["h"]["a"] |= bit+ census["q"]["a"] |= constraint_bit+ rejected = True+ if not bool(row["convalidated"]):+ census["h"]["v"] |= bit+ census["q"]["v"] |= constraint_bit+ rejected = True+ if bool(row["condeferrable"]):+ census["h"]["f"] |= bit+ census["q"]["f"] |= constraint_bit+ rejected = True+ if bool(row["condeferred"]):+ census["h"]["d"] |= bit+ census["q"]["d"] |= constraint_bit+ rejected = True+ if " ".join(str(row["definition"]).split()) != expected_definition:+ census["h"]["s"] |= bit+ census["q"]["s"] |= constraint_bit+ rejected = True++ if rejected:+ invariant = _postgres_ticket_schema_census_invariant(census)+ raise _postgres_ticket_schema_error(invariant)# ── Helpers ────────────────────────────────────────────────────────────────diff --git a/tests/integration/test_postgres_runtime_smoke.py b/tests/integration/test_postgres_runtime_smoke.pyindex bb3fa66..1f63a9f 100644--- a/tests/integration/test_postgres_runtime_smoke.py+++ b/tests/integration/test_postgres_runtime_smoke.py@@ -48,6 +48,275 @@ def _canonical_ticket_relation_metadata() -> dict[str, object]:}+def _canonical_ticket_columns() -> list[dict[str, object]]:+ definitions = (+ ("ticket_hash", "text", True),+ ("campaign_id", "text", True),+ ("user_id", "text", True),+ ("credential_kind", "text", True),+ ("bearer_subject", "text", False),+ ("bearer_jti", "text", False),+ ("bearer_expires_at", "timestamp with time zone", False),+ ("api_key_id", "text", False),+ ("required_scope", "text", False),+ ("created_at", "timestamp with time zone", True),+ ("expires_at", "timestamp with time zone", True),+ ("consumed_at", "timestamp with time zone", False),+ )+ return [+ {+ "column_name": name,+ "data_type": data_type,+ "attnotnull": not_null,+ "attidentity": "",+ "attgenerated": "",+ "collation_is_default": True,+ "column_default": None,+ }+ for name, data_type, not_null in definitions+ ]+++def _expected_ticket_column_contract() -> list[+ tuple[str, str, bool, str, str, bool, str | None]+]:+ return [+ (+ str(row["column_name"]),+ str(row["data_type"]),+ bool(row["attnotnull"]),+ "",+ "",+ True,+ None,+ )+ for row in _canonical_ticket_columns()+ ]+++def _canonical_ticket_constraints() -> list[dict[str, object]]:+ checks = {+ "ck_ws_ticket_hash": "CHECK (ticket_hash ~ '^[0-9a-f]{64}$'::text)",+ "ck_ws_ticket_kind": (+ "CHECK (credential_kind = ANY "+ "(ARRAY['bearer'::text, 'api_key'::text]))"+ ),+ "ck_ws_ticket_created_at": "CHECK (created_at < expires_at)",+ "ck_ws_ticket_expires_at": "CHECK (expires_at > created_at)",+ "ck_ws_ticket_consumed_at": (+ "CHECK (consumed_at IS NULL OR consumed_at < expires_at)"+ ),+ "ck_ws_ticket_bearer_expires_finite": (+ "CHECK (bearer_expires_at IS NULL OR isfinite(bearer_expires_at))"+ ),+ "ck_ws_ticket_created_finite": "CHECK (isfinite(created_at))",+ "ck_ws_ticket_expires_finite": "CHECK (isfinite(expires_at))",+ "ck_ws_ticket_consumed_finite": (+ "CHECK (consumed_at IS NULL OR isfinite(consumed_at))"+ ),+ "ck_ws_ticket_time_order": (+ "CHECK (expires_at > created_at AND "+ "(consumed_at IS NULL OR consumed_at < expires_at))"+ ),+ "ck_ws_ticket_source_shape": (+ "CHECK (credential_kind = 'bearer'::text "+ "AND bearer_subject IS NOT NULL "+ "AND length(btrim(bearer_subject)) > 0 "+ "AND bearer_subject = btrim(bearer_subject) "+ "AND bearer_jti IS NOT NULL "+ "AND length(btrim(bearer_jti)) > 0 "+ "AND bearer_jti = btrim(bearer_jti) "+ "AND bearer_expires_at IS NOT NULL "+ "AND api_key_id IS NULL AND required_scope IS NULL "+ "OR credential_kind = 'api_key'::text "+ "AND bearer_subject IS NULL AND bearer_jti IS NULL "+ "AND bearer_expires_at IS NULL AND api_key_id IS NOT NULL "+ "AND length(btrim(api_key_id)) > 0 "+ "AND api_key_id = btrim(api_key_id) "+ "AND required_scope = 'read'::text)"+ ),+ }+ rows: list[dict[str, object]] = []+ for name, definition in checks.items():+ rows.append(+ {+ "conname": name,+ "contype": b"c",+ "table_oid": 1,+ "referenced_oid": 0,+ "constraint_index_oid": 0,+ "convalidated": True,+ "condeferrable": False,+ "condeferred": False,+ "definition": definition,+ "referenced_schema_oid": None,+ "referenced_schema": None,+ "referenced_table": None,+ "local_columns": [],+ "remote_columns": [],+ "confupdtype": b"a",+ "confdeltype": b"a",+ }+ )+ references = (+ (+ "fk_ws_ticket_api_key",+ "api_key_id",+ "api_keys",+ 201,+ "FOREIGN KEY (api_key_id) REFERENCES api_keys(id) ON DELETE CASCADE",+ ),+ (+ "fk_ws_ticket_campaign",+ "campaign_id",+ "campaigns",+ 202,+ "FOREIGN KEY (campaign_id) REFERENCES campaigns(id) ON DELETE CASCADE",+ ),+ (+ "fk_ws_ticket_user",+ "user_id",+ "users",+ 203,+ "FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE",+ ),+ )+ for name, column, table, referenced_oid, definition in references:+ rows.append(+ {+ "conname": name,+ "contype": b"f",+ "table_oid": 1,+ "referenced_oid": referenced_oid,+ "constraint_index_oid": 0,+ "convalidated": True,+ "condeferrable": False,+ "condeferred": False,+ "definition": definition,+ "referenced_schema_oid": 2,+ "referenced_schema": "public",+ "referenced_table": table,+ "local_columns": [column],+ "remote_columns": ["id"],+ "confupdtype": b"a",+ "confdeltype": b"c",+ }+ )+ rows.append(+ {+ "conname": "websocket_tickets_pkey",+ "contype": b"p",+ "table_oid": 1,+ "referenced_oid": 0,+ "constraint_index_oid": 105,+ "convalidated": True,+ "condeferrable": False,+ "condeferred": False,+ "definition": "PRIMARY KEY (ticket_hash)",+ "referenced_schema_oid": None,+ "referenced_schema": None,+ "referenced_table": None,+ "local_columns": ["ticket_hash"],+ "remote_columns": [],+ "confupdtype": b"a",+ "confdeltype": b"a",+ }+ )+ return sorted(rows, key=lambda row: str(row["conname"]))+++def _canonical_ticket_indexes() -> list[dict[str, object]]:+ definitions = (+ ("idx_ws_tickets_api_key", "api_key_id", "text_ops", 11, False),+ ("idx_ws_tickets_campaign", "campaign_id", "text_ops", 11, False),+ (+ "idx_ws_tickets_expires",+ "expires_at",+ "timestamptz_ops",+ 12,+ False,+ ),+ ("idx_ws_tickets_user", "user_id", "text_ops", 11, False),+ ("websocket_tickets_pkey", "ticket_hash", "text_ops", 11, True),+ )+ rows: list[dict[str, object]] = []+ for ordinal, (name, column, opclass, opclass_oid, primary) in enumerate(+ definitions,+ start=101,+ ):+ rows.append(+ {+ "table_oid": 1,+ "index_oid": 105 if primary else ordinal,+ "index_schema_oid": 2,+ "index_schema": "public",+ "index_name": name,+ "index_relkind": b"i",+ "index_relpersistence": b"p",+ "index_is_partition": False,+ "access_method": "btree",+ "indisunique": primary,+ "indisprimary": primary,+ "indisvalid": True,+ "indisready": True,+ "indislive": True,+ "indnkeyatts": 1,+ "indnatts": 1,+ "columns": [column],+ "column_options": [0],+ "operator_classes": [opclass],+ "operator_class_oids": [opclass_oid],+ "operator_class_namespaces": ["pg_catalog"],+ "column_collations_match": [True],+ "expressions": None,+ "predicate": None,+ "definition": "closed-source-defined-index",+ }+ )+ return rows+++class _CompleteTicketCensusConnection:+ async def fetchrow(self, query: str, *_args: object) -> object:+ if "FROM pg_class AS rel" not in query:+ raise RuntimeError("closed-test-query")+ return {+ "table_oid": 1,+ "schema_oid": 2,+ "schema_name": "public",+ **_canonical_ticket_relation_metadata(),+ }++ async def fetch(self, query: str, *_args: object) -> object:+ if "FROM pg_attribute AS att" in query:+ rows = _canonical_ticket_columns()+ for row in rows:+ row["attidentity"] = b"\x00"+ row["attgenerated"] = b"\x00"+ return rows+ if "FROM pg_constraint AS con" in query:+ return _canonical_ticket_constraints()+ if "FROM pg_index AS ind" in query:+ return _canonical_ticket_indexes()+ if "FROM pg_opclass AS opc" in query:+ return [+ {"opcname": "text_ops", "opclass_oid": 11},+ {"opcname": "timestamptz_ops", "opclass_oid": 12},+ ]+ if "SELECT rel.relname" in query:+ return [+ {"relname": "api_keys", "relation_oid": 201},+ {"relname": "campaigns", "relation_oid": 202},+ {"relname": "users", "relation_oid": 203},+ ]+ raise RuntimeError("closed-test-query")++ async def fetchval(self, query: str, *_args: object) -> object:+ if "current_schema" in query:+ return "public"+ raise RuntimeError("closed-test-query")++def _postgres_test_config() -> dict[str, str | int]:values = {name: os.environ.get(name) for name in _POSTGRES_ENV}configured = [name for name, value in values.items() if value]@@ -398,10 +667,197 @@ def test_postgres_ticket_relation_diagnostic_rendering_is_closed() -> None:)+def test_postgres_ticket_census_covers_every_rejection_predicate() -> None:+ from ares.db.postgres import (+ _POSTGRES_TICKET_CENSUS_REJECTION_FIELDS,+ _POSTGRES_TICKET_INVARIANTS,+ )++ covered_once = (+ set(_POSTGRES_TICKET_CENSUS_REJECTION_FIELDS)+ == set(_POSTGRES_TICKET_INVARIANTS)+ and len(_POSTGRES_TICKET_CENSUS_REJECTION_FIELDS)+ == len(_POSTGRES_TICKET_INVARIANTS)+ and all(+ isinstance(field, str) and field+ for field in _POSTGRES_TICKET_CENSUS_REJECTION_FIELDS.values()+ )+ )+ _require_fixed(+ covered_once,+ "Every ticket-schema rejection requires one closed census field",+ )+++def test_postgres_ticket_census_all_zero_and_closed_rendering() -> None:+ from ares.db.postgres import (+ _POSTGRES_TICKET_CENSUS_FIELDS,+ _is_postgres_ticket_schema_census,+ _new_postgres_ticket_schema_census,+ _postgres_startup_diagnostic_label,+ _postgres_ticket_schema_census_invariant,+ _PostgresStartupDiagnosticError,+ )++ census = _new_postgres_ticket_schema_census()+ invariant = _postgres_ticket_schema_census_invariant(census)+ all_sections_present = all(+ f"{section}=" in invariant+ for section, _fields in _POSTGRES_TICKET_CENSUS_FIELDS+ )+ error = _PostgresStartupDiagnosticError(+ "Incompatible WebSocket ticket schema",+ diagnostic_stage="fallback-validation",+ diagnostic_invariant=invariant,+ )+ label = _postgres_startup_diagnostic_label(error)+ _require_fixed(+ all_sections_present+ and _is_postgres_ticket_schema_census(invariant)+ and label == f"fallback-validation:{invariant}:none",+ "The canonical census requires every deterministic zero section",+ )++ malformed = (+ invariant.replace("r=m000", "r=m400", 1),+ invariant + ";extra=q00",+ invariant.replace(";c=", ";canary=", 1),+ invariant.replace("g0", "g2", 1),+ )+ _require_fixed(+ not any(_is_postgres_ticket_schema_census(item) for item in malformed),+ "Unknown census values must not enter startup diagnostics",+ )+++def test_postgres_ticket_column_census_distinguishes_every_dimension() -> None:+ from ares.db.postgres import _postgres_ticket_column_census++ expected = _expected_ticket_column_contract()+ canonical = _canonical_ticket_columns()+ zero = _postgres_ticket_column_census(canonical, expected)+ _require_fixed(+ not any(zero.values()),+ "Canonical ticket columns must produce a zero census",+ )++ codec = [dict(row) for row in canonical]+ for row in codec:+ row["attidentity"] = b"\x00"+ row["attgenerated"] = b"\x00"+ codec_result = _postgres_ticket_column_census(codec, expected)+ _require_fixed(+ codec_result["a"] == 0xFFF+ and sum(codec_result.values()) == 0xFFF,+ "Known internal-character codecs require only alternate bits",+ )++ mutations: tuple[tuple[str, object, str], ...] = (+ ("data_type", "integer", "t"),+ ("attnotnull", False, "u"),+ ("column_default", "now()", "d"),+ ("collation_is_default", False, "l"),+ ("attidentity", "d", "i"),+ )+ dimensions_are_distinct = True+ for source_field, value, census_field in mutations:+ rows = [dict(row) for row in canonical]+ rows[0][source_field] = value+ result = _postgres_ticket_column_census(rows, expected)+ dimensions_are_distinct = (+ dimensions_are_distinct+ and result[census_field] == 0x001+ and sum(result.values()) == 0x001+ )+ _require_fixed(+ dimensions_are_distinct,+ "Column mutations require distinct bounded census dimensions",+ )++ missing = _postgres_ticket_column_census(canonical[1:], expected)+ extra_rows = [dict(row) for row in canonical]+ extra = dict(canonical[-1])+ extra["column_name"] = "closed-test-extra"+ extra_rows.append(extra)+ extra_result = _postgres_ticket_column_census(extra_rows, expected)+ reordered_rows = [dict(row) for row in canonical]+ reordered_rows[0], reordered_rows[1] = reordered_rows[1], reordered_rows[0]+ reordered = _postgres_ticket_column_census(reordered_rows, expected)+ unexpected_rows = [dict(row) for row in canonical]+ unexpected_rows[0]["attidentity"] = object()+ unexpected = _postgres_ticket_column_census(unexpected_rows, expected)+ _require_fixed(+ missing["m"] == 0x001+ and missing["o"] != 0+ and extra_result["e"] == 1+ and reordered["o"] == 0x003+ and unexpected["x"] == 0x001,+ "Missing, extra, order, and shape failures require distinct fields",+ )+++def test_postgres_ticket_census_reports_cross_group_mutations_together() -> None:+ from ares.db.postgres import (+ _new_postgres_ticket_schema_census,+ _postgres_ticket_schema_census_invariant,+ )++ census = _new_postgres_ticket_schema_census()+ census["c"]["t"] = 0x001+ census["q"]["s"] = 0x0002+ census["f"]["d"] = 0x4+ census["i"]["a"] = 0x08+ invariant = _postgres_ticket_schema_census_invariant(census)+ all_mutations_survive = all(+ fragment in invariant+ for fragment in ("t001", "s0002", "d4", "a08")+ )+ _require_fixed(+ all_mutations_survive,+ "Independent later-group mutations must survive one census payload",+ )+++async def test_postgres_ticket_validator_censuses_all_later_groups() -> None:+ from ares.db.postgres import (+ PostgresDatabase,+ _is_postgres_ticket_schema_census,+ _PostgresStartupDiagnosticError,+ )++ try:+ await PostgresDatabase._validate_websocket_ticket_schema(+ _CompleteTicketCensusConnection()+ )+ except _PostgresStartupDiagnosticError as exc:+ invariant = exc.diagnostic_invariant+ else:+ pytest.fail("Alternate catalog codecs must remain rejected", pytrace=False)++ complete = (+ _is_postgres_ticket_schema_census(invariant)+ and "c=m000,e0,o000,t000,u000,d000,l000,i000,afff,x000,g0"+ in invariant+ and "q=m0000,e0,t0000,v0000,f0000,d0000,s0000,b0000,a7fff,x0000,g0"+ in invariant+ and "h=t000,v000,f000,d000,s000,a7ff,x000,g0" in invariant+ and "f=l0,s0,t0,r0,u0,d0,i0,a7,x0,g0" in invariant+ and "i=m00,e0,o00,b00,u00,p00,v00,r00,l00,k00,n00,c00,d00,a00,j00,s00,t00,y00,h1f,x00,g0"+ in invariant+ and ";o=m0,e0,x0,g0;z=q00,x00" in invariant+ )+ _require_fixed(+ complete,+ "A column rejection must retain canonical and alternate later groups",+ )++@pytest.mark.asyncioasync def test_postgres_startup_diagnostics_are_distinct_and_sanitized() -> None:from ares.db.postgres import (PostgresDatabase,+ _is_postgres_ticket_schema_census,_postgres_restage_startup_error,_postgres_startup_diagnostic_label,_postgres_startup_error,@@ -463,16 +919,28 @@ async def test_postgres_startup_diagnostics_are_distinct_and_sanitized() -> None_postgres_startup_diagnostic_label(unknown_failure),_postgres_startup_diagnostic_label(relation_query_failure),)- labels_are_exact = labels == (+ census_prefix = "managed-validation:"+ census_suffix = ":none"+ relation_census = labels[5]+ relation_invariant = (+ relation_census[len(census_prefix):-len(census_suffix)]+ if relation_census.startswith(census_prefix)+ and relation_census.endswith(census_suffix)+ else ""+ )+ labels_are_exact = labels[:5] == ("ownership:ownership-relation-query:database","fallback-ddl:campaigns-table:database","fallback-validation:ticket-columns:none","unclassified","unclassified",- "managed-validation:ticket-relation-query:database",+ ) and _is_postgres_ticket_schema_census(relation_invariant)+ complete_query_failure = (+ ";z=q7f,x00" in relation_invariant+ and "r=m000,a000,x000,n3ff,g0" in relation_invariant)_require_fixed(- labels_are_exact,+ labels_are_exact and complete_query_failure,"Startup diagnostic stages must remain distinct and fixed",)public_messages_are_generic = (