Skip to content

armgit / ARES

Public Imported from GitHub
0

ARES - authorized red-team engagement automation with dashboard, campaign scope, module orchestration, OPSEC controls, encrypted vault, and reporting.

chore(db): classify PostgreSQL census query q78

Mafifrizi5b7cbeee8532
diff --git a/ares/db/postgres.py b/ares/db/postgres.py
index a731e43..11da9fe 100644
--- a/ares/db/postgres.py
+++ b/ares/db/postgres.py
@@ -677,6 +677,719 @@ def _postgres_internal_character(
return None, False, True
+_POSTGRES_Q78_OPERATION_BITS = {
+ "index-catalog": 0x08,
+ "canonical-opclasses": 0x10,
+ "current-schema": 0x20,
+ "referenced-relations": 0x40,
+}
+_POSTGRES_Q78_SUBPHASES = frozenset(
+ {
+ "construct",
+ "dispatch",
+ "server-execute",
+ "server-fetch",
+ "row-cardinality",
+ "field-presence",
+ "asyncpg-codec",
+ "normalize",
+ "aggregate",
+ "compare",
+ "cleanup",
+ "other",
+ }
+)
+_POSTGRES_Q78_SERVER_CATEGORIES = frozenset(
+ {
+ "syntax",
+ "undefined-table",
+ "undefined-column",
+ "undefined-object",
+ "undefined-function-or-operator",
+ "datatype-mismatch",
+ "cardinality",
+ "permission",
+ "transaction",
+ "connection",
+ "other-postgres",
+ }
+)
+_POSTGRES_Q78_CLIENT_CATEGORIES = frozenset(
+ {
+ "missing-field",
+ "wrong-shape",
+ "wrong-type",
+ "key",
+ "index",
+ "value",
+ "attribute",
+ "aggregate-container",
+ "other-client",
+ }
+)
+_POSTGRES_Q78_ABSENT_SQLSTATE = "-----"
+
+
+def _postgres_q78_server_category(sqlstate: str) -> str:
+ if sqlstate == "42601":
+ return "syntax"
+ if sqlstate == "42P01":
+ return "undefined-table"
+ if sqlstate == "42703":
+ return "undefined-column"
+ if sqlstate == "42704":
+ return "undefined-object"
+ if sqlstate in {"42883", "42809"}:
+ return "undefined-function-or-operator"
+ if sqlstate in {"42804", "42846"}:
+ return "datatype-mismatch"
+ if sqlstate.startswith("21"):
+ return "cardinality"
+ if sqlstate == "42501":
+ return "permission"
+ if sqlstate.startswith(("25", "40")):
+ return "transaction"
+ if sqlstate.startswith("08"):
+ return "connection"
+ return "other-postgres"
+
+
+def _postgres_q78_client_category(error: Exception) -> str:
+ if isinstance(error, KeyError):
+ return "key"
+ if isinstance(error, IndexError):
+ return "index"
+ if isinstance(error, TypeError):
+ return "wrong-type"
+ if isinstance(error, ValueError):
+ return "value"
+ if isinstance(error, AttributeError):
+ return "attribute"
+ return "other-client"
+
+
+def _postgres_q78_subphase(error: Exception) -> str:
+ raw_state = getattr(error, "sqlstate", None)
+ if type(raw_state) is str:
+ return "server-execute"
+ if isinstance(error, (TimeoutError, ConnectionError)):
+ return "dispatch"
+ if isinstance(error, (KeyError, IndexError, TypeError, ValueError, AttributeError)):
+ return "asyncpg-codec"
+ return "server-fetch"
+
+
+def _postgres_q78_failure(
+ error: Exception,
+ *,
+ subphase: str,
+) -> str:
+ safe_subphase = subphase if subphase in _POSTGRES_Q78_SUBPHASES else "other"
+ raw_state = getattr(error, "sqlstate", None)
+ if (
+ type(raw_state) is str
+ and len(raw_state) == 5
+ and all(character in "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ" for character in raw_state)
+ ):
+ state = raw_state
+ category = _postgres_q78_server_category(state)
+ else:
+ state = _POSTGRES_Q78_ABSENT_SQLSTATE
+ category = _postgres_q78_client_category(error)
+ return f"q78:{safe_subphase}:state={state}:category={category}"
+
+
+def _is_postgres_q78_failure(value: str) -> bool:
+ parts = value.split(":")
+ if len(parts) != 4 or parts[0] != "q78":
+ return False
+ subphase = parts[1]
+ state_key, separator, state = parts[2].partition("=")
+ category_key, category_separator, category = parts[3].partition("=")
+ if (
+ subphase not in _POSTGRES_Q78_SUBPHASES
+ or separator != "="
+ or state_key != "state"
+ or category_separator != "="
+ or category_key != "category"
+ ):
+ return False
+ if state == _POSTGRES_Q78_ABSENT_SQLSTATE:
+ return category in _POSTGRES_Q78_CLIENT_CATEGORIES
+ return (
+ len(state) == 5
+ and all(character in "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ" for character in state)
+ and category in _POSTGRES_Q78_SERVER_CATEGORIES
+ and _postgres_q78_server_category(state) == category
+ )
+
+
+_POSTGRES_Q78_SPLIT_INDEX_FIELDS = (
+ ("m", 0x1F, 2), ("e", 0x1, 1), ("o", 0x1F, 2),
+ ("b", 0x1F, 2), ("u", 0x1F, 2), ("p", 0x1F, 2),
+ ("v", 0x1F, 2), ("r", 0x1F, 2), ("l", 0x1F, 2),
+ ("k", 0x1F, 2), ("n", 0x1F, 2), ("c", 0x1F, 2),
+ ("d", 0x1F, 2), ("a", 0x1F, 2), ("j", 0x1F, 2),
+ ("s", 0x1F, 2), ("t", 0x1F, 2), ("y", 0x1F, 2),
+ ("h", 0x1F, 2), ("x", 0x1F, 2), ("g", 0x1, 1),
+)
+
+
+def _postgres_q78_split_invariant(
+ index: dict[str, int],
+ *,
+ primary_binding: int,
+ opclass_missing: int,
+ opclass_extra: int,
+ opclass_shape: int,
+ schema_mismatch: int,
+ schema_shape: int,
+ reference_missing: int,
+ reference_extra: int,
+ reference_order: int,
+ reference_binding: int,
+ reference_shape: int,
+ query_failure: int,
+) -> str:
+ encoded_index: list[str] = []
+ for field, maximum, width in _POSTGRES_Q78_SPLIT_INDEX_FIELDS:
+ value = index.get(field)
+ if type(value) is not int or not 0 <= value <= maximum:
+ return "q78-split-unclassified"
+ encoded_index.append(f"{field}{value:0{width}x}")
+ scalar_values = (
+ primary_binding,
+ opclass_missing,
+ opclass_extra,
+ opclass_shape,
+ schema_mismatch,
+ schema_shape,
+ reference_missing,
+ reference_extra,
+ reference_order,
+ reference_binding,
+ reference_shape,
+ query_failure,
+ )
+ if (
+ any(
+ type(value) is not int or not 0 <= value <= 0x7
+ for value in scalar_values[:-1]
+ )
+ or type(query_failure) is not int
+ or not 0 <= query_failure <= 0xF
+ ):
+ return "q78-split-unclassified"
+ return (
+ "q78-split:i=" + ",".join(encoded_index)
+ + f";p=i{primary_binding:x}"
+ + f";o=m{opclass_missing:x},e{opclass_extra:x},x{opclass_shape:x}"
+ + f";s=m{schema_mismatch:x},x{schema_shape:x}"
+ + (
+ f";f=m{reference_missing:x},e{reference_extra:x},"
+ f"o{reference_order:x},b{reference_binding:x},x{reference_shape:x}"
+ )
+ + f";z=q{query_failure:x}"
+ )
+
+
+def _is_postgres_q78_split(value: str) -> bool:
+ if not value.startswith("q78-split:i="):
+ return False
+ try:
+ index_part, primary_part, opclass_part, schema_part, reference_part, query_part = (
+ value.removeprefix("q78-split:").split(";")
+ )
+ if not index_part.startswith("i="):
+ return False
+ encoded_fields = index_part.removeprefix("i=").split(",")
+ if len(encoded_fields) != len(_POSTGRES_Q78_SPLIT_INDEX_FIELDS):
+ return False
+ index: dict[str, int] = {}
+ for encoded, (field, maximum, width) in zip(
+ encoded_fields, _POSTGRES_Q78_SPLIT_INDEX_FIELDS, strict=True
+ ):
+ raw = encoded.removeprefix(field)
+ if (
+ not encoded.startswith(field)
+ or len(raw) != width
+ or any(character not in "0123456789abcdef" for character in raw)
+ ):
+ return False
+ parsed = int(raw, 16)
+ if parsed > maximum:
+ return False
+ index[field] = parsed
+ scalar_parts = (
+ (primary_part, ("p=i",)),
+ (opclass_part, ("o=m", ",e", ",x")),
+ (schema_part, ("s=m", ",x")),
+ (reference_part, ("f=m", ",e", ",o", ",b", ",x")),
+ (query_part, ("z=q",)),
+ )
+ scalars: list[int] = []
+ for encoded, markers in scalar_parts:
+ position = 0
+ for marker in markers:
+ if not encoded.startswith(marker, position):
+ return False
+ position += len(marker)
+ allowed = "0123456789abcdef" if marker == "z=q" else "01234567"
+ if position >= len(encoded) or encoded[position] not in allowed:
+ return False
+ scalars.append(int(encoded[position], 16))
+ position += 1
+ if position != len(encoded):
+ return False
+ except (TypeError, ValueError):
+ return False
+ return (
+ _postgres_q78_split_invariant(
+ index,
+ primary_binding=scalars[0],
+ opclass_missing=scalars[1],
+ opclass_extra=scalars[2],
+ opclass_shape=scalars[3],
+ schema_mismatch=scalars[4],
+ schema_shape=scalars[5],
+ reference_missing=scalars[6],
+ reference_extra=scalars[7],
+ reference_order=scalars[8],
+ reference_binding=scalars[9],
+ reference_shape=scalars[10],
+ query_failure=scalars[11],
+ )
+ == value
+ )
+
+
+def _postgres_q78_index_split_census(
+ rows: object,
+ *,
+ canonical_opclasses: dict[str, int],
+ table_oid: int,
+ schema_oid: int,
+ schema_name: str,
+ constraint_rows: object,
+) -> tuple[dict[str, int], int]:
+ section = {field: 0 for field, _maximum, _width in _POSTGRES_Q78_SPLIT_INDEX_FIELDS}
+ expected = {
+ "idx_ws_tickets_api_key": (
+ "btree", False, False, True, True, True,
+ 1, 1, ("api_key_id",), (0,), ("text_ops",), (True,), None, None,
+ ),
+ "idx_ws_tickets_campaign": (
+ "btree", False, False, True, True, True,
+ 1, 1, ("campaign_id",), (0,), ("text_ops",), (True,), None, None,
+ ),
+ "idx_ws_tickets_expires": (
+ "btree", False, False, True, True, True,
+ 1, 1, ("expires_at",), (0,), ("timestamptz_ops",), (True,), None, None,
+ ),
+ "idx_ws_tickets_user": (
+ "btree", False, False, True, True, True,
+ 1, 1, ("user_id",), (0,), ("text_ops",), (True,), None, None,
+ ),
+ "websocket_tickets_pkey": (
+ "btree", True, True, True, True, True,
+ 1, 1, ("ticket_hash",), (0,), ("text_ops",), (True,), None, None,
+ ),
+ }
+ expected_names = tuple(expected)
+ expected_ordinals = {
+ name: ordinal for ordinal, name in enumerate(expected_names)
+ }
+ if type(rows) not in {list, tuple}:
+ section["x"] = 0x1F
+ return section, 1
+ actual_names: list[str] = []
+ actual: dict[str, tuple[object, ...]] = {}
+ index_oids: set[int] = set()
+ primary_index_oid: int | None = None
+ for raw_row in rows:
+ try:
+ name = raw_row["index_name"]
+ if type(name) is not str:
+ raise TypeError
+ ordinal = expected_ordinals.get(name)
+ bit = 0 if ordinal is None else 1 << ordinal
+ index_oid = int(raw_row["index_oid"])
+ relkind, relkind_alternate, relkind_unexpected = (
+ _postgres_internal_character(raw_row["index_relkind"])
+ )
+ persistence, persistence_alternate, persistence_unexpected = (
+ _postgres_internal_character(raw_row["index_relpersistence"])
+ )
+ operator_classes = tuple(
+ str(value) for value in raw_row["operator_classes"]
+ )
+ operator_class_oids = tuple(
+ int(value) for value in raw_row["operator_class_oids"]
+ )
+ canonical_oids = tuple(
+ canonical_opclasses.get(value, -1) for value in operator_classes
+ )
+ identity_noncodec_is_exact = (
+ int(raw_row["table_oid"]) == table_oid
+ and int(raw_row["index_schema_oid"]) == schema_oid
+ and str(raw_row["index_schema"]) == schema_name
+ and not bool(raw_row["index_is_partition"])
+ and tuple(
+ str(value)
+ for value in raw_row["operator_class_namespaces"]
+ )
+ == ("pg_catalog",)
+ and operator_class_oids == canonical_oids
+ and index_oid not in index_oids
+ )
+ identity_is_exact = (
+ identity_noncodec_is_exact
+ and type(raw_row["index_relkind"]) is str
+ and relkind == "i"
+ and type(raw_row["index_relpersistence"]) is str
+ and persistence == "p"
+ )
+ if not identity_is_exact and bit:
+ if (
+ identity_noncodec_is_exact
+ and not relkind_unexpected
+ and not persistence_unexpected
+ and relkind == "i"
+ and persistence == "p"
+ and (relkind_alternate or persistence_alternate)
+ ):
+ section["h"] |= bit
+ else:
+ section["y"] |= bit
+ index_oids.add(index_oid)
+ actual_names.append(name)
+ actual[name] = (
+ str(raw_row["access_method"]),
+ bool(raw_row["indisunique"]),
+ bool(raw_row["indisprimary"]),
+ bool(raw_row["indisvalid"]),
+ bool(raw_row["indisready"]),
+ bool(raw_row["indislive"]),
+ int(raw_row["indnkeyatts"]),
+ int(raw_row["indnatts"]),
+ tuple(
+ None if value is None else str(value)
+ for value in raw_row["columns"]
+ ),
+ tuple(int(value) for value in raw_row["column_options"]),
+ operator_classes,
+ tuple(bool(value) for value in raw_row["column_collations_match"]),
+ None if raw_row["expressions"] is None else str(raw_row["expressions"]),
+ None if raw_row["predicate"] is None else str(raw_row["predicate"]),
+ )
+ if name == "websocket_tickets_pkey":
+ primary_index_oid = index_oid
+ except (KeyError, IndexError, TypeError, ValueError):
+ section["x"] |= 0x1F
+ continue
+ actual_name_set = set(actual_names)
+ for name in set(expected_names) - actual_name_set:
+ section["m"] |= 1 << expected_ordinals[name]
+ section["e"] = int(
+ bool(actual_name_set - set(expected_names))
+ or len(actual) != len(actual_names)
+ )
+ for ordinal, name in enumerate(expected_names):
+ bit = 1 << ordinal
+ if ordinal >= len(actual_names) or actual_names[ordinal] != name:
+ section["o"] |= bit
+ actual_item = actual.get(name)
+ if actual_item is None:
+ continue
+ expected_item = expected[name]
+ for field, position in (
+ ("b", 0), ("u", 1), ("p", 2), ("v", 3),
+ ("r", 4), ("l", 5), ("k", 6), ("n", 7),
+ ("c", 8), ("d", 9), ("a", 10), ("j", 11),
+ ("s", 12), ("t", 13),
+ ):
+ if actual_item[position] != expected_item[position]:
+ section[field] |= bit
+ if actual != expected and not any(section.values()):
+ section["g"] = 1
+ primary_binding = 1
+ if type(constraint_rows) in {list, tuple} and primary_index_oid is not None:
+ for raw_row in constraint_rows:
+ try:
+ if raw_row["conname"] == "websocket_tickets_pkey":
+ primary_binding = int(
+ int(raw_row["constraint_index_oid"])
+ != primary_index_oid
+ )
+ break
+ except (KeyError, IndexError, TypeError, ValueError):
+ continue
+ return section, primary_binding
+
+
+_POSTGRES_Q78_SPLIT_INDEX_QUERY = """
+SELECT ind.indrelid::bigint AS table_oid,
+ ind.indexrelid::bigint AS index_oid,
+ idx.relnamespace::bigint AS index_schema_oid,
+ idx_nsp.nspname AS index_schema,
+ idx.relname AS index_name,
+ idx.relkind AS index_relkind,
+ idx.relpersistence AS index_relpersistence,
+ idx.relispartition AS index_is_partition,
+ am.amname AS access_method,
+ ind.indisunique,
+ ind.indisprimary,
+ ind.indisvalid,
+ ind.indisready,
+ ind.indislive,
+ ind.indnkeyatts,
+ ind.indnatts,
+ ARRAY(
+ SELECT CASE WHEN key.attnum=0 THEN NULL ELSE att.attname END
+ FROM unnest(ind.indkey::smallint[]) WITH ORDINALITY
+ AS key(attnum, ordinality)
+ LEFT JOIN pg_attribute AS att
+ ON att.attrelid=ind.indrelid AND att.attnum=key.attnum
+ ORDER BY key.ordinality
+ ) AS columns,
+ ARRAY(
+ SELECT option
+ FROM unnest(ind.indoption::smallint[]) WITH ORDINALITY
+ AS item(option, ordinality)
+ ORDER BY item.ordinality
+ ) AS column_options,
+ ARRAY(
+ SELECT opc.opcname
+ FROM unnest(ind.indclass::oid[]) WITH ORDINALITY
+ AS item(opclass, ordinality)
+ JOIN pg_opclass AS opc ON opc.oid=item.opclass
+ ORDER BY item.ordinality
+ ) AS operator_classes,
+ ARRAY(
+ SELECT opc.oid::bigint
+ FROM unnest(ind.indclass::oid[]) WITH ORDINALITY
+ AS item(opclass, ordinality)
+ JOIN pg_opclass AS opc ON opc.oid=item.opclass
+ ORDER BY item.ordinality
+ ) AS operator_class_oids,
+ ARRAY(
+ SELECT opc_nsp.nspname
+ FROM unnest(ind.indclass::oid[]) WITH ORDINALITY
+ AS item(opclass, ordinality)
+ JOIN pg_opclass AS opc ON opc.oid=item.opclass
+ JOIN pg_namespace AS opc_nsp ON opc_nsp.oid=opc.opcnamespace
+ ORDER BY item.ordinality
+ ) AS operator_class_namespaces,
+ ARRAY(
+ SELECT item.collation=att.attcollation
+ FROM unnest(ind.indcollation::oid[]) WITH ORDINALITY
+ AS item(collation, ordinality)
+ JOIN unnest(ind.indkey::smallint[]) WITH ORDINALITY
+ AS key(attnum, ordinality)
+ ON key.ordinality=item.ordinality
+ LEFT JOIN pg_attribute AS att
+ ON att.attrelid=ind.indrelid AND att.attnum=key.attnum
+ ORDER BY item.ordinality
+ ) AS column_collations_match,
+ pg_get_expr(ind.indexprs, ind.indrelid) AS expressions,
+ pg_get_expr(ind.indpred, ind.indrelid) AS predicate,
+ pg_get_indexdef(ind.indexrelid) AS definition
+FROM pg_index AS ind
+JOIN pg_class AS idx ON idx.oid=ind.indexrelid
+JOIN pg_namespace AS idx_nsp ON idx_nsp.oid=idx.relnamespace
+JOIN pg_am AS am ON am.oid=idx.relam
+WHERE ind.indrelid=$1::oid
+ORDER BY idx.relname
+"""
+
+
+async def _postgres_q78_isolated_call(
+ connection: Any,
+ method_name: str,
+ query: str,
+ *args: object,
+) -> object:
+ method = getattr(connection, method_name)
+ transaction_factory = getattr(connection, "transaction", None)
+ if callable(transaction_factory):
+ async with transaction_factory():
+ return await method(query, *args)
+ return await method(query, *args)
+
+
+async def _postgres_q78_split_observation(
+ connection: Any,
+ *,
+ table_oid: int,
+ schema_oid: int,
+ schema_name: str,
+ constraint_rows: object,
+) -> str:
+ query_failure = 0
+ split_index_rows: object = []
+ opclass_rows: object = []
+ raw_schema: object = None
+ referenced_rows: object = []
+ try:
+ split_index_rows = await _postgres_q78_isolated_call(
+ connection,
+ "fetch",
+ _POSTGRES_Q78_SPLIT_INDEX_QUERY,
+ table_oid,
+ )
+ except asyncio.CancelledError:
+ raise
+ except Exception:
+ query_failure |= 0x1
+ try:
+ opclass_rows = await _postgres_q78_isolated_call(
+ connection,
+ "fetch",
+ """
+ SELECT opc.opcname, opc.oid::bigint AS opclass_oid
+ FROM pg_opclass AS opc
+ JOIN pg_namespace AS nsp ON nsp.oid=opc.opcnamespace
+ JOIN pg_am AS am ON am.oid=opc.opcmethod
+ WHERE nsp.nspname='pg_catalog'
+ AND am.amname='btree'
+ AND opc.opcname=ANY($1::text[])
+ """,
+ ["text_ops", "timestamptz_ops"],
+ )
+ except asyncio.CancelledError:
+ raise
+ except Exception:
+ query_failure |= 0x2
+ try:
+ raw_schema = await _postgres_q78_isolated_call(
+ connection,
+ "fetchval",
+ "SELECT current_schema()",
+ )
+ except asyncio.CancelledError:
+ raise
+ except Exception:
+ query_failure |= 0x4
+ try:
+ referenced_rows = await _postgres_q78_isolated_call(
+ connection,
+ "fetch",
+ """
+ SELECT rel.relname, rel.oid::bigint AS relation_oid
+ FROM pg_class AS rel
+ WHERE rel.relnamespace=$1::oid
+ AND rel.relname=ANY($2::text[])
+ ORDER BY rel.relname
+ """,
+ schema_oid,
+ ["api_keys", "campaigns", "users"],
+ )
+ except asyncio.CancelledError:
+ raise
+ except Exception:
+ query_failure |= 0x8
+
+ opclass_missing = 0x3
+ opclass_extra = 0
+ opclass_shape = 0
+ canonical_opclasses: dict[str, int] = {}
+ if type(opclass_rows) in {list, tuple}:
+ try:
+ canonical_opclasses = {
+ str(row["opcname"]): int(row["opclass_oid"])
+ for row in opclass_rows
+ }
+ opclass_missing = (
+ int("text_ops" not in canonical_opclasses)
+ | (int("timestamptz_ops" not in canonical_opclasses) << 1)
+ )
+ opclass_extra = int(
+ bool(set(canonical_opclasses) - {"text_ops", "timestamptz_ops"})
+ )
+ except (KeyError, IndexError, TypeError, ValueError):
+ opclass_shape = 1
+ else:
+ opclass_shape = 1
+
+ index, primary_binding = _postgres_q78_index_split_census(
+ split_index_rows,
+ canonical_opclasses=canonical_opclasses,
+ table_oid=table_oid,
+ schema_oid=schema_oid,
+ schema_name=schema_name,
+ constraint_rows=constraint_rows,
+ )
+ schema_shape = int(type(raw_schema) is not str)
+ schema_mismatch = int(type(raw_schema) is str and raw_schema != schema_name)
+
+ expected_reference_names = ("api_keys", "campaigns", "users")
+ reference_missing = 0x7
+ reference_extra = 0
+ reference_order = 0
+ reference_binding = 0x7
+ reference_shape = 0
+ reference_oids: dict[str, int] = {}
+ reference_names: list[str] = []
+ if type(referenced_rows) in {list, tuple}:
+ try:
+ reference_names = [str(row["relname"]) for row in referenced_rows]
+ reference_oids = {
+ str(row["relname"]): int(row["relation_oid"])
+ for row in referenced_rows
+ }
+ reference_missing = 0
+ for ordinal, name in enumerate(expected_reference_names):
+ if name not in reference_oids:
+ reference_missing |= 1 << ordinal
+ if ordinal >= len(reference_names) or reference_names[ordinal] != name:
+ reference_order |= 1 << ordinal
+ reference_extra = int(
+ bool(set(reference_oids) - set(expected_reference_names))
+ or len(reference_oids) != len(reference_names)
+ )
+ reference_binding = 0
+ expected_constraint_names = (
+ "fk_ws_ticket_api_key",
+ "fk_ws_ticket_campaign",
+ "fk_ws_ticket_user",
+ )
+ if type(constraint_rows) not in {list, tuple}:
+ reference_binding = 0x7
+ else:
+ constraints = {
+ str(row["conname"]): row for row in constraint_rows
+ }
+ for ordinal, (constraint_name, relation_name) in enumerate(
+ zip(expected_constraint_names, expected_reference_names, strict=True)
+ ):
+ row = constraints.get(constraint_name)
+ if (
+ row is None
+ or int(row["referenced_oid"])
+ != reference_oids.get(relation_name)
+ ):
+ reference_binding |= 1 << ordinal
+ except (KeyError, IndexError, TypeError, ValueError):
+ reference_shape = 1
+ else:
+ reference_shape = 1
+ return _postgres_q78_split_invariant(
+ index,
+ primary_binding=primary_binding,
+ opclass_missing=opclass_missing,
+ opclass_extra=opclass_extra,
+ opclass_shape=opclass_shape,
+ schema_mismatch=schema_mismatch,
+ schema_shape=schema_shape,
+ reference_missing=reference_missing,
+ reference_extra=reference_extra,
+ reference_order=reference_order,
+ reference_binding=reference_binding,
+ reference_shape=reference_shape,
+ query_failure=query_failure,
+ )
+
+
_POSTGRES_TICKET_CENSUS_REJECTION_FIELDS = {
"ticket-relation-present": "r.n",
"ticket-relation-query": "z.q",
@@ -887,6 +1600,8 @@ class _PostgresStartupDiagnosticError(RuntimeError):
diagnostic_stage: str,
diagnostic_invariant: str,
operational_category: str = "none",
+ q78_failure: str = "",
+ q78_split: str = "",
) -> None:
super().__init__(public_message)
self.diagnostic_stage = (
@@ -913,6 +1628,12 @@ class _PostgresStartupDiagnosticError(RuntimeError):
in _POSTGRES_STARTUP_OPERATIONAL_CATEGORIES
else "other"
)
+ self.q78_failure = (
+ q78_failure if _is_postgres_q78_failure(q78_failure) else ""
+ )
+ self.q78_split = (
+ q78_split if _is_postgres_q78_split(q78_split) else ""
+ )
def _postgres_startup_diagnostic_label(error: BaseException) -> str:
@@ -941,11 +1662,16 @@ def _postgres_startup_diagnostic_label(error: BaseException) -> str:
not in _POSTGRES_STARTUP_OPERATIONAL_CATEGORIES
):
return "unclassified"
- return (
+ label = (
f"{error.diagnostic_stage}:"
f"{error.diagnostic_invariant}:"
f"{error.operational_category}"
)
+ if error.q78_failure or error.q78_split:
+ if not error.q78_failure or not error.q78_split:
+ return "unclassified"
+ label = f"{label}:{error.q78_failure}:{error.q78_split}"
+ return label
def _postgres_startup_error(
@@ -991,17 +1717,29 @@ def _postgres_restage_startup_error(
diagnostic_stage=stage,
diagnostic_invariant=error.diagnostic_invariant,
operational_category=error.operational_category,
+ q78_failure=error.q78_failure,
+ q78_split=error.q78_split,
)
def _postgres_ticket_schema_error(
invariant: str,
+ *,
+ cause: Exception | None = None,
+ q78_failure: str = "",
+ q78_split: str = "",
) -> _PostgresStartupDiagnosticError:
- return _postgres_startup_error(
+ error = _postgres_startup_error(
"Incompatible WebSocket ticket schema",
stage="managed-validation",
invariant=invariant,
+ cause=cause,
+ )
+ error.q78_failure = (
+ q78_failure if _is_postgres_q78_failure(q78_failure) else ""
)
+ error.q78_split = q78_split if _is_postgres_q78_split(q78_split) else ""
+ return error
_POSTGRES_MANAGED_REVISION = "0008"
@@ -2831,6 +3569,9 @@ class PostgresDatabase:
"""Reject any ticket catalog that differs from the owned schema."""
census = _new_postgres_ticket_schema_census()
rejected = False
+ q78_cause: Exception | None = None
+ q78_failure = ""
+ q78_split = ""
try:
relation = await connection.fetchrow(
"""
@@ -3075,7 +3816,37 @@ class PostgresDatabase:
rejected = True
try:
- index_rows = await connection.fetch(
+ q78_split = await _postgres_q78_split_observation(
+ connection,
+ table_oid=table_oid,
+ schema_oid=schema_oid,
+ schema_name=schema_name,
+ constraint_rows=constraint_rows,
+ )
+ except asyncio.CancelledError:
+ raise
+ except Exception:
+ q78_split = _postgres_q78_split_invariant(
+ {field: (0x1F if field == "x" else 0)
+ for field, _maximum, _width in _POSTGRES_Q78_SPLIT_INDEX_FIELDS},
+ primary_binding=1,
+ opclass_missing=3,
+ opclass_extra=0,
+ opclass_shape=1,
+ schema_mismatch=0,
+ schema_shape=1,
+ reference_missing=7,
+ reference_extra=0,
+ reference_order=7,
+ reference_binding=7,
+ reference_shape=1,
+ query_failure=7,
+ )
+
+ try:
+ index_rows = await _postgres_q78_isolated_call(
+ connection,
+ "fetch",
"""
SELECT ind.indrelid::bigint AS table_oid,
ind.indexrelid::bigint AS index_oid,
@@ -3155,12 +3926,19 @@ class PostgresDatabase:
)
except asyncio.CancelledError:
raise
- except Exception:
+ except Exception as exc:
index_rows = []
census["z"]["q"] |= 0x08
rejected = True
+ q78_cause = exc
+ q78_failure = _postgres_q78_failure(
+ exc,
+ subphase=_postgres_q78_subphase(exc),
+ )
try:
- canonical_opclass_rows = await connection.fetch(
+ canonical_opclass_rows = await _postgres_q78_isolated_call(
+ connection,
+ "fetch",
"""
SELECT opc.opcname, opc.oid::bigint AS opclass_oid
FROM pg_opclass AS opc
@@ -3174,10 +3952,16 @@ class PostgresDatabase:
)
except asyncio.CancelledError:
raise
- except Exception:
+ except Exception as exc:
canonical_opclass_rows = []
census["z"]["q"] |= 0x10
rejected = True
+ if not q78_failure:
+ q78_cause = exc
+ q78_failure = _postgres_q78_failure(
+ exc,
+ subphase=_postgres_q78_subphase(exc),
+ )
canonical_opclasses = {
str(row["opcname"]): int(row["opclass_oid"])
for row in canonical_opclass_rows
@@ -3387,8 +4171,10 @@ class PostgresDatabase:
rejected = rejected or mismatch
try:
- raw_referenced_schema = await connection.fetchval(
- "SELECT current_schema()"
+ raw_referenced_schema = await _postgres_q78_isolated_call(
+ connection,
+ "fetchval",
+ "SELECT current_schema()",
)
referenced_schema = (
raw_referenced_schema
@@ -3400,10 +4186,16 @@ class PostgresDatabase:
rejected = True
except asyncio.CancelledError:
raise
- except Exception:
+ except Exception as exc:
referenced_schema = ""
census["z"]["q"] |= 0x20
rejected = True
+ if not q78_failure:
+ q78_cause = exc
+ q78_failure = _postgres_q78_failure(
+ exc,
+ subphase=_postgres_q78_subphase(exc),
+ )
expected_foreign_keys = {
"fk_ws_ticket_campaign": (
("campaign_id",), referenced_schema, "campaigns", ("id",), "a", "c",
@@ -3416,7 +4208,9 @@ class PostgresDatabase:
),
}
try:
- referenced_rows = await connection.fetch(
+ referenced_rows = await _postgres_q78_isolated_call(
+ connection,
+ "fetch",
"""
SELECT rel.relname, rel.oid::bigint AS relation_oid
FROM pg_class AS rel
@@ -3428,10 +4222,16 @@ class PostgresDatabase:
)
except asyncio.CancelledError:
raise
- except Exception:
+ except Exception as exc:
referenced_rows = []
census["z"]["q"] |= 0x40
rejected = True
+ if not q78_failure:
+ q78_cause = exc
+ q78_failure = _postgres_q78_failure(
+ exc,
+ subphase=_postgres_q78_subphase(exc),
+ )
referenced_oids = {
str(row["relname"]): int(row["relation_oid"])
for row in referenced_rows
@@ -3615,7 +4415,12 @@ class PostgresDatabase:
if rejected:
invariant = _postgres_ticket_schema_census_invariant(census)
- raise _postgres_ticket_schema_error(invariant)
+ raise _postgres_ticket_schema_error(
+ invariant,
+ cause=q78_cause,
+ q78_failure=q78_failure,
+ q78_split=q78_split,
+ )
# ── Helpers ────────────────────────────────────────────────────────────────
diff --git a/tests/integration/test_postgres_runtime_smoke.py b/tests/integration/test_postgres_runtime_smoke.py
index 1f63a9f..6ddb2b8 100644
--- a/tests/integration/test_postgres_runtime_smoke.py
+++ b/tests/integration/test_postgres_runtime_smoke.py
@@ -1,6 +1,7 @@
"""Real PostgreSQL runtime smoke test for the optional database backend."""
from __future__ import annotations
+import asyncio
import os
import re
import sys
@@ -317,6 +318,58 @@ class _CompleteTicketCensusConnection:
raise RuntimeError("closed-test-query")
+class _NoopDiagnosticTransaction:
+ async def __aenter__(self) -> None:
+ return None
+
+ async def __aexit__(self, *_exc_info: object) -> None:
+ return None
+
+
+class _Q78ServerProbeError(PostgresError):
+ sqlstate = "42883"
+
+ def __init__(self, canary: str) -> None:
+ super().__init__(canary)
+ self.detail = canary
+ self.hint = canary
+ self.context = canary
+
+
+class _Q78FailureConnection(_CompleteTicketCensusConnection):
+ def __init__(self, canary: str) -> None:
+ self._canary = canary
+ self.queries_are_read_only = True
+
+ def transaction(self) -> _NoopDiagnosticTransaction:
+ return _NoopDiagnosticTransaction()
+
+ async def fetch(self, query: str, *args: object) -> object:
+ normalized = " ".join(query.split()).upper()
+ self.queries_are_read_only = (
+ self.queries_are_read_only
+ and normalized.startswith("SELECT ")
+ and not any(
+ token in normalized
+ for token in (
+ " INSERT ",
+ " UPDATE ",
+ " DELETE ",
+ " CREATE ",
+ " ALTER ",
+ " DROP ",
+ " ALEMBIC_VERSION ",
+ )
+ )
+ )
+ if (
+ "FROM PG_INDEX AS IND" in normalized
+ and "IND.INDKEY::SMALLINT[]" not in normalized
+ ):
+ raise _Q78ServerProbeError(self._canary)
+ return await super().fetch(query, *args)
+
+
def _postgres_test_config() -> dict[str, str | int]:
values = {name: os.environ.get(name) for name in _POSTGRES_ENV}
configured = [name for name, value in values.items() if value]
@@ -730,6 +783,164 @@ def test_postgres_ticket_census_all_zero_and_closed_rendering() -> None:
)
+def test_postgres_q78_operation_and_failure_contract_is_closed() -> None:
+ from ares.db.postgres import (
+ _POSTGRES_Q78_OPERATION_BITS,
+ _POSTGRES_Q78_SUBPHASES,
+ _is_postgres_q78_failure,
+ _postgres_q78_failure,
+ )
+
+ operation_map_is_exact = _POSTGRES_Q78_OPERATION_BITS == {
+ "index-catalog": 0x08,
+ "canonical-opclasses": 0x10,
+ "current-schema": 0x20,
+ "referenced-relations": 0x40,
+ } and sum(_POSTGRES_Q78_OPERATION_BITS.values()) == 0x78
+ _require_fixed(
+ operation_map_is_exact,
+ "q78 must map exactly to its four source operations",
+ )
+ subphases_are_distinct = True
+ for subphase in _POSTGRES_Q78_SUBPHASES:
+ rendered = _postgres_q78_failure(
+ RuntimeError("closed-client-probe"),
+ subphase=subphase,
+ )
+ subphases_are_distinct = (
+ subphases_are_distinct
+ and _is_postgres_q78_failure(rendered)
+ and rendered.startswith(f"q78:{subphase}:")
+ and "none" not in rendered
+ )
+ _require_fixed(
+ subphases_are_distinct,
+ "Every q78 subphase requires a distinct closed rendering",
+ )
+
+
+ ("sqlstate", "category"),
+ [
+ pytest.param("42601", "syntax", id="syntax"),
+ pytest.param("42P01", "undefined-table", id="undefined-table"),
+ pytest.param("42703", "undefined-column", id="undefined-column"),
+ pytest.param("42704", "undefined-object", id="undefined-object"),
+ pytest.param(
+ "42883",
+ "undefined-function-or-operator",
+ id="undefined-function",
+ ),
+ pytest.param("42804", "datatype-mismatch", id="datatype"),
+ pytest.param("21000", "cardinality", id="cardinality"),
+ pytest.param("42501", "permission", id="permission"),
+ pytest.param("25P02", "transaction", id="transaction"),
+ pytest.param("08006", "connection", id="connection"),
+ pytest.param("XX000", "other-postgres", id="other"),
+ ],
+)
+def test_postgres_q78_sqlstate_categories_are_allowlisted(
+ sqlstate: str,
+ category: str,
+) -> None:
+ from ares.db.postgres import (
+ _is_postgres_q78_failure,
+ _postgres_q78_failure,
+ )
+
+ error = PostgresError("closed-server-probe")
+ error.sqlstate = sqlstate
+ rendered = _postgres_q78_failure(error, subphase="server-execute")
+ _require_fixed(
+ _is_postgres_q78_failure(rendered)
+ and rendered
+ == f"q78:server-execute:state={sqlstate}:category={category}",
+ "PostgreSQL q78 failures require fixed SQLSTATE categories",
+ )
+
+
+async def test_postgres_q78_failure_retains_complete_split_observation() -> None:
+ from ares.db.postgres import (
+ PostgresDatabase,
+ _is_postgres_q78_failure,
+ _is_postgres_q78_split,
+ _postgres_startup_diagnostic_label,
+ _PostgresStartupDiagnosticError,
+ )
+
+ canary = "q78-diagnostic-canary-not-for-output"
+ connection = _Q78FailureConnection(canary)
+ try:
+ await PostgresDatabase._validate_websocket_ticket_schema(connection)
+ except _PostgresStartupDiagnosticError as exc:
+ failure = exc.q78_failure
+ split = exc.q78_split
+ label = _postgres_startup_diagnostic_label(exc)
+ invariant = exc.diagnostic_invariant
+ public_message = str(exc)
+ else:
+ pytest.fail("The unchanged q78 operation must remain rejecting", pytrace=False)
+
+ split_is_complete = (
+ _is_postgres_q78_split(split)
+ and "i=m00,e0,o00,b00,u00,p00,v00,r00,l00,k00,n00,c00,d00,a00,j00,s00,t00,y00,h1f,x00,g0"
+ in split
+ and ";p=i0;o=m0,e0,x0;s=m0,x0;f=m0,e0,o0,b0,x0;z=q0"
+ in split
+ )
+ q78_is_exact = (
+ failure
+ == (
+ "q78:server-execute:state=42883:"
+ "category=undefined-function-or-operator"
+ )
+ and _is_postgres_q78_failure(failure)
+ and ";z=q08,x00" in invariant
+ and ";o=m0,e0,x0,g0" in invariant
+ )
+ rendered = " ".join((failure, split, label, invariant, public_message))
+ _require_fixed(
+ q78_is_exact and split_is_complete,
+ "q78 failure must retain a complete independent semantic split",
+ )
+ _require_fixed(
+ connection.queries_are_read_only,
+ "q78 split probes must remain read-only and unstamped",
+ )
+ _require_fixed(
+ canary not in rendered and "none" not in label,
+ "q78 diagnostics must not expose server content or none",
+ )
+
+
+async def test_postgres_q78_cancellation_propagates() -> None:
+ from ares.db.postgres import PostgresDatabase
+
+ class _CancelledQ78Connection(_Q78FailureConnection):
+ async def fetch(self, query: str, *args: object) -> object:
+ normalized = " ".join(query.split()).upper()
+ if (
+ "FROM PG_INDEX AS IND" in normalized
+ and "IND.INDKEY::SMALLINT[]" not in normalized
+ ):
+ raise asyncio.CancelledError
+ return await super().fetch(query, *args)
+
+ propagated = False
+ try:
+ await PostgresDatabase._validate_websocket_ticket_schema(
+ _CancelledQ78Connection("closed-cancellation-probe")
+ )
+ except asyncio.CancelledError:
+ propagated = True
+ _require_fixed(
+ propagated,
+ "q78 cancellation must propagate without classification",
+ )
+
+
def test_postgres_ticket_column_census_distinguishes_every_dimension() -> None:
from ares.db.postgres import _postgres_ticket_column_census
@@ -919,14 +1130,20 @@ async def test_postgres_startup_diagnostics_are_distinct_and_sanitized() -> None
_postgres_startup_diagnostic_label(unknown_failure),
_postgres_startup_diagnostic_label(relation_query_failure),
)
- census_prefix = "managed-validation:"
- census_suffix = ":none"
- relation_census = labels[5]
- relation_invariant = (
- relation_census[len(census_prefix):-len(census_suffix)]
- if relation_census.startswith(census_prefix)
- and relation_census.endswith(census_suffix)
- else ""
+ relation_invariant = getattr(
+ relation_query_failure,
+ "diagnostic_invariant",
+ "",
+ )
+ relation_q78_failure = getattr(
+ relation_query_failure,
+ "q78_failure",
+ "",
+ )
+ relation_q78_split = getattr(
+ relation_query_failure,
+ "q78_split",
+ "",
)
labels_are_exact = labels[:5] == (
"ownership:ownership-relation-query:database",
@@ -938,6 +1155,10 @@ async def test_postgres_startup_diagnostics_are_distinct_and_sanitized() -> None
complete_query_failure = (
";z=q7f,x00" in relation_invariant
and "r=m000,a000,x000,n3ff,g0" in relation_invariant
+ and relation_q78_failure
+ == "q78:asyncpg-codec:state=-----:category=attribute"
+ and relation_q78_split.endswith(";z=qf")
+ and "none" not in labels[5]
)
_require_fixed(
labels_are_exact and complete_query_failure,