ARES - authorized red-team engagement automation with dashboard, campaign scope, module orchestration, OPSEC controls, encrypted vault, and reporting.
Python active-directorycloud-securitydefensive-validationfastapigood-first-issuemitre-attackopsecreactred-teamreportingsecurity
chore(db): refine PostgreSQL relation diagnostics
Mafifrizia5b5fae3fa12
diff --git a/ares/db/postgres.py b/ares/db/postgres.pyindex 6107202..298cc37 100644--- a/ares/db/postgres.py+++ b/ares/db/postgres.py@@ -385,9 +385,198 @@ _POSTGRES_STARTUP_STAGES = frozenset(_POSTGRES_STARTUP_OPERATIONAL_CATEGORIES = frozenset({"none", "timeout", "connection", "database", "runtime", "other"})+_POSTGRES_TICKET_RELATION_PROPERTY_BITS = {+ "relkind": 0x001,+ "relpersistence": 0x002,+ "relispartition": 0x004,+ "parent_count": 0x008,+ "child_count": 0x010,+ "relrowsecurity": 0x020,+ "relforcerowsecurity": 0x040,+ "policy_count": 0x080,+ "user_trigger_count": 0x100,+ "user_rule_count": 0x200,+}+_POSTGRES_TICKET_RELATION_PROPERTY_MASK = 0x3FF+_POSTGRES_TICKET_RELATION_AGGREGATE_ONLY = (+ "ticket-relation-metadata:aggregate-comparison-only"+)+_POSTGRES_TICKET_RELATION_METADATA_PREFIX = (+ "ticket-relation-metadata:"+)+_POSTGRES_TICKET_RELATION_MISSING = object()+_POSTGRES_TICKET_RELATION_UNREADABLE = object()+++def _postgres_ticket_relation_value(+ relation: Any,+ field: str,+) -> object:+ try:+ return relation[field]+ except (KeyError, IndexError):+ return _POSTGRES_TICKET_RELATION_MISSING+ except Exception:+ return _POSTGRES_TICKET_RELATION_UNREADABLE+++def _postgres_ticket_relation_is_canonical(relation: Any) -> bool:+ try:+ relation_contract = (+ str(relation["relkind"]),+ str(relation["relpersistence"]),+ bool(relation["relispartition"]),+ bool(relation["relrowsecurity"]),+ bool(relation["relforcerowsecurity"]),+ int(relation["parent_count"]),+ int(relation["child_count"]),+ int(relation["policy_count"]),+ int(relation["user_trigger_count"]),+ int(relation["user_rule_count"]),+ )+ except (KeyError, IndexError, TypeError, ValueError):+ return False+ return relation_contract == (+ "r",+ "p",+ False,+ False,+ False,+ 0,+ 0,+ 0,+ 0,+ 0,+ )+++def _postgres_ticket_relation_diagnostic_masks(+ relation: Any,+) -> tuple[int, int, int, int]:+ mismatched = 0+ alternate = 0+ unexpected = 0+ missing = 0++ for field, expected in (+ ("relkind", "r"),+ ("relpersistence", "p"),+ ):+ bit = _POSTGRES_TICKET_RELATION_PROPERTY_BITS[field]+ value = _postgres_ticket_relation_value(relation, field)+ if value is _POSTGRES_TICKET_RELATION_MISSING:+ missing |= bit+ elif value is _POSTGRES_TICKET_RELATION_UNREADABLE:+ unexpected |= bit+ elif type(value) is str:+ if value != expected:+ mismatched |= bit+ elif type(value) is bytes and len(value) == 1:+ mismatched |= bit+ if value == expected.encode("ascii"):+ alternate |= bit+ else:+ unexpected |= bit++ for field in (+ "relispartition",+ "relrowsecurity",+ "relforcerowsecurity",+ ):+ bit = _POSTGRES_TICKET_RELATION_PROPERTY_BITS[field]+ value = _postgres_ticket_relation_value(relation, field)+ if value is _POSTGRES_TICKET_RELATION_MISSING:+ missing |= bit+ elif value is _POSTGRES_TICKET_RELATION_UNREADABLE:+ unexpected |= bit+ elif type(value) is not bool:+ unexpected |= bit+ elif value:+ mismatched |= bit++ for field in (+ "parent_count",+ "child_count",+ "policy_count",+ "user_trigger_count",+ "user_rule_count",+ ):+ bit = _POSTGRES_TICKET_RELATION_PROPERTY_BITS[field]+ value = _postgres_ticket_relation_value(relation, field)+ if value is _POSTGRES_TICKET_RELATION_MISSING:+ missing |= bit+ elif value is _POSTGRES_TICKET_RELATION_UNREADABLE:+ unexpected |= bit+ elif type(value) is not int:+ unexpected |= bit+ elif value != 0:+ mismatched |= bit++ return mismatched, alternate, unexpected, missing+++def _postgres_ticket_relation_diagnostic_invariant(+ masks: tuple[int, int, int, int],+) -> str:+ if (+ type(masks) is not tuple+ or len(masks) != 4+ or any(type(value) is not int for value in masks)+ ):+ return "ticket-validation-unclassified"+ mismatched, alternate, unexpected, missing = masks+ if any(+ value < 0+ or value & ~_POSTGRES_TICKET_RELATION_PROPERTY_MASK+ for value in masks+ ):+ return "ticket-validation-unclassified"+ if (+ alternate & ~mismatched+ or unexpected & (mismatched | alternate | missing)+ or missing & (mismatched | alternate | unexpected)+ ):+ return "ticket-validation-unclassified"+ if not (mismatched or alternate or unexpected or missing):+ return _POSTGRES_TICKET_RELATION_AGGREGATE_ONLY+ return (+ f"{_POSTGRES_TICKET_RELATION_METADATA_PREFIX}"+ f"m={mismatched:03x};a={alternate:03x};"+ f"x={unexpected:03x};n={missing:03x}"+ )+++def _is_postgres_ticket_relation_diagnostic(invariant: str) -> bool:+ if invariant == _POSTGRES_TICKET_RELATION_AGGREGATE_ONLY:+ return True+ prefix = _POSTGRES_TICKET_RELATION_METADATA_PREFIX+ if not invariant.startswith(prefix):+ return False+ parts = invariant[len(prefix):].split(";")+ if len(parts) != 4:+ return False+ expected_keys = ("m", "a", "x", "n")+ values: list[int] = []+ for part, expected_key in zip(parts, expected_keys, strict=True):+ key, separator, encoded = part.partition("=")+ if (+ separator != "="+ or key != expected_key+ or len(encoded) != 3+ or any(character not in "0123456789abcdef" for character in encoded)+ ):+ return False+ values.append(int(encoded, 16))+ return (+ _postgres_ticket_relation_diagnostic_invariant(tuple(values))+ == invariant+ )++_POSTGRES_TICKET_INVARIANTS = frozenset({"ticket-relation-present",+ "ticket-relation-query","ticket-relation-metadata","ticket-columns","ticket-constraint-inventory",@@ -478,7 +667,12 @@ class _PostgresStartupDiagnosticError(RuntimeError):)self.diagnostic_invariant = (diagnostic_invariant- if diagnostic_invariant in _POSTGRES_STARTUP_INVARIANTS+ if (+ diagnostic_invariant in _POSTGRES_STARTUP_INVARIANTS+ or _is_postgres_ticket_relation_diagnostic(+ diagnostic_invariant+ )+ )else "ownership-unclassified")self.operational_category = (@@ -501,7 +695,13 @@ def _postgres_startup_diagnostic_label(error: BaseException) -> str:return "unclassified"if (error.diagnostic_stage not in _POSTGRES_STARTUP_STAGES- or error.diagnostic_invariant not in _POSTGRES_STARTUP_INVARIANTS+ or (+ error.diagnostic_invariant+ not in _POSTGRES_STARTUP_INVARIANTS+ and not _is_postgres_ticket_relation_diagnostic(+ error.diagnostic_invariant+ )+ )or error.operational_categorynot in _POSTGRES_STARTUP_OPERATIONAL_CATEGORIES):@@ -2394,9 +2594,10 @@ class PostgresDatabase:@staticmethodasync def _validate_websocket_ticket_schema(connection: Any) -> None:"""Reject any ticket catalog that differs from the owned schema."""- relation = await connection.fetchrow(- """- SELECT rel.oid::bigint AS table_oid,+ try:+ relation = await connection.fetchrow(+ """+ SELECT rel.oid::bigint AS table_oid,nsp.oid::bigint AS schema_oid,nsp.nspname AS schema_name,rel.relkind,@@ -2432,10 +2633,19 @@ class PostgresDatabase:) AS user_rule_countFROM pg_class AS relJOIN pg_namespace AS nsp ON nsp.oid=rel.relnamespace- WHERE nsp.nspname=current_schema()- AND rel.relname='websocket_tickets'- """- )+ WHERE nsp.nspname=current_schema()+ AND rel.relname='websocket_tickets'+ """+ )+ except asyncio.CancelledError:+ raise+ except Exception as exc:+ raise _postgres_startup_error(+ "Incompatible WebSocket ticket schema",+ stage="managed-validation",+ invariant="ticket-relation-query",+ cause=exc,+ ) from Noneif relation is None:raise _postgres_ticket_schema_error("ticket-relation-present"@@ -2443,32 +2653,12 @@ class PostgresDatabase:table_oid = int(relation["table_oid"])schema_oid = int(relation["schema_oid"])schema_name = str(relation["schema_name"])- relation_contract = (- str(relation["relkind"]),- str(relation["relpersistence"]),- bool(relation["relispartition"]),- bool(relation["relrowsecurity"]),- bool(relation["relforcerowsecurity"]),- int(relation["parent_count"]),- int(relation["child_count"]),- int(relation["policy_count"]),- int(relation["user_trigger_count"]),- int(relation["user_rule_count"]),- )- if relation_contract != (- "r",- "p",- False,- False,- False,- 0,- 0,- 0,- 0,- 0,- ):+ if not _postgres_ticket_relation_is_canonical(relation):+ diagnostic = _postgres_ticket_relation_diagnostic_invariant(+ _postgres_ticket_relation_diagnostic_masks(relation)+ )raise _postgres_ticket_schema_error(- "ticket-relation-metadata"+ diagnostic)rows = await connection.fetch(diff --git a/tests/integration/test_postgres_runtime_smoke.py b/tests/integration/test_postgres_runtime_smoke.pyindex 1b1fd7c..9066e2a 100644--- a/tests/integration/test_postgres_runtime_smoke.py+++ b/tests/integration/test_postgres_runtime_smoke.py@@ -33,6 +33,21 @@ def _require_fixed(condition: bool, message: str) -> None:pytest.fail(message, pytrace=False)+def _canonical_ticket_relation_metadata() -> dict[str, object]:+ return {+ "relkind": "r",+ "relpersistence": "p",+ "relispartition": False,+ "parent_count": 0,+ "child_count": 0,+ "relrowsecurity": False,+ "relforcerowsecurity": False,+ "policy_count": 0,+ "user_trigger_count": 0,+ "user_rule_count": 0,+ }++def _postgres_test_config() -> dict[str, str | int]:values = {name: os.environ.get(name) for name in _POSTGRES_ENV}configured = [name for name, value in values.items() if value]@@ -142,6 +157,247 @@ def test_postgres_startup_diagnostic_statement_contract() -> None:)+def test_postgres_ticket_relation_diagnostic_property_contract() -> None:+ from ares.db.postgres import (+ _POSTGRES_TICKET_RELATION_PROPERTY_BITS,+ _POSTGRES_TICKET_RELATION_PROPERTY_MASK,+ _postgres_ticket_relation_diagnostic_masks,+ _postgres_ticket_relation_is_canonical,+ )++ expected_fields = frozenset(_canonical_ticket_relation_metadata())+ bits = tuple(_POSTGRES_TICKET_RELATION_PROPERTY_BITS.values())+ property_contract_is_exact = (+ frozenset(_POSTGRES_TICKET_RELATION_PROPERTY_BITS)+ == expected_fields+ and len(bits) == len(set(bits)) == 10+ and sum(bits) == _POSTGRES_TICKET_RELATION_PROPERTY_MASK == 0x3FF+ and all(bit > 0 and bit & (bit - 1) == 0 for bit in bits)+ )+ _require_fixed(+ property_contract_is_exact,+ "Relation diagnostic properties require unique bounded bits",+ )++ canonical = _canonical_ticket_relation_metadata()+ canonical_masks = _postgres_ticket_relation_diagnostic_masks(+ canonical+ )+ _require_fixed(+ _postgres_ticket_relation_is_canonical(canonical)+ and canonical_masks == (0, 0, 0, 0),+ "Canonical relation metadata must produce zero masks",+ )+++ ("field", "violating_value"),+ [+ pytest.param("relkind", "v", id="kind"),+ pytest.param("relpersistence", "u", id="persistence"),+ pytest.param("relispartition", True, id="partition"),+ pytest.param("parent_count", 1, id="parent"),+ pytest.param("child_count", 1, id="child"),+ pytest.param("relrowsecurity", True, id="rls"),+ pytest.param("relforcerowsecurity", True, id="forced-rls"),+ pytest.param("policy_count", 1, id="policy"),+ pytest.param("user_trigger_count", 1, id="user-trigger"),+ pytest.param("user_rule_count", 1, id="user-rule"),+ ],+)+def test_postgres_ticket_relation_single_property_masks(+ field: str,+ violating_value: object,+) -> None:+ from ares.db.postgres import (+ _POSTGRES_TICKET_RELATION_PROPERTY_BITS,+ _postgres_ticket_relation_diagnostic_masks,+ _postgres_ticket_relation_is_canonical,+ )++ relation = _canonical_ticket_relation_metadata()+ relation[field] = violating_value+ masks = _postgres_ticket_relation_diagnostic_masks(relation)+ expected_bit = _POSTGRES_TICKET_RELATION_PROPERTY_BITS[field]+ _require_fixed(+ masks == (expected_bit, 0, 0, 0)+ and not _postgres_ticket_relation_is_canonical(relation),+ "A relation mutation must set only its assigned mismatch bit",+ )+++def test_postgres_ticket_relation_combined_and_alternate_masks() -> None:+ from ares.db.postgres import (+ _POSTGRES_TICKET_RELATION_AGGREGATE_ONLY,+ _postgres_ticket_relation_diagnostic_invariant,+ _postgres_ticket_relation_diagnostic_masks,+ _postgres_ticket_relation_is_canonical,+ )++ combined = {+ field: (+ "v"+ if field == "relkind"+ else "u"+ if field == "relpersistence"+ else True+ if field in {+ "relispartition",+ "relrowsecurity",+ "relforcerowsecurity",+ }+ else 1+ )+ for field in _canonical_ticket_relation_metadata()+ }+ combined_masks = _postgres_ticket_relation_diagnostic_masks(+ combined+ )+ _require_fixed(+ combined_masks == (0x3FF, 0, 0, 0)+ and not _postgres_ticket_relation_is_canonical(combined),+ "Combined relation mutations must not short-circuit",+ )++ alternate = _canonical_ticket_relation_metadata()+ alternate["relkind"] = b"r"+ alternate["relpersistence"] = b"p"+ alternate_masks = _postgres_ticket_relation_diagnostic_masks(+ alternate+ )+ alternate_invariant = _postgres_ticket_relation_diagnostic_invariant(+ alternate_masks+ )+ _require_fixed(+ alternate_masks == (0x003, 0x003, 0, 0)+ and alternate_invariant+ == "ticket-relation-metadata:m=003;a=003;x=000;n=000"+ and not _postgres_ticket_relation_is_canonical(alternate),+ "Alternate internal-character representations need exact masks",+ )++ wrong_internal = _canonical_ticket_relation_metadata()+ wrong_internal["relkind"] = b"v"+ wrong_internal["relpersistence"] = b"u"+ _require_fixed(+ _postgres_ticket_relation_diagnostic_masks(wrong_internal)+ == (0x003, 0, 0, 0),+ "Wrong internal-character values must not be alternate matches",+ )+ _require_fixed(+ _postgres_ticket_relation_diagnostic_invariant((0, 0, 0, 0))+ == _POSTGRES_TICKET_RELATION_AGGREGATE_ONLY,+ "Zero masks after rejection require the aggregate-only tripwire",+ )+++def test_postgres_ticket_relation_missing_and_unexpected_masks() -> None:+ from ares.db.postgres import (+ _POSTGRES_TICKET_RELATION_PROPERTY_BITS,+ _postgres_ticket_relation_diagnostic_masks,+ _postgres_ticket_relation_is_canonical,+ )++ missing_cases_are_exact = True+ unexpected_cases_are_exact = True+ for field in _canonical_ticket_relation_metadata():+ bit = _POSTGRES_TICKET_RELATION_PROPERTY_BITS[field]++ missing = _canonical_ticket_relation_metadata()+ missing.pop(field)+ missing_cases_are_exact = (+ missing_cases_are_exact+ and _postgres_ticket_relation_diagnostic_masks(missing)+ == (0, 0, 0, bit)+ and not _postgres_ticket_relation_is_canonical(missing)+ )++ unexpected = _canonical_ticket_relation_metadata()+ unexpected[field] = object()+ unexpected_cases_are_exact = (+ unexpected_cases_are_exact+ and _postgres_ticket_relation_diagnostic_masks(unexpected)+ == (0, 0, bit, 0)+ and not _postgres_ticket_relation_is_canonical(unexpected)+ )++ _require_fixed(+ missing_cases_are_exact,+ "Missing relation properties must set only missing bits",+ )+ _require_fixed(+ unexpected_cases_are_exact,+ "Unexpected relation property types must set only type bits",+ )+++def test_postgres_ticket_relation_diagnostic_rendering_is_closed() -> None:+ from ares.db.postgres import (+ _is_postgres_ticket_relation_diagnostic,+ _postgres_startup_diagnostic_label,+ _postgres_ticket_relation_diagnostic_invariant,+ _PostgresStartupDiagnosticError,+ )++ canary = "relation-diagnostic-canary-not-for-output"+ high_bit_result = _postgres_ticket_relation_diagnostic_invariant(+ (0x400, 0, 0, 0)+ )+ overlap_result = _postgres_ticket_relation_diagnostic_invariant(+ (0x001, 0, 0x001, 0)+ )+ valid_invariant = _postgres_ticket_relation_diagnostic_invariant(+ (0x003, 0x003, 0, 0)+ )+ invalid_error = _PostgresStartupDiagnosticError(+ "Incompatible WebSocket ticket schema",+ diagnostic_stage="fallback-validation",+ diagnostic_invariant=canary,+ )+ valid_error = _PostgresStartupDiagnosticError(+ "Incompatible WebSocket ticket schema",+ diagnostic_stage="fallback-validation",+ diagnostic_invariant=valid_invariant,+ )+ valid_label = _postgres_startup_diagnostic_label(valid_error)+ invalid_label = _postgres_startup_diagnostic_label(invalid_error)+ closed_rendering = (+ high_bit_result == "ticket-validation-unclassified"+ and overlap_result == "ticket-validation-unclassified"+ and _is_postgres_ticket_relation_diagnostic(valid_invariant)+ and not _is_postgres_ticket_relation_diagnostic(+ "ticket-relation-metadata:m=400;a=000;x=000;n=000"+ )+ and valid_label+ == (+ "fallback-validation:"+ "ticket-relation-metadata:m=003;a=003;x=000;n=000:none"+ )+ and invalid_label == "unclassified"+ )+ _require_fixed(+ closed_rendering,+ "Relation diagnostic rendering must reject unknown masks",+ )+ rendered = " ".join(+ (+ high_bit_result,+ overlap_result,+ valid_invariant,+ valid_label,+ invalid_label,+ str(valid_error),+ repr(valid_error),+ str(invalid_error),+ repr(invalid_error),+ )+ )+ _require_fixed(+ canary not in rendered,+ "Relation diagnostics must not expose source values",+ )++@pytest.mark.asyncioasync def test_postgres_startup_diagnostics_are_distinct_and_sanitized() -> None:from ares.db.postgres import (@@ -158,6 +414,10 @@ async def test_postgres_startup_diagnostics_are_distinct_and_sanitized() -> Noneasync def fetch(self, *_args: object) -> object:raise PostgresError(canary)+ class _RelationQueryFailureConnection:+ async def fetchrow(self, *_args: object) -> object:+ raise PostgresError(canary)+try:await PostgresDatabase._managed_schema_revision(_OwnershipFailureConnection()@@ -167,6 +427,15 @@ async def test_postgres_startup_diagnostics_are_distinct_and_sanitized() -> Noneelse:pytest.fail("Ownership failure must be diagnosed", pytrace=False)+ try:+ await PostgresDatabase._validate_websocket_ticket_schema(+ _RelationQueryFailureConnection()+ )+ except Exception as exc:+ relation_query_failure: BaseException = exc+ else:+ pytest.fail("Relation query failure must be diagnosed", pytrace=False)+ddl_cause = PostgresError(canary)ddl_failure = _postgres_startup_error("PostgreSQL schema validation failed",@@ -192,6 +461,7 @@ async def test_postgres_startup_diagnostics_are_distinct_and_sanitized() -> None_postgres_startup_diagnostic_label(validation_failure),_postgres_startup_diagnostic_label(typed_unknown_failure),_postgres_startup_diagnostic_label(unknown_failure),+ _postgres_startup_diagnostic_label(relation_query_failure),)labels_are_exact = labels == ("ownership:ownership-relation-query:database",@@ -199,6 +469,7 @@ async def test_postgres_startup_diagnostics_are_distinct_and_sanitized() -> None"fallback-validation:ticket-columns:none","unclassified","unclassified",+ "managed-validation:ticket-relation-query:database",)_require_fixed(labels_are_exact,@@ -223,6 +494,8 @@ async def test_postgres_startup_diagnostics_are_distinct_and_sanitized() -> Nonerepr(ddl_failure),str(validation_failure),repr(validation_failure),+ str(relation_query_failure),+ repr(relation_query_failure),))_require_fixed(