:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-modeling): add a command to clear chosen node suspensions (#121112)
Jovan Sakovicb3a98dd6ff70
diff --git a/products/data_modeling/backend/management/commands/clear_node_suspensions.py b/products/data_modeling/backend/management/commands/clear_node_suspensions.pynew file mode 100644index 000000000000..965679db5a55--- /dev/null+++ b/products/data_modeling/backend/management/commands/clear_node_suspensions.py@@ -0,0 +1,160 @@+"""Clear suspension markers on a chosen set of nodes.++`clear_unearned_node_suspensions` frees markers the failure counter would no longer produce. This+command is for markers the counter did earn, but for a cause on our side: a ClickHouse incident that+failed every run with the same error, or a Trino shadow compiler gap that has since been fixed.+The operator names the set; the command never decides what deserves to be cleared.++At least one of --node-ids, --saved-query-ids or --reason-contains is required, so a run cannot+clear the whole fleet by omission. Filters combine with AND. Region-agnostic - run it once per region.++ python manage.py clear_node_suspensions --engine clickhouse --reason-contains TOO_MANY_SIMULTANEOUS_QUERIES \\+ --suspended-from <iso-time> --suspended-to <iso-time>+ python manage.py clear_node_suspensions --engine managed_warehouse --team-id 2 \\+ --reason-contains TRINO_AST_EXPANSION_LIMIT --apply+ python manage.py clear_node_suspensions --saved-query-ids <uuid>,<uuid> --apply+"""++import datetime as dt+from collections.abc import Iterator+from dataclasses import dataclass+from typing import Any+from uuid import UUID++from django.core.management.base import BaseCommand, CommandError, CommandParser++from products.data_modeling.backend.logic.node_suspension import suspension_state, unsuspend_nodes+from products.data_modeling.backend.models.node import Node++RESUMED_BY = "operator"+++def _uuid_list(value: str) -> list[UUID]:+ return [UUID(item.strip()) for item in value.split(",") if item.strip()]+++def _aware_datetime(value: str) -> dt.datetime:+ parsed = dt.datetime.fromisoformat(value)+ if parsed.tzinfo is None:+ raise ValueError("include a UTC offset, for example 2026-09-29T00:00:00+00:00")+ return parsed+++@dataclass(frozen=True, kw_only=True)+class Selection:+ engine: str | None+ reason_contains: tuple[str, ...]+ suspended_from: dt.datetime | None+ suspended_to: dt.datetime | None++ def matches(self, engine: str, entry: dict) -> bool:+ if self.engine is not None and engine != self.engine:+ return False+ reason = entry.get("reason") or ""+ if self.reason_contains and not any(fragment in reason for fragment in self.reason_contains):+ return False+ if self.suspended_from is None and self.suspended_to is None:+ return True+ try:+ at = dt.datetime.fromisoformat(entry.get("at") or "")+ except ValueError:+ return False # a window was asked for, and a marker with no readable time is outside it+ if self.suspended_from is not None and at < self.suspended_from:+ return False+ return self.suspended_to is None or at < self.suspended_to+++@dataclass(frozen=True, kw_only=True)+class Marker:+ node: Node+ engine: str+ at: str+ reason: str++ def unchanged(self, node: Node) -> bool:+ """A model can re-suspend between the report and the write. A newer marker records a newer+ failure the operator never saw, so it stays."""+ return (suspension_state(node).get(self.engine) or {}).get("at") == self.at+++class Command(BaseCommand):+ help = "Clear data modeling suspension markers on an operator-chosen set of nodes"++ def add_arguments(self, parser: CommandParser) -> None:+ parser.add_argument("--team-id", type=int, default=None, help="Limit to one team")+ parser.add_argument("--node-ids", type=_uuid_list, default=None, help="Comma-separated node ids")+ parser.add_argument(+ "--saved-query-ids",+ type=_uuid_list,+ default=None,+ help="Comma-separated saved query ids; selects every node backing each query",+ )+ parser.add_argument("--engine", default=None, help="Only markers for this engine, e.g. clickhouse")+ parser.add_argument(+ "--reason-contains",+ action="append",+ default=[],+ help="Only markers whose recorded error contains this text. Repeat to match any of several.",+ )+ parser.add_argument("--suspended-from", type=_aware_datetime, default=None, help="ISO time, inclusive")+ parser.add_argument("--suspended-to", type=_aware_datetime, default=None, help="ISO time, exclusive")+ parser.add_argument("--by", default=RESUMED_BY, help="Recorded on the reset watermark")+ parser.add_argument("--apply", action="store_true", default=False, help="Clear them (default: report only)")++ def handle(self, *args: Any, **options: Any) -> None:+ node_ids: list[UUID] | None = options["node_ids"]+ saved_query_ids: list[UUID] | None = options["saved_query_ids"]+ # An empty fragment is a substring of every reason, so it would select every marker.+ if any(not fragment.strip() for fragment in options["reason_contains"]):+ raise CommandError("--reason-contains cannot be empty.")+ if not (node_ids or saved_query_ids or options["reason_contains"]):+ raise CommandError("Name the set: pass --node-ids, --saved-query-ids or --reason-contains.")++ selection = Selection(+ engine=options["engine"],+ reason_contains=tuple(options["reason_contains"]),+ suspended_from=options["suspended_from"],+ suspended_to=options["suspended_to"],+ )+ markers = list(self._markers(options["team_id"], node_ids, saved_query_ids, selection))++ for marker in markers:+ self.stdout.write(+ f"team={marker.node.team_id} node={marker.node.id} saved_query={marker.node.saved_query_id} "+ f"engine={marker.engine} at={marker.at} reason={marker.reason[:160]!r}"+ )+ teams = len({marker.node.team_id for marker in markers})+ self.stdout.write(f"Matched {len(markers)} marker(s) across {teams} team(s).")++ if not options["apply"]:+ self.stdout.write("Dry run. Re-run with --apply to clear them.")+ return++ # one call per marker, because clearing every engine of a node would free markers outside the selection+ cleared = sum(+ unsuspend_nodes([marker.node], by=options["by"], engine=marker.engine, only_if=marker.unchanged)+ for marker in markers+ )+ self.stdout.write(f"Cleared {cleared} marker(s).")+ if cleared < len(markers):+ self.stdout.write(f" {len(markers) - cleared} re-suspended or changed since the report; left alone.")++ def _markers(+ self,+ team_id: int | None,+ node_ids: list[UUID] | None,+ saved_query_ids: list[UUID] | None,+ selection: Selection,+ ) -> Iterator[Marker]:+ nodes = Node.objects.filter(properties__system__has_key="suspended")+ if team_id is not None:+ nodes = nodes.filter(team_id=team_id)+ if node_ids:+ nodes = nodes.filter(id__in=node_ids)+ if saved_query_ids:+ nodes = nodes.filter(saved_query_id__in=saved_query_ids)++ for node in nodes.order_by("team_id", "id").iterator():+ for engine, entry in suspension_state(node).items():+ if isinstance(entry, dict) and selection.matches(engine, entry):+ yield Marker(node=node, engine=engine, at=entry.get("at") or "", reason=entry.get("reason") or "")diff --git a/products/data_modeling/backend/test/test_clear_node_suspensions.py b/products/data_modeling/backend/test/test_clear_node_suspensions.pynew file mode 100644index 000000000000..1e4c960e644b--- /dev/null+++ b/products/data_modeling/backend/test/test_clear_node_suspensions.py@@ -0,0 +1,122 @@+from io import StringIO+from uuid import uuid4++import pytest+from posthog.test.base import BaseTest++from django.core.management import call_command+from django.core.management.base import CommandError++from parameterized import parameterized++from products.data_modeling.backend.logic.node_suspension import is_node_suspended, suspension_state+from products.data_modeling.backend.logic.saved_query_dag_sync import sync_saved_query_to_dag+from products.data_modeling.backend.models.data_modeling_job import DataModelingJobEngine+from products.data_modeling.backend.models.datawarehouse_saved_query import DataWarehouseSavedQuery+from products.data_modeling.backend.models.node import Node++CLICKHOUSE = DataModelingJobEngine.CLICKHOUSE+TRINO = DataModelingJobEngine.MANAGED_WAREHOUSE+OVERFLOW = "Code: 407. DB::Exception: Can't compare decimal number due to overflow. (DECIMAL_OVERFLOW)"+CUSTOMER_ERROR = "Code: 47. DB::Exception: Unknown expression identifier 'foo'"+INCIDENT = "2026-09-30T00:30:00+00:00"+++class TestClearNodeSuspensions(BaseTest):+ def _node(self, markers: dict[str, tuple[str, str]]) -> Node:+ saved_query = DataWarehouseSavedQuery.objects.create(+ name=f"model_{uuid4().hex[:8]}",+ team=self.team,+ query={"query": "SELECT 1", "kind": "HogQLQuery"},+ )+ node = sync_saved_query_to_dag(saved_query)+ assert node is not None+ node.properties = {+ **(node.properties or {}),+ "system": {+ "suspended": {+ engine: {"at": at, "reason": reason, "job_id": str(uuid4()), "query_fingerprint": None}+ for engine, (reason, at) in markers.items()+ }+ },+ }+ node.save()+ return node++ def _run(self, *args: str) -> str:+ out = StringIO()+ call_command("clear_node_suspensions", *args, stdout=out)+ return out.getvalue()++ @parameterized.expand(+ [+ ("by_reason_and_engine", ["--engine", "clickhouse", "--reason-contains", "DECIMAL_OVERFLOW"]),+ (+ "by_reason_inside_window",+ [+ "--reason-contains",+ "DECIMAL_OVERFLOW",+ "--engine",+ "clickhouse",+ "--suspended-from",+ "2026-09-29T00:00:00+00:00",+ "--suspended-to",+ "2026-10-01T00:00:00+00:00",+ ],+ ),+ ]+ )+ def test_clears_only_the_selected_markers(self, _name: str, selection: list[str]) -> None:+ incident = self._node({CLICKHOUSE: (OVERFLOW, INCIDENT), TRINO: (CUSTOMER_ERROR, INCIDENT)})+ same_error_outside_window = self._node({CLICKHOUSE: (OVERFLOW, "2026-10-12T00:00:00+00:00")})+ customer = self._node({CLICKHOUSE: (CUSTOMER_ERROR, INCIDENT)})+ expected_to_survive = [(customer, CLICKHOUSE), (incident, TRINO)]+ if "--suspended-from" in selection:+ expected_to_survive.append((same_error_outside_window, CLICKHOUSE))++ dry_run = self._run(*selection)++ incident.refresh_from_db()+ assert is_node_suspended(incident, CLICKHOUSE)+ assert "Dry run" in dry_run++ self._run(*selection, "--apply")++ for node, engine in expected_to_survive:+ node.refresh_from_db()+ assert is_node_suspended(node, engine), (node.id, engine)+ incident.refresh_from_db()+ assert not is_node_suspended(incident, CLICKHOUSE)+ assert incident.properties["system"]["suspension_reset"][CLICKHOUSE]["by"] == "operator"++ def test_selects_by_saved_query_across_engines(self) -> None:+ node = self._node({CLICKHOUSE: (OVERFLOW, INCIDENT), TRINO: (CUSTOMER_ERROR, INCIDENT)})+ other = self._node({CLICKHOUSE: (OVERFLOW, INCIDENT)})++ self._run("--saved-query-ids", str(node.saved_query_id), "--apply")++ node.refresh_from_db()+ other.refresh_from_db()+ assert suspension_state(node) == {}+ assert is_node_suspended(other, CLICKHOUSE)++ @parameterized.expand(+ [+ ("no_selector", ["--engine", "clickhouse"], "Name the set"),+ ("empty_reason", ["--reason-contains", ""], "empty"),+ (+ "blank_reason_among_others",+ ["--reason-contains", "DECIMAL_OVERFLOW", "--reason-contains", " "],+ "empty",+ ),+ ]+ )+ def test_refuses_without_a_real_selector(self, _name: str, args: list[str], message: str) -> None:+ node = self._node({CLICKHOUSE: (OVERFLOW, INCIDENT)})++ with pytest.raises(CommandError, match=message):+ self._run(*args, "--apply")++ node.refresh_from_db()+ assert is_node_suspended(node, CLICKHOUSE)