Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-modeling): add a command to clear chosen node suspensions (#121112)

Jovan Sakovicb3a98dd6ff70
diff --git a/products/data_modeling/backend/management/commands/clear_node_suspensions.py b/products/data_modeling/backend/management/commands/clear_node_suspensions.py
new file mode 100644
index 000000000000..965679db5a55
--- /dev/null
+++ b/products/data_modeling/backend/management/commands/clear_node_suspensions.py
@@ -0,0 +1,160 @@
+"""Clear suspension markers on a chosen set of nodes.
+
+`clear_unearned_node_suspensions` frees markers the failure counter would no longer produce. This
+command is for markers the counter did earn, but for a cause on our side: a ClickHouse incident that
+failed every run with the same error, or a Trino shadow compiler gap that has since been fixed.
+The operator names the set; the command never decides what deserves to be cleared.
+
+At least one of --node-ids, --saved-query-ids or --reason-contains is required, so a run cannot
+clear the whole fleet by omission. Filters combine with AND. Region-agnostic - run it once per region.
+
+ python manage.py clear_node_suspensions --engine clickhouse --reason-contains TOO_MANY_SIMULTANEOUS_QUERIES \\
+ --suspended-from <iso-time> --suspended-to <iso-time>
+ python manage.py clear_node_suspensions --engine managed_warehouse --team-id 2 \\
+ --reason-contains TRINO_AST_EXPANSION_LIMIT --apply
+ python manage.py clear_node_suspensions --saved-query-ids <uuid>,<uuid> --apply
+"""
+
+import datetime as dt
+from collections.abc import Iterator
+from dataclasses import dataclass
+from typing import Any
+from uuid import UUID
+
+from django.core.management.base import BaseCommand, CommandError, CommandParser
+
+from products.data_modeling.backend.logic.node_suspension import suspension_state, unsuspend_nodes
+from products.data_modeling.backend.models.node import Node
+
+RESUMED_BY = "operator"
+
+
+def _uuid_list(value: str) -> list[UUID]:
+ return [UUID(item.strip()) for item in value.split(",") if item.strip()]
+
+
+def _aware_datetime(value: str) -> dt.datetime:
+ parsed = dt.datetime.fromisoformat(value)
+ if parsed.tzinfo is None:
+ raise ValueError("include a UTC offset, for example 2026-09-29T00:00:00+00:00")
+ return parsed
+
+
+@dataclass(frozen=True, kw_only=True)
+class Selection:
+ engine: str | None
+ reason_contains: tuple[str, ...]
+ suspended_from: dt.datetime | None
+ suspended_to: dt.datetime | None
+
+ def matches(self, engine: str, entry: dict) -> bool:
+ if self.engine is not None and engine != self.engine:
+ return False
+ reason = entry.get("reason") or ""
+ if self.reason_contains and not any(fragment in reason for fragment in self.reason_contains):
+ return False
+ if self.suspended_from is None and self.suspended_to is None:
+ return True
+ try:
+ at = dt.datetime.fromisoformat(entry.get("at") or "")
+ except ValueError:
+ return False # a window was asked for, and a marker with no readable time is outside it
+ if self.suspended_from is not None and at < self.suspended_from:
+ return False
+ return self.suspended_to is None or at < self.suspended_to
+
+
+@dataclass(frozen=True, kw_only=True)
+class Marker:
+ node: Node
+ engine: str
+ at: str
+ reason: str
+
+ def unchanged(self, node: Node) -> bool:
+ """A model can re-suspend between the report and the write. A newer marker records a newer
+ failure the operator never saw, so it stays."""
+ return (suspension_state(node).get(self.engine) or {}).get("at") == self.at
+
+
+class Command(BaseCommand):
+ help = "Clear data modeling suspension markers on an operator-chosen set of nodes"
+
+ def add_arguments(self, parser: CommandParser) -> None:
+ parser.add_argument("--team-id", type=int, default=None, help="Limit to one team")
+ parser.add_argument("--node-ids", type=_uuid_list, default=None, help="Comma-separated node ids")
+ parser.add_argument(
+ "--saved-query-ids",
+ type=_uuid_list,
+ default=None,
+ help="Comma-separated saved query ids; selects every node backing each query",
+ )
+ parser.add_argument("--engine", default=None, help="Only markers for this engine, e.g. clickhouse")
+ parser.add_argument(
+ "--reason-contains",
+ action="append",
+ default=[],
+ help="Only markers whose recorded error contains this text. Repeat to match any of several.",
+ )
+ parser.add_argument("--suspended-from", type=_aware_datetime, default=None, help="ISO time, inclusive")
+ parser.add_argument("--suspended-to", type=_aware_datetime, default=None, help="ISO time, exclusive")
+ parser.add_argument("--by", default=RESUMED_BY, help="Recorded on the reset watermark")
+ parser.add_argument("--apply", action="store_true", default=False, help="Clear them (default: report only)")
+
+ def handle(self, *args: Any, **options: Any) -> None:
+ node_ids: list[UUID] | None = options["node_ids"]
+ saved_query_ids: list[UUID] | None = options["saved_query_ids"]
+ # An empty fragment is a substring of every reason, so it would select every marker.
+ if any(not fragment.strip() for fragment in options["reason_contains"]):
+ raise CommandError("--reason-contains cannot be empty.")
+ if not (node_ids or saved_query_ids or options["reason_contains"]):
+ raise CommandError("Name the set: pass --node-ids, --saved-query-ids or --reason-contains.")
+
+ selection = Selection(
+ engine=options["engine"],
+ reason_contains=tuple(options["reason_contains"]),
+ suspended_from=options["suspended_from"],
+ suspended_to=options["suspended_to"],
+ )
+ markers = list(self._markers(options["team_id"], node_ids, saved_query_ids, selection))
+
+ for marker in markers:
+ self.stdout.write(
+ f"team={marker.node.team_id} node={marker.node.id} saved_query={marker.node.saved_query_id} "
+ f"engine={marker.engine} at={marker.at} reason={marker.reason[:160]!r}"
+ )
+ teams = len({marker.node.team_id for marker in markers})
+ self.stdout.write(f"Matched {len(markers)} marker(s) across {teams} team(s).")
+
+ if not options["apply"]:
+ self.stdout.write("Dry run. Re-run with --apply to clear them.")
+ return
+
+ # one call per marker, because clearing every engine of a node would free markers outside the selection
+ cleared = sum(
+ unsuspend_nodes([marker.node], by=options["by"], engine=marker.engine, only_if=marker.unchanged)
+ for marker in markers
+ )
+ self.stdout.write(f"Cleared {cleared} marker(s).")
+ if cleared < len(markers):
+ self.stdout.write(f" {len(markers) - cleared} re-suspended or changed since the report; left alone.")
+
+ def _markers(
+ self,
+ team_id: int | None,
+ node_ids: list[UUID] | None,
+ saved_query_ids: list[UUID] | None,
+ selection: Selection,
+ ) -> Iterator[Marker]:
+ nodes = Node.objects.filter(properties__system__has_key="suspended")
+ if team_id is not None:
+ nodes = nodes.filter(team_id=team_id)
+ if node_ids:
+ nodes = nodes.filter(id__in=node_ids)
+ if saved_query_ids:
+ nodes = nodes.filter(saved_query_id__in=saved_query_ids)
+
+ for node in nodes.order_by("team_id", "id").iterator():
+ for engine, entry in suspension_state(node).items():
+ if isinstance(entry, dict) and selection.matches(engine, entry):
+ yield Marker(node=node, engine=engine, at=entry.get("at") or "", reason=entry.get("reason") or "")
diff --git a/products/data_modeling/backend/test/test_clear_node_suspensions.py b/products/data_modeling/backend/test/test_clear_node_suspensions.py
new file mode 100644
index 000000000000..1e4c960e644b
--- /dev/null
+++ b/products/data_modeling/backend/test/test_clear_node_suspensions.py
@@ -0,0 +1,122 @@
+from io import StringIO
+from uuid import uuid4
+
+import pytest
+from posthog.test.base import BaseTest
+
+from django.core.management import call_command
+from django.core.management.base import CommandError
+
+from parameterized import parameterized
+
+from products.data_modeling.backend.logic.node_suspension import is_node_suspended, suspension_state
+from products.data_modeling.backend.logic.saved_query_dag_sync import sync_saved_query_to_dag
+from products.data_modeling.backend.models.data_modeling_job import DataModelingJobEngine
+from products.data_modeling.backend.models.datawarehouse_saved_query import DataWarehouseSavedQuery
+from products.data_modeling.backend.models.node import Node
+
+CLICKHOUSE = DataModelingJobEngine.CLICKHOUSE
+TRINO = DataModelingJobEngine.MANAGED_WAREHOUSE
+OVERFLOW = "Code: 407. DB::Exception: Can't compare decimal number due to overflow. (DECIMAL_OVERFLOW)"
+CUSTOMER_ERROR = "Code: 47. DB::Exception: Unknown expression identifier 'foo'"
+INCIDENT = "2026-09-30T00:30:00+00:00"
+
+
+class TestClearNodeSuspensions(BaseTest):
+ def _node(self, markers: dict[str, tuple[str, str]]) -> Node:
+ saved_query = DataWarehouseSavedQuery.objects.create(
+ name=f"model_{uuid4().hex[:8]}",
+ team=self.team,
+ query={"query": "SELECT 1", "kind": "HogQLQuery"},
+ )
+ node = sync_saved_query_to_dag(saved_query)
+ assert node is not None
+ node.properties = {
+ **(node.properties or {}),
+ "system": {
+ "suspended": {
+ engine: {"at": at, "reason": reason, "job_id": str(uuid4()), "query_fingerprint": None}
+ for engine, (reason, at) in markers.items()
+ }
+ },
+ }
+ node.save()
+ return node
+
+ def _run(self, *args: str) -> str:
+ out = StringIO()
+ call_command("clear_node_suspensions", *args, stdout=out)
+ return out.getvalue()
+
+ @parameterized.expand(
+ [
+ ("by_reason_and_engine", ["--engine", "clickhouse", "--reason-contains", "DECIMAL_OVERFLOW"]),
+ (
+ "by_reason_inside_window",
+ [
+ "--reason-contains",
+ "DECIMAL_OVERFLOW",
+ "--engine",
+ "clickhouse",
+ "--suspended-from",
+ "2026-09-29T00:00:00+00:00",
+ "--suspended-to",
+ "2026-10-01T00:00:00+00:00",
+ ],
+ ),
+ ]
+ )
+ def test_clears_only_the_selected_markers(self, _name: str, selection: list[str]) -> None:
+ incident = self._node({CLICKHOUSE: (OVERFLOW, INCIDENT), TRINO: (CUSTOMER_ERROR, INCIDENT)})
+ same_error_outside_window = self._node({CLICKHOUSE: (OVERFLOW, "2026-10-12T00:00:00+00:00")})
+ customer = self._node({CLICKHOUSE: (CUSTOMER_ERROR, INCIDENT)})
+ expected_to_survive = [(customer, CLICKHOUSE), (incident, TRINO)]
+ if "--suspended-from" in selection:
+ expected_to_survive.append((same_error_outside_window, CLICKHOUSE))
+
+ dry_run = self._run(*selection)
+
+ incident.refresh_from_db()
+ assert is_node_suspended(incident, CLICKHOUSE)
+ assert "Dry run" in dry_run
+
+ self._run(*selection, "--apply")
+
+ for node, engine in expected_to_survive:
+ node.refresh_from_db()
+ assert is_node_suspended(node, engine), (node.id, engine)
+ incident.refresh_from_db()
+ assert not is_node_suspended(incident, CLICKHOUSE)
+ assert incident.properties["system"]["suspension_reset"][CLICKHOUSE]["by"] == "operator"
+
+ def test_selects_by_saved_query_across_engines(self) -> None:
+ node = self._node({CLICKHOUSE: (OVERFLOW, INCIDENT), TRINO: (CUSTOMER_ERROR, INCIDENT)})
+ other = self._node({CLICKHOUSE: (OVERFLOW, INCIDENT)})
+
+ self._run("--saved-query-ids", str(node.saved_query_id), "--apply")
+
+ node.refresh_from_db()
+ other.refresh_from_db()
+ assert suspension_state(node) == {}
+ assert is_node_suspended(other, CLICKHOUSE)
+
+ @parameterized.expand(
+ [
+ ("no_selector", ["--engine", "clickhouse"], "Name the set"),
+ ("empty_reason", ["--reason-contains", ""], "empty"),
+ (
+ "blank_reason_among_others",
+ ["--reason-contains", "DECIMAL_OVERFLOW", "--reason-contains", " "],
+ "empty",
+ ),
+ ]
+ )
+ def test_refuses_without_a_real_selector(self, _name: str, args: list[str], message: str) -> None:
+ node = self._node({CLICKHOUSE: (OVERFLOW, INCIDENT)})
+
+ with pytest.raises(CommandError, match=message):
+ self._run(*args, "--apply")
+
+ node.refresh_from_db()
+ assert is_node_suspended(node, CLICKHOUSE)