Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-modeling): send fewer and lighter queries under the kill switch (#123986)

Sinan Kucukkoselerfa45926e12b3
diff --git a/posthog/temporal/data_modeling/__init__.py b/posthog/temporal/data_modeling/__init__.py
index 589d1067bc7b..5a249337afb3 100644
--- a/posthog/temporal/data_modeling/__init__.py
+++ b/posthog/temporal/data_modeling/__init__.py
@@ -6,6 +6,7 @@ from posthog.temporal.data_modeling.activities import (
create_data_modeling_job_activity,
enrich_view_semantics_activity,
fail_materialization_activity,
+ get_clickhouse_kill_switch_level_activity,
get_dag_structure_activity,
materialize_view_activity,
materialize_view_managed_warehouse_activity,
@@ -38,6 +39,7 @@ ACTIVITIES = [
clear_cdp_staging_activity,
create_data_modeling_job_activity,
record_skipped_data_modeling_jobs_activity,
+ get_clickhouse_kill_switch_level_activity,
get_dag_structure_activity,
fail_materialization_activity,
materialize_view_activity,
diff --git a/posthog/temporal/data_modeling/activities/__init__.py b/posthog/temporal/data_modeling/activities/__init__.py
index 6581d5b43f1f..ace834b9824a 100644
--- a/posthog/temporal/data_modeling/activities/__init__.py
+++ b/posthog/temporal/data_modeling/activities/__init__.py
@@ -9,6 +9,7 @@ from .create_data_modeling_job import (
from .enrich_view_semantics import EnrichViewSemanticsInputs, enrich_view_semantics_activity
from .fail_materialization import FailMaterializationInputs, fail_materialization_activity
from .get_dag_structure import GetDAGStructureInputs, get_dag_structure_activity
+from .kill_switch import get_clickhouse_kill_switch_level_activity
from .materialize_view import (
ClearCDPStagingInputs,
MaterializeViewInputs,
@@ -81,6 +82,7 @@ __all__ = [
"materialize_view_activity",
"materialize_view_managed_warehouse_activity",
"get_dag_structure_activity",
+ "get_clickhouse_kill_switch_level_activity",
"preempt_dag_run_activity",
"prepare_queryable_table_activity",
"publish_queryable_table_activity",
diff --git a/posthog/temporal/data_modeling/activities/kill_switch.py b/posthog/temporal/data_modeling/activities/kill_switch.py
new file mode 100644
index 000000000000..a7e80ef07742
--- /dev/null
+++ b/posthog/temporal/data_modeling/activities/kill_switch.py
@@ -0,0 +1,10 @@
+from temporalio import activity
+
+from posthog.clickhouse.client.execute import resolve_kill_switch_level
+from posthog.sync import database_sync_to_async_pool
+
+
+async def get_clickhouse_kill_switch_level_activity(team_id: int) -> str:
+ level = await database_sync_to_async_pool(resolve_kill_switch_level)(team_id)
+ return level.value
diff --git a/posthog/temporal/data_modeling/activities/materialize_view.py b/posthog/temporal/data_modeling/activities/materialize_view.py
index aac71908ae21..83b88b4c19a6 100644
--- a/posthog/temporal/data_modeling/activities/materialize_view.py
+++ b/posthog/temporal/data_modeling/activities/materialize_view.py
@@ -31,7 +31,7 @@ from posthog.hogql.query import HogQLQueryExecutor
from posthog.hogql.resolver import ResolverFactory
from posthog.hogql.visitor import CloningVisitor
-from posthog.clickhouse.client.execute import ClickHouseExternalTable
+from posthog.clickhouse.client.execute import ClickHouseExternalTable, KillSwitchLevel, resolve_kill_switch_level
from posthog.clickhouse.query_tagging import Feature, Product, tag_queries, tags_context
from posthog.dataclasses import frozen
from posthog.exceptions_capture import capture_exception
@@ -207,6 +207,8 @@ DELTA_KERNEL_ERROR_RETRY_DELAY = dt.timedelta(minutes=2)
# all back within seconds, so wait longer, and jitter the wait so they do not return together.
TOO_MANY_SIMULTANEOUS_QUERIES_RETRY_DELAY_SECONDS = (60, 120)
+KILL_SWITCH_MAX_THREADS: dict[KillSwitchLevel, int] = {KillSwitchLevel.LIGHT: 16, KillSwitchLevel.FULL: 8}
+
# Above this many files, the per-run compaction is worth its full-table rewrite. Below it, skipping
# keeps an incremental run's cost proportional to the rows it changed rather than the table's size.
INCREMENTAL_COMPACT_FILE_THRESHOLD = 200
@@ -613,6 +615,9 @@ async def hogql_table(
settings = HogQLGlobalSettings()
settings.max_execution_time = HOGQL_INCREASED_MAX_EXECUTION_TIME
+ kill_switch_level = await database_sync_to_async_pool(resolve_kill_switch_level)(team.pk)
+ if kill_switch_level in KILL_SWITCH_MAX_THREADS:
+ settings.max_threads = KILL_SWITCH_MAX_THREADS[kill_switch_level]
modifiers = await database_sync_to_async_pool(create_default_modifiers_for_team)(team)
context = HogQLContext(
diff --git a/posthog/temporal/data_modeling/workflows/execute_dag.py b/posthog/temporal/data_modeling/workflows/execute_dag.py
index 5911586de7f8..8d2d590fe856 100644
--- a/posthog/temporal/data_modeling/workflows/execute_dag.py
+++ b/posthog/temporal/data_modeling/workflows/execute_dag.py
@@ -20,6 +20,7 @@ from posthog.temporal.data_modeling.activities import (
RecordSkippedDataModelingJobsInputs,
SkippedDataModelingNode,
check_team_managed_warehouse_shadow_eligibility_activity,
+ get_clickhouse_kill_switch_level_activity,
get_dag_structure_activity,
notify_dag_materialization_failures_activity,
preempt_dag_run_activity,
@@ -49,6 +50,13 @@ MAX_CONCURRENT_CHILDREN = 10
TRINO_SHADOW_MAX_CONCURRENT_CHILDREN = 25
TRINO_SHADOW_CONCURRENCY_PATCH = "data-modeling-trino-shadow-concurrency"
+KILL_SWITCH_WINDOW_PATCH = "data-modeling-kill-switch-window"
+# Under a ClickHouse kill switch a DAG run starts fewer materializations at once, so the CPU that the
+# switch frees goes to interactive queries. A node that waits for a slot waits inside this workflow and
+# holds no activity slot, so a smaller window does not grow the task queue that the worker scales on.
+KILL_SWITCH_MAX_CONCURRENT_CHILDREN = {"off": MAX_CONCURRENT_CHILDREN, "light": 3, "full": 1}
+KILL_SWITCH_RECHECK_INTERVAL = dt.timedelta(minutes=2)
+
TRINO_SHADOW_DAG_RUN_PATCH = "data-modeling-trino-shadow-dag-run"
# Under a day, so a run that uses its whole budget still ends before the next daily shadow starts.
TRINO_SHADOW_RUN_TIMEOUT = dt.timedelta(hours=23)
@@ -69,6 +77,78 @@ def trino_shadow_workflow_id(workflow_id: str, dag_id: str) -> str:
return f"execute-dag-trino-{dag_id}"
+async def _kill_switch_child_limit(team_id: int) -> int:
+ try:
+ level = await temporalio.workflow.execute_activity(
+ get_clickhouse_kill_switch_level_activity,
+ team_id,
+ start_to_close_timeout=dt.timedelta(seconds=30),
+ retry_policy=temporalio.common.RetryPolicy(maximum_attempts=1),
+ )
+ except temporalio.exceptions.ActivityError:
+ return MAX_CONCURRENT_CHILDREN
+ return KILL_SWITCH_MAX_CONCURRENT_CHILDREN.get(level, MAX_CONCURRENT_CHILDREN)
+
+
+class _KillSwitchChildWindow:
+ def __init__(self, team_id: int) -> None:
+ self._team_id = team_id
+ self._limit = MAX_CONCURRENT_CHILDREN
+ self._running = 0
+ self._waiting = 0
+ self._checked_at: dt.datetime | None = None
+ self._reading = False
+ self._closed = False
+ self._refresher: asyncio.Task[None] | None = None
+
+ async def _refresh(self) -> None:
+ if self._closed:
+ return
+ # The children of a level all arrive in the same instant. Without this wait they would enter
+ # with the old limit while the first one is still reading the new one.
+ if self._reading:
+ await temporalio.workflow.wait_condition(lambda: not self._reading)
+ return
+ now = temporalio.workflow.now()
+ if self._checked_at is not None and now - self._checked_at < KILL_SWITCH_RECHECK_INTERVAL:
+ return
+ self._reading = True
+ try:
+ self._checked_at = now
+ self._limit = await _kill_switch_child_limit(self._team_id)
+ finally:
+ self._reading = False
+
+ async def _refresh_while_children_wait(self) -> None:
+ # One shared timer rereads the level. A timeout on each waiter would put a timer in the history
+ # for every waiter on every completion, and frequent completions would keep resetting those
+ # timeouts so that the level was never read again.
+ while True:
+ await temporalio.workflow.wait_condition(lambda: self._waiting > 0 or self._closed)
+ if self._closed:
+ return
+ await temporalio.workflow.sleep(KILL_SWITCH_RECHECK_INTERVAL)
+ await self._refresh()
+
+ def close(self) -> None:
+ self._closed = True
+
+ async def __aenter__(self) -> None:
+ if self._refresher is None:
+ self._refresher = asyncio.create_task(self._refresh_while_children_wait())
+ await self._refresh()
+ while self._running >= self._limit:
+ self._waiting += 1
+ try:
+ await temporalio.workflow.wait_condition(lambda: self._running < self._limit)
+ finally:
+ self._waiting -= 1
+ self._running += 1
+
+ async def __aexit__(self, *_: object) -> None:
+ self._running -= 1
+
+
class EmptyDAGOrCycleError(Exception):
"""Raised when the DAG is empty or contains a cycle according to _dag_execution_levels."""
@@ -336,7 +416,13 @@ class ExecuteDAGWorkflow(PostHogWorkflow):
max_concurrent_children = MAX_CONCURRENT_CHILDREN
if managed_warehouse_only and temporalio.workflow.patched(TRINO_SHADOW_CONCURRENCY_PATCH):
max_concurrent_children = TRINO_SHADOW_MAX_CONCURRENT_CHILDREN
- semaphore = asyncio.Semaphore(max_concurrent_children)
+ # Histories from before the patch keep the semaphore, because its wake-up order decides the
+ # order of child-start commands, and replay must reproduce that order.
+ child_window: asyncio.Semaphore | _KillSwitchChildWindow
+ if not managed_warehouse_only and temporalio.workflow.patched(KILL_SWITCH_WINDOW_PATCH):
+ child_window = _KillSwitchChildWindow(inputs.team_id)
+ else:
+ child_window = asyncio.Semaphore(max_concurrent_children)
for i, level in enumerate(levels):
temporalio.workflow.logger.info(
f"Executing level {i + 1}/{len(levels)}",
@@ -414,7 +500,7 @@ class ExecuteDAGWorkflow(PostHogWorkflow):
continue
async def _run_child(node_id: str) -> NodeResult:
- async with semaphore:
+ async with child_window:
blocked_trino_upstreams = sorted(
upstream for upstream in unavailable_trino_nodes if node_id in downstreams[upstream]
)
@@ -518,6 +604,9 @@ class ExecuteDAGWorkflow(PostHogWorkflow):
if nr.quality_failed:
quality_failed_node_set.add(nr.node_id)
+ if isinstance(child_window, _KillSwitchChildWindow):
+ child_window.close()
+
if skipped_jobs:
await temporalio.workflow.execute_activity(
record_skipped_data_modeling_jobs_activity,
diff --git a/posthog/temporal/tests/data_modeling/test_execute_dag_workflow.py b/posthog/temporal/tests/data_modeling/test_execute_dag_workflow.py
index b0036cd83893..ac81998d06f8 100644
--- a/posthog/temporal/tests/data_modeling/test_execute_dag_workflow.py
+++ b/posthog/temporal/tests/data_modeling/test_execute_dag_workflow.py
@@ -12,6 +12,7 @@ from temporalio import (
activity as temporal_activity,
workflow as temporal_workflow,
)
+from temporalio.api.enums.v1 import EventType
from temporalio.exceptions import WorkflowAlreadyStartedError
from temporalio.testing import WorkflowEnvironment
from temporalio.workflow import ParentClosePolicy
@@ -575,6 +576,22 @@ class MockMaterializeViewWorkflow:
)
+_child_concurrency = {"running": 0, "peak": 0, "sleep_seconds": 10}
+
+
+@temporal_workflow.defn(name="data-modeling-materialize-view")
+class SlowMockMaterializeViewWorkflow:
+ @temporal_workflow.run
+ async def run(self, inputs: MaterializeViewWorkflowInputs) -> MaterializeViewWorkflowResult:
+ _child_concurrency["running"] += 1
+ _child_concurrency["peak"] = max(_child_concurrency["peak"], _child_concurrency["running"])
+ await temporal_workflow.sleep(dt.timedelta(seconds=_child_concurrency["sleep_seconds"]))
+ _child_concurrency["running"] -= 1
+ return MaterializeViewWorkflowResult(
+ job_id="test-job", node_id=inputs.node_id, rows_materialized=1, duration_seconds=10.0
+ )
+
+
class TestExecuteDAGWorkflowWithMocks:
"""Tests that use module-level mock workflows to test orchestration behavior."""
@@ -924,6 +941,97 @@ class TestExecuteDAGWorkflowWithMocks:
assert all(r.success for r in result.node_results)
assert _notified_dag_failures == []
+ @pytest.mark.parametrize(
+ "kill_switch_level,expected_peak",
+ [
+ ("off", 4),
+ ("light", 3),
+ ("full", 1),
+ (None, 4),
+ ],
+ )
+ async def test_kill_switch_narrows_the_child_window(self, kill_switch_level, expected_peak):
+ node_ids = [str(uuid.uuid4()) for _ in range(4)]
+ _child_concurrency.update(running=0, peak=0, sleep_seconds=10)
+
+ @temporal_activity.defn(name="get_dag_structure_activity")
+ async def stub_get_dag_structure(_: GetDAGStructureInputs) -> DAGPlan:
+ return DAGPlan(nodes=node_ids, executable_nodes=node_ids, edges=[])
+
+ @temporal_activity.defn(name="get_clickhouse_kill_switch_level_activity")
+ async def stub_kill_switch_level(_: int) -> str:
+ if kill_switch_level is None:
+ raise temporalio.exceptions.ApplicationError("instance settings unavailable")
+ return kill_switch_level
+
+ async with await WorkflowEnvironment.start_time_skipping() as env:
+ async with temporalio.worker.Worker(
+ env.client,
+ task_queue="test-queue",
+ workflows=[ExecuteDAGWorkflow, SlowMockMaterializeViewWorkflow],
+ activities=[
+ stub_preempt_dag_run,
+ stub_check_team_shadow_eligibility,
+ stub_get_dag_structure,
+ stub_kill_switch_level,
+ stub_notify_dag_materialization_failures,
+ ],
+ workflow_runner=temporalio.worker.UnsandboxedWorkflowRunner(),
+ ):
+ result: ExecuteDAGResult = await env.client.execute_workflow(
+ ExecuteDAGWorkflow.run,
+ ExecuteDAGInputs(team_id=1, dag_id="test-dag"),
+ id=f"test-kill-switch-window-{uuid.uuid4()}",
+ task_queue="test-queue",
+ execution_timeout=dt.timedelta(minutes=10),
+ )
+
+ assert result.successful_nodes == 4
+ assert _child_concurrency["peak"] == expected_peak
+
+ async def test_kill_switch_is_reread_while_children_keep_finishing(self):
+ node_ids = [str(uuid.uuid4()) for _ in range(12)]
+ _child_concurrency.update(running=0, peak=0, sleep_seconds=30)
+ level_reads: list[str] = []
+ workflow_id = f"test-kill-switch-reread-{uuid.uuid4()}"
+
+ @temporal_activity.defn(name="get_dag_structure_activity")
+ async def stub_get_dag_structure(_: GetDAGStructureInputs) -> DAGPlan:
+ return DAGPlan(nodes=node_ids, executable_nodes=node_ids, edges=[])
+
+ @temporal_activity.defn(name="get_clickhouse_kill_switch_level_activity")
+ async def stub_kill_switch_level(_: int) -> str:
+ level_reads.append("read")
+ return "full" if len(level_reads) == 1 else "off"
+
+ async with await WorkflowEnvironment.start_time_skipping() as env:
+ async with temporalio.worker.Worker(
+ env.client,
+ task_queue="test-queue",
+ workflows=[ExecuteDAGWorkflow, SlowMockMaterializeViewWorkflow],
+ activities=[
+ stub_preempt_dag_run,
+ stub_check_team_shadow_eligibility,
+ stub_get_dag_structure,
+ stub_kill_switch_level,
+ stub_notify_dag_materialization_failures,
+ ],
+ workflow_runner=temporalio.worker.UnsandboxedWorkflowRunner(),
+ ):
+ result: ExecuteDAGResult = await env.client.execute_workflow(
+ ExecuteDAGWorkflow.run,
+ ExecuteDAGInputs(team_id=1, dag_id="test-dag"),
+ id=workflow_id,
+ task_queue="test-queue",
+ execution_timeout=dt.timedelta(minutes=30),
+ )
+ history = await env.client.get_workflow_handle(workflow_id).fetch_history()
+
+ timers_started = sum(1 for event in history.events if event.event_type == EventType.EVENT_TYPE_TIMER_STARTED)
+ assert result.successful_nodes == 12
+ assert _child_concurrency["peak"] > 1
+ assert timers_started <= 2
+
async def test_ephemeral_nodes_are_skipped(self):
"""Test that ephemeral nodes are recorded as successful no-ops without starting child workflows."""
dag_id = "test-dag"
diff --git a/posthog/temporal/tests/data_modeling/test_materialize_view_activities.py b/posthog/temporal/tests/data_modeling/test_materialize_view_activities.py
index cf0cd5fcd271..deb4e7738dd3 100644
--- a/posthog/temporal/tests/data_modeling/test_materialize_view_activities.py
+++ b/posthog/temporal/tests/data_modeling/test_materialize_view_activities.py
@@ -23,6 +23,7 @@ from trino.exceptions import TrinoUserError
from posthog.hogql.resolver import ResolverFactory
from posthog.hogql.transforms.trino.errors import TrinoLoweringError
+from posthog.clickhouse.client.execute import KillSwitchLevel
from posthog.clickhouse.query_tagging import get_query_tags
from posthog.models import Team, User
from posthog.sync import database_sync_to_async
@@ -2256,6 +2257,40 @@ class TestHogqlTableModifiers:
)
assert captured_sql is not None and expected_sql in captured_sql
+ @pytest.mark.parametrize(
+ "kill_switch_level,expected_max_threads",
+ [
+ (KillSwitchLevel.OFF, None),
+ (KillSwitchLevel.LIGHT, 16),
+ (KillSwitchLevel.FULL, 8),
+ ],
+ )
+ async def test_kill_switch_lowers_the_threads_per_query(
+ self, ateam: Team, kill_switch_level: KillSwitchLevel, expected_max_threads: int | None
+ ) -> None:
+ client = _EmptyArrowClient(pa.schema([pa.field("id", pa.int64())]))
+
+ @contextlib.asynccontextmanager
+ async def fake_get_client(**kwargs: Any) -> AsyncIterator[_EmptyArrowClient]:
+ yield client
+
+ with (
+ unittest.mock.patch(
+ "posthog.temporal.data_modeling.activities.materialize_view.get_clickhouse_client", fake_get_client
+ ),
+ unittest.mock.patch(
+ "posthog.temporal.data_modeling.activities.materialize_view.resolve_kill_switch_level",
+ lambda _team_id: kill_switch_level,
+ ),
+ ):
+ _ = [batch async for batch in hogql_table("SELECT 1", ateam, LOGGER.bind())]
+
+ assert client.arrow_query is not None
+ if expected_max_threads is None:
+ assert "max_threads" not in client.arrow_query
+ else:
+ assert f"max_threads={expected_max_threads}" in client.arrow_query
+
async def test_compiles_account_dependencies_without_a_user(self, ateam: Team) -> None:
schema_fields: list[pa.Field[Any]] = [
pa.field("id", pa.string()),