:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-modeling): send fewer and lighter queries under the kill switch (#123986)
Sinan Kucukkoselerfa45926e12b3
diff --git a/posthog/temporal/data_modeling/__init__.py b/posthog/temporal/data_modeling/__init__.pyindex 589d1067bc7b..5a249337afb3 100644--- a/posthog/temporal/data_modeling/__init__.py+++ b/posthog/temporal/data_modeling/__init__.py@@ -6,6 +6,7 @@ from posthog.temporal.data_modeling.activities import (create_data_modeling_job_activity,enrich_view_semantics_activity,fail_materialization_activity,+ get_clickhouse_kill_switch_level_activity,get_dag_structure_activity,materialize_view_activity,materialize_view_managed_warehouse_activity,@@ -38,6 +39,7 @@ ACTIVITIES = [clear_cdp_staging_activity,create_data_modeling_job_activity,record_skipped_data_modeling_jobs_activity,+ get_clickhouse_kill_switch_level_activity,get_dag_structure_activity,fail_materialization_activity,materialize_view_activity,diff --git a/posthog/temporal/data_modeling/activities/__init__.py b/posthog/temporal/data_modeling/activities/__init__.pyindex 6581d5b43f1f..ace834b9824a 100644--- a/posthog/temporal/data_modeling/activities/__init__.py+++ b/posthog/temporal/data_modeling/activities/__init__.py@@ -9,6 +9,7 @@ from .create_data_modeling_job import (from .enrich_view_semantics import EnrichViewSemanticsInputs, enrich_view_semantics_activityfrom .fail_materialization import FailMaterializationInputs, fail_materialization_activityfrom .get_dag_structure import GetDAGStructureInputs, get_dag_structure_activity+from .kill_switch import get_clickhouse_kill_switch_level_activityfrom .materialize_view import (ClearCDPStagingInputs,MaterializeViewInputs,@@ -81,6 +82,7 @@ __all__ = ["materialize_view_activity","materialize_view_managed_warehouse_activity","get_dag_structure_activity",+ "get_clickhouse_kill_switch_level_activity","preempt_dag_run_activity","prepare_queryable_table_activity","publish_queryable_table_activity",diff --git a/posthog/temporal/data_modeling/activities/kill_switch.py b/posthog/temporal/data_modeling/activities/kill_switch.pynew file mode 100644index 000000000000..a7e80ef07742--- /dev/null+++ b/posthog/temporal/data_modeling/activities/kill_switch.py@@ -0,0 +1,10 @@+from temporalio import activity++from posthog.clickhouse.client.execute import resolve_kill_switch_level+from posthog.sync import database_sync_to_async_pool+++async def get_clickhouse_kill_switch_level_activity(team_id: int) -> str:+ level = await database_sync_to_async_pool(resolve_kill_switch_level)(team_id)+ return level.valuediff --git a/posthog/temporal/data_modeling/activities/materialize_view.py b/posthog/temporal/data_modeling/activities/materialize_view.pyindex aac71908ae21..83b88b4c19a6 100644--- a/posthog/temporal/data_modeling/activities/materialize_view.py+++ b/posthog/temporal/data_modeling/activities/materialize_view.py@@ -31,7 +31,7 @@ from posthog.hogql.query import HogQLQueryExecutorfrom posthog.hogql.resolver import ResolverFactoryfrom posthog.hogql.visitor import CloningVisitor-from posthog.clickhouse.client.execute import ClickHouseExternalTable+from posthog.clickhouse.client.execute import ClickHouseExternalTable, KillSwitchLevel, resolve_kill_switch_levelfrom posthog.clickhouse.query_tagging import Feature, Product, tag_queries, tags_contextfrom posthog.dataclasses import frozenfrom posthog.exceptions_capture import capture_exception@@ -207,6 +207,8 @@ DELTA_KERNEL_ERROR_RETRY_DELAY = dt.timedelta(minutes=2)# all back within seconds, so wait longer, and jitter the wait so they do not return together.TOO_MANY_SIMULTANEOUS_QUERIES_RETRY_DELAY_SECONDS = (60, 120)+KILL_SWITCH_MAX_THREADS: dict[KillSwitchLevel, int] = {KillSwitchLevel.LIGHT: 16, KillSwitchLevel.FULL: 8}+# Above this many files, the per-run compaction is worth its full-table rewrite. Below it, skipping# keeps an incremental run's cost proportional to the rows it changed rather than the table's size.INCREMENTAL_COMPACT_FILE_THRESHOLD = 200@@ -613,6 +615,9 @@ async def hogql_table(settings = HogQLGlobalSettings()settings.max_execution_time = HOGQL_INCREASED_MAX_EXECUTION_TIME+ kill_switch_level = await database_sync_to_async_pool(resolve_kill_switch_level)(team.pk)+ if kill_switch_level in KILL_SWITCH_MAX_THREADS:+ settings.max_threads = KILL_SWITCH_MAX_THREADS[kill_switch_level]modifiers = await database_sync_to_async_pool(create_default_modifiers_for_team)(team)context = HogQLContext(diff --git a/posthog/temporal/data_modeling/workflows/execute_dag.py b/posthog/temporal/data_modeling/workflows/execute_dag.pyindex 5911586de7f8..8d2d590fe856 100644--- a/posthog/temporal/data_modeling/workflows/execute_dag.py+++ b/posthog/temporal/data_modeling/workflows/execute_dag.py@@ -20,6 +20,7 @@ from posthog.temporal.data_modeling.activities import (RecordSkippedDataModelingJobsInputs,SkippedDataModelingNode,check_team_managed_warehouse_shadow_eligibility_activity,+ get_clickhouse_kill_switch_level_activity,get_dag_structure_activity,notify_dag_materialization_failures_activity,preempt_dag_run_activity,@@ -49,6 +50,13 @@ MAX_CONCURRENT_CHILDREN = 10TRINO_SHADOW_MAX_CONCURRENT_CHILDREN = 25TRINO_SHADOW_CONCURRENCY_PATCH = "data-modeling-trino-shadow-concurrency"+KILL_SWITCH_WINDOW_PATCH = "data-modeling-kill-switch-window"+# Under a ClickHouse kill switch a DAG run starts fewer materializations at once, so the CPU that the+# switch frees goes to interactive queries. A node that waits for a slot waits inside this workflow and+# holds no activity slot, so a smaller window does not grow the task queue that the worker scales on.+KILL_SWITCH_MAX_CONCURRENT_CHILDREN = {"off": MAX_CONCURRENT_CHILDREN, "light": 3, "full": 1}+KILL_SWITCH_RECHECK_INTERVAL = dt.timedelta(minutes=2)+TRINO_SHADOW_DAG_RUN_PATCH = "data-modeling-trino-shadow-dag-run"# Under a day, so a run that uses its whole budget still ends before the next daily shadow starts.TRINO_SHADOW_RUN_TIMEOUT = dt.timedelta(hours=23)@@ -69,6 +77,78 @@ def trino_shadow_workflow_id(workflow_id: str, dag_id: str) -> str:return f"execute-dag-trino-{dag_id}"+async def _kill_switch_child_limit(team_id: int) -> int:+ try:+ level = await temporalio.workflow.execute_activity(+ get_clickhouse_kill_switch_level_activity,+ team_id,+ start_to_close_timeout=dt.timedelta(seconds=30),+ retry_policy=temporalio.common.RetryPolicy(maximum_attempts=1),+ )+ except temporalio.exceptions.ActivityError:+ return MAX_CONCURRENT_CHILDREN+ return KILL_SWITCH_MAX_CONCURRENT_CHILDREN.get(level, MAX_CONCURRENT_CHILDREN)+++class _KillSwitchChildWindow:+ def __init__(self, team_id: int) -> None:+ self._team_id = team_id+ self._limit = MAX_CONCURRENT_CHILDREN+ self._running = 0+ self._waiting = 0+ self._checked_at: dt.datetime | None = None+ self._reading = False+ self._closed = False+ self._refresher: asyncio.Task[None] | None = None++ async def _refresh(self) -> None:+ if self._closed:+ return+ # The children of a level all arrive in the same instant. Without this wait they would enter+ # with the old limit while the first one is still reading the new one.+ if self._reading:+ await temporalio.workflow.wait_condition(lambda: not self._reading)+ return+ now = temporalio.workflow.now()+ if self._checked_at is not None and now - self._checked_at < KILL_SWITCH_RECHECK_INTERVAL:+ return+ self._reading = True+ try:+ self._checked_at = now+ self._limit = await _kill_switch_child_limit(self._team_id)+ finally:+ self._reading = False++ async def _refresh_while_children_wait(self) -> None:+ # One shared timer rereads the level. A timeout on each waiter would put a timer in the history+ # for every waiter on every completion, and frequent completions would keep resetting those+ # timeouts so that the level was never read again.+ while True:+ await temporalio.workflow.wait_condition(lambda: self._waiting > 0 or self._closed)+ if self._closed:+ return+ await temporalio.workflow.sleep(KILL_SWITCH_RECHECK_INTERVAL)+ await self._refresh()++ def close(self) -> None:+ self._closed = True++ async def __aenter__(self) -> None:+ if self._refresher is None:+ self._refresher = asyncio.create_task(self._refresh_while_children_wait())+ await self._refresh()+ while self._running >= self._limit:+ self._waiting += 1+ try:+ await temporalio.workflow.wait_condition(lambda: self._running < self._limit)+ finally:+ self._waiting -= 1+ self._running += 1++ async def __aexit__(self, *_: object) -> None:+ self._running -= 1++class EmptyDAGOrCycleError(Exception):"""Raised when the DAG is empty or contains a cycle according to _dag_execution_levels."""@@ -336,7 +416,13 @@ class ExecuteDAGWorkflow(PostHogWorkflow):max_concurrent_children = MAX_CONCURRENT_CHILDRENif managed_warehouse_only and temporalio.workflow.patched(TRINO_SHADOW_CONCURRENCY_PATCH):max_concurrent_children = TRINO_SHADOW_MAX_CONCURRENT_CHILDREN- semaphore = asyncio.Semaphore(max_concurrent_children)+ # Histories from before the patch keep the semaphore, because its wake-up order decides the+ # order of child-start commands, and replay must reproduce that order.+ child_window: asyncio.Semaphore | _KillSwitchChildWindow+ if not managed_warehouse_only and temporalio.workflow.patched(KILL_SWITCH_WINDOW_PATCH):+ child_window = _KillSwitchChildWindow(inputs.team_id)+ else:+ child_window = asyncio.Semaphore(max_concurrent_children)for i, level in enumerate(levels):temporalio.workflow.logger.info(f"Executing level {i + 1}/{len(levels)}",@@ -414,7 +500,7 @@ class ExecuteDAGWorkflow(PostHogWorkflow):continueasync def _run_child(node_id: str) -> NodeResult:- async with semaphore:+ async with child_window:blocked_trino_upstreams = sorted(upstream for upstream in unavailable_trino_nodes if node_id in downstreams[upstream])@@ -518,6 +604,9 @@ class ExecuteDAGWorkflow(PostHogWorkflow):if nr.quality_failed:quality_failed_node_set.add(nr.node_id)+ if isinstance(child_window, _KillSwitchChildWindow):+ child_window.close()+if skipped_jobs:await temporalio.workflow.execute_activity(record_skipped_data_modeling_jobs_activity,diff --git a/posthog/temporal/tests/data_modeling/test_execute_dag_workflow.py b/posthog/temporal/tests/data_modeling/test_execute_dag_workflow.pyindex b0036cd83893..ac81998d06f8 100644--- a/posthog/temporal/tests/data_modeling/test_execute_dag_workflow.py+++ b/posthog/temporal/tests/data_modeling/test_execute_dag_workflow.py@@ -12,6 +12,7 @@ from temporalio import (activity as temporal_activity,workflow as temporal_workflow,)+from temporalio.api.enums.v1 import EventTypefrom temporalio.exceptions import WorkflowAlreadyStartedErrorfrom temporalio.testing import WorkflowEnvironmentfrom temporalio.workflow import ParentClosePolicy@@ -575,6 +576,22 @@ class MockMaterializeViewWorkflow:)+_child_concurrency = {"running": 0, "peak": 0, "sleep_seconds": 10}+++@temporal_workflow.defn(name="data-modeling-materialize-view")+class SlowMockMaterializeViewWorkflow:+ @temporal_workflow.run+ async def run(self, inputs: MaterializeViewWorkflowInputs) -> MaterializeViewWorkflowResult:+ _child_concurrency["running"] += 1+ _child_concurrency["peak"] = max(_child_concurrency["peak"], _child_concurrency["running"])+ await temporal_workflow.sleep(dt.timedelta(seconds=_child_concurrency["sleep_seconds"]))+ _child_concurrency["running"] -= 1+ return MaterializeViewWorkflowResult(+ job_id="test-job", node_id=inputs.node_id, rows_materialized=1, duration_seconds=10.0+ )++class TestExecuteDAGWorkflowWithMocks:"""Tests that use module-level mock workflows to test orchestration behavior."""@@ -924,6 +941,97 @@ class TestExecuteDAGWorkflowWithMocks:assert all(r.success for r in result.node_results)assert _notified_dag_failures == []+ @pytest.mark.parametrize(+ "kill_switch_level,expected_peak",+ [+ ("off", 4),+ ("light", 3),+ ("full", 1),+ (None, 4),+ ],+ )+ async def test_kill_switch_narrows_the_child_window(self, kill_switch_level, expected_peak):+ node_ids = [str(uuid.uuid4()) for _ in range(4)]+ _child_concurrency.update(running=0, peak=0, sleep_seconds=10)++ @temporal_activity.defn(name="get_dag_structure_activity")+ async def stub_get_dag_structure(_: GetDAGStructureInputs) -> DAGPlan:+ return DAGPlan(nodes=node_ids, executable_nodes=node_ids, edges=[])++ @temporal_activity.defn(name="get_clickhouse_kill_switch_level_activity")+ async def stub_kill_switch_level(_: int) -> str:+ if kill_switch_level is None:+ raise temporalio.exceptions.ApplicationError("instance settings unavailable")+ return kill_switch_level++ async with await WorkflowEnvironment.start_time_skipping() as env:+ async with temporalio.worker.Worker(+ env.client,+ task_queue="test-queue",+ workflows=[ExecuteDAGWorkflow, SlowMockMaterializeViewWorkflow],+ activities=[+ stub_preempt_dag_run,+ stub_check_team_shadow_eligibility,+ stub_get_dag_structure,+ stub_kill_switch_level,+ stub_notify_dag_materialization_failures,+ ],+ workflow_runner=temporalio.worker.UnsandboxedWorkflowRunner(),+ ):+ result: ExecuteDAGResult = await env.client.execute_workflow(+ ExecuteDAGWorkflow.run,+ ExecuteDAGInputs(team_id=1, dag_id="test-dag"),+ id=f"test-kill-switch-window-{uuid.uuid4()}",+ task_queue="test-queue",+ execution_timeout=dt.timedelta(minutes=10),+ )++ assert result.successful_nodes == 4+ assert _child_concurrency["peak"] == expected_peak++ async def test_kill_switch_is_reread_while_children_keep_finishing(self):+ node_ids = [str(uuid.uuid4()) for _ in range(12)]+ _child_concurrency.update(running=0, peak=0, sleep_seconds=30)+ level_reads: list[str] = []+ workflow_id = f"test-kill-switch-reread-{uuid.uuid4()}"++ @temporal_activity.defn(name="get_dag_structure_activity")+ async def stub_get_dag_structure(_: GetDAGStructureInputs) -> DAGPlan:+ return DAGPlan(nodes=node_ids, executable_nodes=node_ids, edges=[])++ @temporal_activity.defn(name="get_clickhouse_kill_switch_level_activity")+ async def stub_kill_switch_level(_: int) -> str:+ level_reads.append("read")+ return "full" if len(level_reads) == 1 else "off"++ async with await WorkflowEnvironment.start_time_skipping() as env:+ async with temporalio.worker.Worker(+ env.client,+ task_queue="test-queue",+ workflows=[ExecuteDAGWorkflow, SlowMockMaterializeViewWorkflow],+ activities=[+ stub_preempt_dag_run,+ stub_check_team_shadow_eligibility,+ stub_get_dag_structure,+ stub_kill_switch_level,+ stub_notify_dag_materialization_failures,+ ],+ workflow_runner=temporalio.worker.UnsandboxedWorkflowRunner(),+ ):+ result: ExecuteDAGResult = await env.client.execute_workflow(+ ExecuteDAGWorkflow.run,+ ExecuteDAGInputs(team_id=1, dag_id="test-dag"),+ id=workflow_id,+ task_queue="test-queue",+ execution_timeout=dt.timedelta(minutes=30),+ )+ history = await env.client.get_workflow_handle(workflow_id).fetch_history()++ timers_started = sum(1 for event in history.events if event.event_type == EventType.EVENT_TYPE_TIMER_STARTED)+ assert result.successful_nodes == 12+ assert _child_concurrency["peak"] > 1+ assert timers_started <= 2+async def test_ephemeral_nodes_are_skipped(self):"""Test that ephemeral nodes are recorded as successful no-ops without starting child workflows."""dag_id = "test-dag"diff --git a/posthog/temporal/tests/data_modeling/test_materialize_view_activities.py b/posthog/temporal/tests/data_modeling/test_materialize_view_activities.pyindex cf0cd5fcd271..deb4e7738dd3 100644--- a/posthog/temporal/tests/data_modeling/test_materialize_view_activities.py+++ b/posthog/temporal/tests/data_modeling/test_materialize_view_activities.py@@ -23,6 +23,7 @@ from trino.exceptions import TrinoUserErrorfrom posthog.hogql.resolver import ResolverFactoryfrom posthog.hogql.transforms.trino.errors import TrinoLoweringError+from posthog.clickhouse.client.execute import KillSwitchLevelfrom posthog.clickhouse.query_tagging import get_query_tagsfrom posthog.models import Team, Userfrom posthog.sync import database_sync_to_async@@ -2256,6 +2257,40 @@ class TestHogqlTableModifiers:)assert captured_sql is not None and expected_sql in captured_sql+ @pytest.mark.parametrize(+ "kill_switch_level,expected_max_threads",+ [+ (KillSwitchLevel.OFF, None),+ (KillSwitchLevel.LIGHT, 16),+ (KillSwitchLevel.FULL, 8),+ ],+ )+ async def test_kill_switch_lowers_the_threads_per_query(+ self, ateam: Team, kill_switch_level: KillSwitchLevel, expected_max_threads: int | None+ ) -> None:+ client = _EmptyArrowClient(pa.schema([pa.field("id", pa.int64())]))++ @contextlib.asynccontextmanager+ async def fake_get_client(**kwargs: Any) -> AsyncIterator[_EmptyArrowClient]:+ yield client++ with (+ unittest.mock.patch(+ "posthog.temporal.data_modeling.activities.materialize_view.get_clickhouse_client", fake_get_client+ ),+ unittest.mock.patch(+ "posthog.temporal.data_modeling.activities.materialize_view.resolve_kill_switch_level",+ lambda _team_id: kill_switch_level,+ ),+ ):+ _ = [batch async for batch in hogql_table("SELECT 1", ateam, LOGGER.bind())]++ assert client.arrow_query is not None+ if expected_max_threads is None:+ assert "max_threads" not in client.arrow_query+ else:+ assert f"max_threads={expected_max_threads}" in client.arrow_query+async def test_compiles_account_dependencies_without_a_user(self, ateam: Team) -> None:schema_fields: list[pa.Field[Any]] = [pa.field("id", pa.string()),