:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(data-modeling): report the run mode of failed clickhouse runs (#121748)
Jovan Sakovic81bd2694903c
diff --git a/posthog/temporal/data_modeling/activities/fail_materialization.py b/posthog/temporal/data_modeling/activities/fail_materialization.pyindex b64e79fe6309..959a42be60d2 100644--- a/posthog/temporal/data_modeling/activities/fail_materialization.py+++ b/posthog/temporal/data_modeling/activities/fail_materialization.py@@ -102,7 +102,7 @@ def _revert_materialization_on_unknown_table(job: DataModelingJob, saved_query:@activity.defn-async def fail_materialization_activity(inputs: FailMaterializationInputs) -> None:+async def fail_materialization_activity(inputs: FailMaterializationInputs) -> str | None:"""Mark materialization as failed and update node properties."""bind_contextvars(team_id=inputs.team_id)logger = LOGGER.bind()@@ -119,13 +119,13 @@ async def fail_materialization_activity(inputs: FailMaterializationInputs) -> Noawait logger.aerror(f"Failed materialization job: {job_context} error={inputs.error}", write_only=True)# error-specific recovery: revert on unknown table, else suspend the node after repeated failuresif not inputs.update_node:- return+ return Noneerror = inputs.errorsaved_query = Nonetry:saved_query = await _get_saved_query_for_job(job)if saved_query is None:- return+ return Noneif "Unknown table" in error:await logger.ainfo(@@ -166,3 +166,4 @@ async def fail_materialization_activity(inputs: FailMaterializationInputs) -> Noawait logger.aexception(f"Failed to notify materialization failure for node {inputs.node_id}: {strip_hostname_from_error(str(e))}")+ return Nonediff --git a/posthog/temporal/data_modeling/workflows/materialize_view.py b/posthog/temporal/data_modeling/workflows/materialize_view.pyindex a9b1e5868b2d..ddcac5c72f88 100644--- a/posthog/temporal/data_modeling/workflows/materialize_view.py+++ b/posthog/temporal/data_modeling/workflows/materialize_view.py@@ -506,8 +506,9 @@ class MaterializeViewWorkflow(PostHogWorkflow):# rows behind. The activity cleans up after its own failures itself.if materialize_result is not None:await self._discard_staged_cdp_rows(inputs, job_id, materialize_result)+ failed_run_mode = Nonetry:- await temporalio.workflow.execute_activity(+ failed_run_mode = await temporalio.workflow.execute_activity(fail_materialization_activity,FailMaterializationInputs(team_id=inputs.team_id,@@ -531,7 +532,8 @@ class MaterializeViewWorkflow(PostHogWorkflow):"cancelled" if cancelled else "failed",CLICKHOUSE_ENGINE,tier=tier,- run_mode=_run_mode(materialize_result),+ # A history recorded before the activity returned the mode decodes it as None.+ run_mode=failed_run_mode or _run_mode(materialize_result),error_class="none" if cancelled else error_class(error_message),).add(1)raisediff --git a/posthog/temporal/tests/data_modeling/test_materialize_view_workflow.py b/posthog/temporal/tests/data_modeling/test_materialize_view_workflow.pyindex ff195d80715a..c679ac4e6827 100644--- a/posthog/temporal/tests/data_modeling/test_materialize_view_workflow.py+++ b/posthog/temporal/tests/data_modeling/test_materialize_view_workflow.py@@ -200,6 +200,22 @@ class TestQualityGateBranching:assert started.count("create_data_modeling_job_activity") == 1assert "check_managed_warehouse_shadow_eligibility_activity" not in started+ @pytest.mark.parametrize("recorded_run_mode,expected", [("incremental", "incremental"), (None, "unknown")])+ async def test_clickhouse_failure_reports_the_jobs_run_mode(self, recorded_run_mode, expected) -> None:+ async def execute_activity(activity, *args, **kwargs):+ if activity.__name__ == "materialize_view_activity":+ raise RuntimeError("Code: 241. Memory limit exceeded")+ if activity.__name__ == "fail_materialization_activity":+ return recorded_run_mode+ return {"create_data_modeling_job_activity": "job-1"}.get(activity.__name__)++ with pytest.raises(RuntimeError):+ await self._run([], {}, execute_activity=AsyncMock(side_effect=execute_activity))++ finished = self.metrics["get_node_finished_metric"]+ assert finished.call_args.args == ("failed", "clickhouse")+ assert finished.call_args.kwargs["run_mode"] == expected+async def test_blocking_failures_stop_the_publish(self):activity_results = ["job-1", # create job