Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-modeling): report the run mode of failed clickhouse runs (#121748)

Jovan Sakovic81bd2694903c
diff --git a/posthog/temporal/data_modeling/activities/fail_materialization.py b/posthog/temporal/data_modeling/activities/fail_materialization.py
index b64e79fe6309..959a42be60d2 100644
--- a/posthog/temporal/data_modeling/activities/fail_materialization.py
+++ b/posthog/temporal/data_modeling/activities/fail_materialization.py
@@ -102,7 +102,7 @@ def _revert_materialization_on_unknown_table(job: DataModelingJob, saved_query:
@activity.defn
-async def fail_materialization_activity(inputs: FailMaterializationInputs) -> None:
+async def fail_materialization_activity(inputs: FailMaterializationInputs) -> str | None:
"""Mark materialization as failed and update node properties."""
bind_contextvars(team_id=inputs.team_id)
logger = LOGGER.bind()
@@ -119,13 +119,13 @@ async def fail_materialization_activity(inputs: FailMaterializationInputs) -> No
await logger.aerror(f"Failed materialization job: {job_context} error={inputs.error}", write_only=True)
# error-specific recovery: revert on unknown table, else suspend the node after repeated failures
if not inputs.update_node:
- return
+ return None
error = inputs.error
saved_query = None
try:
saved_query = await _get_saved_query_for_job(job)
if saved_query is None:
- return
+ return None
if "Unknown table" in error:
await logger.ainfo(
@@ -166,3 +166,4 @@ async def fail_materialization_activity(inputs: FailMaterializationInputs) -> No
await logger.aexception(
f"Failed to notify materialization failure for node {inputs.node_id}: {strip_hostname_from_error(str(e))}"
)
+ return None
diff --git a/posthog/temporal/data_modeling/workflows/materialize_view.py b/posthog/temporal/data_modeling/workflows/materialize_view.py
index a9b1e5868b2d..ddcac5c72f88 100644
--- a/posthog/temporal/data_modeling/workflows/materialize_view.py
+++ b/posthog/temporal/data_modeling/workflows/materialize_view.py
@@ -506,8 +506,9 @@ class MaterializeViewWorkflow(PostHogWorkflow):
# rows behind. The activity cleans up after its own failures itself.
if materialize_result is not None:
await self._discard_staged_cdp_rows(inputs, job_id, materialize_result)
+ failed_run_mode = None
try:
- await temporalio.workflow.execute_activity(
+ failed_run_mode = await temporalio.workflow.execute_activity(
fail_materialization_activity,
FailMaterializationInputs(
team_id=inputs.team_id,
@@ -531,7 +532,8 @@ class MaterializeViewWorkflow(PostHogWorkflow):
"cancelled" if cancelled else "failed",
CLICKHOUSE_ENGINE,
tier=tier,
- run_mode=_run_mode(materialize_result),
+ # A history recorded before the activity returned the mode decodes it as None.
+ run_mode=failed_run_mode or _run_mode(materialize_result),
error_class="none" if cancelled else error_class(error_message),
).add(1)
raise
diff --git a/posthog/temporal/tests/data_modeling/test_materialize_view_workflow.py b/posthog/temporal/tests/data_modeling/test_materialize_view_workflow.py
index ff195d80715a..c679ac4e6827 100644
--- a/posthog/temporal/tests/data_modeling/test_materialize_view_workflow.py
+++ b/posthog/temporal/tests/data_modeling/test_materialize_view_workflow.py
@@ -200,6 +200,22 @@ class TestQualityGateBranching:
assert started.count("create_data_modeling_job_activity") == 1
assert "check_managed_warehouse_shadow_eligibility_activity" not in started
+ @pytest.mark.parametrize("recorded_run_mode,expected", [("incremental", "incremental"), (None, "unknown")])
+ async def test_clickhouse_failure_reports_the_jobs_run_mode(self, recorded_run_mode, expected) -> None:
+ async def execute_activity(activity, *args, **kwargs):
+ if activity.__name__ == "materialize_view_activity":
+ raise RuntimeError("Code: 241. Memory limit exceeded")
+ if activity.__name__ == "fail_materialization_activity":
+ return recorded_run_mode
+ return {"create_data_modeling_job_activity": "job-1"}.get(activity.__name__)
+
+ with pytest.raises(RuntimeError):
+ await self._run([], {}, execute_activity=AsyncMock(side_effect=execute_activity))
+
+ finished = self.metrics["get_node_finished_metric"]
+ assert finished.call_args.args == ("failed", "clickhouse")
+ assert finished.call_args.kwargs["run_mode"] == expected
+
async def test_blocking_failures_stop_the_publish(self):
activity_results = [
"job-1", # create job