Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-modeling): build each trino model in one run at a time (#126460)

Jovan Sakovicdbe4712c403b
diff --git a/docs/internal/hogql-trino-compiler.md b/docs/internal/hogql-trino-compiler.md
index ef1c49b276b1..94f470b1dacf 100644
--- a/docs/internal/hogql-trino-compiler.md
+++ b/docs/internal/hogql-trino-compiler.md
@@ -272,6 +272,8 @@ The shadow run has a fixed workflow id per DAG and cadence tier: `execute-dag-tr
If the previous shadow run with that id is still running, the new one is skipped, so a slow Trino never builds a backlog.
Shadow runs record their outcomes in managed warehouse job rows and metrics only: they run no data quality checks and send no failure notifications.
Preemption is scoped to the run's engine, so the ClickHouse and Trino runs of a tier never cancel each other's jobs.
+A Trino run never cancels a running build, because cancelling a Trino build does not stop its SQL. It only reaps job rows whose workflow has ended.
+Each Trino build of a node has a fixed workflow id, so Temporal starts at most one build of that node at a time. A run that reaches a node another run is building skips it and its Trino dependents.
A single-model materialization started outside a DAG run does not start a Trino shadow.
Within the shadow run, the DAG waits for upstream Trino builds and skips dependent Trino builds when an upstream model fails or is ineligible for shadowing.
Skipped managed warehouse jobs record the upstream node IDs. Existing Temporal histories retain their previous behavior, including the shadow inside the ClickHouse materialization.
diff --git a/posthog/temporal/data_modeling/activities/create_data_modeling_job.py b/posthog/temporal/data_modeling/activities/create_data_modeling_job.py
index 3c5be23190fb..6527f2bc51e3 100644
--- a/posthog/temporal/data_modeling/activities/create_data_modeling_job.py
+++ b/posthog/temporal/data_modeling/activities/create_data_modeling_job.py
@@ -1,3 +1,4 @@
+import datetime as dt
import dataclasses
from django.db import transaction
@@ -43,6 +44,7 @@ class SkippedDataModelingNode:
failed_upstream_total: int = 0
suspended_upstream_node_ids: list[str] = dataclasses.field(default_factory=list)
suspended_upstream_total: int = 0
+ built_by_another_run: bool = False
@dataclasses.dataclass(frozen=True, kw_only=True)
@@ -51,6 +53,8 @@ class RecordSkippedDataModelingJobsInputs:
dag_id: str
engine: str
skipped_nodes: list[SkippedDataModelingNode]
+ # Trino shadows reuse one parent workflow id across runs, so jobs from earlier runs must not count as already recorded.
+ run_started_at: str | None = None
@database_sync_to_async_pool
@@ -114,7 +118,11 @@ def _subject(names: list[str], total: int) -> str:
return f"{noun} {shown[0]}"
-def _skip_reason(*, failed: list[str], failed_total: int, suspended: list[str], suspended_total: int) -> str:
+def _skip_reason(
+ *, failed: list[str], failed_total: int, suspended: list[str], suspended_total: int, built_by_another_run: bool
+) -> str:
+ if built_by_another_run:
+ return "Skipped because another run is already building this view."
if failed_total and suspended_total:
subject = _subject(failed + suspended, failed_total + suspended_total)
return f"Skipped because {subject} are failing or suspended."
@@ -144,14 +152,15 @@ def _record_skipped_data_modeling_jobs(
}
skipped_nodes = [nodes[skipped.node_id] for skipped in inputs.skipped_nodes if skipped.node_id in nodes]
saved_query_ids = {node.saved_query_id for node in skipped_nodes if node.saved_query_id is not None}
- existing_saved_query_ids = set(
- DataModelingJob.objects.filter(
- team_id=inputs.team_id,
- engine=inputs.engine,
- parent_workflow_id=workflow_id,
- saved_query_id__in=saved_query_ids,
- ).values_list("saved_query_id", flat=True)
+ existing_jobs = DataModelingJob.objects.filter(
+ team_id=inputs.team_id,
+ engine=inputs.engine,
+ parent_workflow_id=workflow_id,
+ saved_query_id__in=saved_query_ids,
)
+ if inputs.run_started_at:
+ existing_jobs = existing_jobs.filter(created_at__gte=dt.datetime.fromisoformat(inputs.run_started_at))
+ existing_saved_query_ids = set(existing_jobs.values_list("saved_query_id", flat=True))
jobs = []
for skipped in inputs.skipped_nodes:
@@ -170,6 +179,7 @@ def _record_skipped_data_modeling_jobs(
failed_total=skipped.failed_upstream_total,
suspended=[nodes[i].name for i in skipped.suspended_upstream_node_ids if i in nodes],
suspended_total=skipped.suspended_upstream_total,
+ built_by_another_run=skipped.built_by_another_run,
),
workflow_id=workflow_id,
workflow_run_id=workflow_run_id,
diff --git a/posthog/temporal/data_modeling/activities/preempt_dag_run.py b/posthog/temporal/data_modeling/activities/preempt_dag_run.py
index b0db1b40285a..0031a11cde34 100644
--- a/posthog/temporal/data_modeling/activities/preempt_dag_run.py
+++ b/posthog/temporal/data_modeling/activities/preempt_dag_run.py
@@ -34,7 +34,7 @@ def _node_id_from_workflow_id(workflow_id: str, dag_id: str) -> str | None:
prefix = f"materialize-view-{dag_id}-"
if not workflow_id.startswith(prefix):
return None
- # The remainder is `{node_id}-{timestamp}`. Node ids are UUIDs, so take the first five
+ # The remainder is `{node_id}-{suffix}`. Node ids are UUIDs, so take the first five
# dash-separated groups rather than assuming a fixed overall length.
groups = workflow_id[len(prefix) :].split("-")
return "-".join(groups[:5]) if len(groups) >= 5 else None
@@ -44,7 +44,7 @@ def _node_id_from_workflow_id(workflow_id: str, dag_id: str) -> str | None:
def _get_running_jobs_for_dag(team_id: int, dag_id: str, engine: str) -> list[DataModelingJob]:
"""Every RUNNING job of one engine in this DAG, whether or not this run owns its node.
- Child workflow IDs follow the pattern `materialize-view-{dag_id}-{node_id}-{timestamp}`,
+ Child workflow IDs follow the pattern `materialize-view-{dag_id}-{node_id}-{suffix}`,
so we can match jobs by their workflow_id prefix. The ClickHouse and Trino runs of a DAG
materialize the same nodes at the same time, so each run only touches its own engine's jobs.
"""
@@ -100,7 +100,8 @@ async def _abandoned_jobs(temporal: Client, candidates: list[DataModelingJob]) -
if not job.workflow_id:
continue
try:
- description = await temporal.get_workflow_handle(job.workflow_id).describe()
+ # Trino builds reuse one workflow id, so describe this row's run rather than the latest one.
+ description = await temporal.get_workflow_handle(job.workflow_id, run_id=job.workflow_run_id).describe()
except RPCError as e:
# NOT_FOUND is the only error that proves absence. Timeouts, unavailability and
# permission errors say nothing about the workflow, and marking a live job Failed is
@@ -137,8 +138,10 @@ async def preempt_dag_run_activity(inputs: PreemptDAGRunInputs) -> None:
cadence tiers would take down the whole run that still owns its siblings.
A parent starts each of its nodes once, so there is no child to re-spawn behind us.
- Rows outside that node set are only reaped, and only once Temporal confirms their workflow
- is no longer running: they are marked, never cancelled, because there is nothing left to
+ A Trino run preempts nothing, because cancelling a Trino build does not stop its write.
+
+ Rows outside that node set (every row, on Trino) are only reaped, and only once Temporal
+ confirms their workflow is no longer running: they are marked, never cancelled, because there is nothing left to
cancel and cancelling on another tier's behalf is exactly what this activity must not do.
"""
bind_contextvars(team_id=inputs.team_id)
@@ -146,6 +149,9 @@ async def preempt_dag_run_activity(inputs: PreemptDAGRunInputs) -> None:
running_jobs = await _get_running_jobs_for_dag(inputs.team_id, inputs.dag_id, inputs.engine)
partitioned = partition_running_jobs(running_jobs, inputs.dag_id, inputs.node_ids)
+ if inputs.engine == DataModelingJobEngine.MANAGED_WAREHOUSE:
+ # The Trino client cannot kill a running statement. The fixed Trino child id stops a second build instead.
+ partitioned = PartitionedJobs(owned=[], others=partitioned.owned + partitioned.others)
if not partitioned.owned and not partitioned.others:
await logger.adebug("No previous DAG run to preempt")
return
diff --git a/posthog/temporal/data_modeling/workflows/execute_dag.py b/posthog/temporal/data_modeling/workflows/execute_dag.py
index 4b150bc10bb8..9b95f3d9288b 100644
--- a/posthog/temporal/data_modeling/workflows/execute_dag.py
+++ b/posthog/temporal/data_modeling/workflows/execute_dag.py
@@ -50,6 +50,9 @@ MAX_CONCURRENT_CHILDREN = 10
TRINO_SHADOW_MAX_CONCURRENT_CHILDREN = 25
TRINO_SHADOW_CONCURRENCY_PATCH = "data-modeling-trino-shadow-concurrency"
+TRINO_CHILD_LOCK_PATCH = "data-modeling-trino-child-lock"
+BUILT_BY_ANOTHER_RUN_SKIP_REASON = "Being built by another run"
+
KILL_SWITCH_WINDOW_PATCH = "data-modeling-kill-switch-window"
# Under a ClickHouse kill switch a DAG run starts fewer materializations at once, so the CPU that the
# switch frees goes to interactive queries. A node that waits for a slot waits inside this workflow and
@@ -77,6 +80,18 @@ def trino_shadow_workflow_id(workflow_id: str, dag_id: str) -> str:
return f"execute-dag-trino-{dag_id}"
+def trino_build_workflow_id(dag_id: str, node_id: str) -> str:
+ """The fixed id of the Trino build of one node.
+
+ Temporal runs one workflow per id at a time, so a second run that reaches this node while the
+ first still builds it cannot start a build of its own. Cancelling a Trino build does not stop
+ its SQL, so this refusal is the only thing that keeps two runs from writing one table at once.
+ The `materialize-view-{dag_id}-{node_id}` prefix is shared with ClickHouse builds, which is
+ what preemption, job lookups and the abandoned-job reaper match on.
+ """
+ return f"materialize-view-{dag_id}-{node_id}-trino"
+
+
def manual_trino_workflow_id(dag_id: str) -> str:
"""The fixed id of a Trino run that a person starts by hand.
@@ -432,6 +447,8 @@ class ExecuteDAGWorkflow(PostHogWorkflow):
max_concurrent_children = MAX_CONCURRENT_CHILDREN
if managed_warehouse_only and temporalio.workflow.patched(TRINO_SHADOW_CONCURRENCY_PATCH):
max_concurrent_children = TRINO_SHADOW_MAX_CONCURRENT_CHILDREN
+ # Histories from before the patch must replay their timestamped child ids.
+ trino_build_lock = managed_warehouse_only and temporalio.workflow.patched(TRINO_CHILD_LOCK_PATCH)
# Histories from before the patch keep the semaphore, because its wake-up order decides the
# order of child-start commands, and replay must reproduce that order.
child_window: asyncio.Semaphore | _KillSwitchChildWindow
@@ -547,15 +564,36 @@ class ExecuteDAGWorkflow(PostHogWorkflow):
skip_reason="Upstream Trino materialization unavailable",
trino_materialized=False,
)
- handle = await temporalio.workflow.start_child_workflow(
- MaterializeViewWorkflow.run,
- child_inputs,
- id=f"materialize-view-{inputs.dag_id}-{node_id}-{start_time.isoformat()}",
- parent_close_policy=ParentClosePolicy.REQUEST_CANCEL,
- retry_policy=temporalio.common.RetryPolicy(
- maximum_attempts=1, # retries handled within child workflow
- ),
+ child_id = (
+ trino_build_workflow_id(inputs.dag_id, node_id)
+ if trino_build_lock
+ else f"materialize-view-{inputs.dag_id}-{node_id}-{start_time.isoformat()}"
)
+ try:
+ handle = await temporalio.workflow.start_child_workflow(
+ MaterializeViewWorkflow.run,
+ child_inputs,
+ id=child_id,
+ parent_close_policy=ParentClosePolicy.REQUEST_CANCEL,
+ retry_policy=temporalio.common.RetryPolicy(
+ maximum_attempts=1, # retries handled within child workflow
+ ),
+ )
+ except WorkflowAlreadyStartedError:
+ if not trino_build_lock:
+ raise
+ temporalio.workflow.logger.info(
+ f"Node {node_id} is being built on Trino by another run, skipping",
+ extra=inputs.properties_to_log,
+ )
+ skipped_trino_jobs.append(SkippedDataModelingNode(node_id=node_id, built_by_another_run=True))
+ return NodeResult(
+ node_id=node_id,
+ success=False,
+ skipped=True,
+ skip_reason=BUILT_BY_ANOTHER_RUN_SKIP_REASON,
+ trino_materialized=False,
+ )
try:
result: MaterializeViewWorkflowResult = await handle
if trino_skip is not None and result.trino_materialized is False:
@@ -616,7 +654,8 @@ class ExecuteDAGWorkflow(PostHogWorkflow):
unavailable_trino_nodes.discard(nr.node_id)
if nr.trino_materialized is False or (not managed_warehouse_only and nr.trino_materialized is None):
unavailable_trino_nodes.add(nr.node_id)
- if not nr.success:
+ # A node another run builds did not fail. Its dependents skip through unavailable_trino_nodes instead.
+ if not nr.success and nr.skip_reason != BUILT_BY_ANOTHER_RUN_SKIP_REASON:
failed_node_set.add(nr.node_id)
if nr.quality_failed:
quality_failed_node_set.add(nr.node_id)
@@ -645,6 +684,7 @@ class ExecuteDAGWorkflow(PostHogWorkflow):
dag_id=inputs.dag_id,
engine=DataModelingJobEngine.MANAGED_WAREHOUSE.value,
skipped_nodes=skipped_trino_jobs,
+ run_started_at=start_time.isoformat(),
),
start_to_close_timeout=dt.timedelta(minutes=5),
retry_policy=temporalio.common.RetryPolicy(maximum_attempts=3),
diff --git a/posthog/temporal/tests/data_modeling/test_execute_dag_workflow.py b/posthog/temporal/tests/data_modeling/test_execute_dag_workflow.py
index ac81998d06f8..1d0d28013a6d 100644
--- a/posthog/temporal/tests/data_modeling/test_execute_dag_workflow.py
+++ b/posthog/temporal/tests/data_modeling/test_execute_dag_workflow.py
@@ -1,11 +1,14 @@
import uuid
import asyncio
+import logging
import datetime as dt
+from collections.abc import Callable, Iterator
import pytest
from unittest.mock import AsyncMock, MagicMock, patch
import pytest_asyncio
+import temporalio.client
import temporalio.worker
import temporalio.exceptions
from temporalio import (
@@ -30,6 +33,7 @@ from posthog.temporal.data_modeling.activities import (
)
from posthog.temporal.data_modeling.activities.get_dag_structure import DAG as DAGPlan
from posthog.temporal.data_modeling.workflows.execute_dag import (
+ TRINO_CHILD_LOCK_PATCH,
EmptyDAGOrCycleError,
ExecuteDAGInputs,
ExecuteDAGResult,
@@ -1338,3 +1342,207 @@ class TestTrinoShadowRun:
if call.args[0] == preempt_dag_run_activity
)
assert preempt_inputs.engine == engine.value
+
+
+@frozen(slots=False)
+class HeldChild:
+ node_id: str
+ parent_workflow_id: str
+
+
+_held_children: dict[str, HeldChild] = {}
+_child_starts: list[HeldChild] = []
+_peak_builds_per_node: dict[str, int] = {}
+
+
+@temporal_workflow.defn(name="data-modeling-materialize-view")
+class HeldMockMaterializeViewWorkflow:
+ def __init__(self) -> None:
+ self._released = False
+
+ @temporal_workflow.signal
+ def release(self) -> None:
+ self._released = True
+
+ @temporal_workflow.run
+ async def run(self, inputs: MaterializeViewWorkflowInputs) -> MaterializeViewWorkflowResult:
+ info = temporal_workflow.info()
+ assert info.parent is not None
+ child = HeldChild(node_id=inputs.node_id, parent_workflow_id=info.parent.workflow_id)
+ _held_children[info.workflow_id] = child
+ _child_starts.append(child)
+ building = sum(1 for held in _held_children.values() if held.node_id == inputs.node_id)
+ _peak_builds_per_node[inputs.node_id] = max(_peak_builds_per_node.get(inputs.node_id, 0), building)
+ try:
+ await temporal_workflow.wait_condition(lambda: self._released)
+ finally:
+ _held_children.pop(info.workflow_id, None)
+ return MaterializeViewWorkflowResult(
+ job_id=f"job-{inputs.node_id}",
+ node_id=inputs.node_id,
+ rows_materialized=1,
+ duration_seconds=1,
+ trino_materialized=True,
+ )
+
+
+async def _wait_until(predicate: Callable[[], bool], timeout_seconds: float = 20) -> None:
+ deadline = asyncio.get_running_loop().time() + timeout_seconds
+ while not predicate():
+ assert asyncio.get_running_loop().time() < deadline, "condition not reached"
+ await asyncio.sleep(0.05)
+
+
+class TestTrinoModelLock:
+ @pytest.fixture(autouse=True)
+ def reset_state(self, caplog: pytest.LogCaptureFixture) -> Iterator[None]:
+ caplog.set_level(logging.INFO, logger="temporalio.workflow")
+ caplog.set_level(logging.INFO, logger="temporalio.activity")
+ for state in (_held_children, _child_starts, _peak_builds_per_node, _recorded_skipped_nodes):
+ state.clear()
+ yield
+ for state in (_held_children, _child_starts, _peak_builds_per_node, _recorded_skipped_nodes):
+ state.clear()
+
+ @pytest.mark.parametrize(
+ "winner_nodes,loser_nodes,edges,built_side_by_side,expected_loser_skips",
+ [
+ pytest.param(
+ ["a", "b"],
+ ["b", "c"],
+ [],
+ {"a", "b", "c"},
+ {"b": "Being built by another run"},
+ id="overlapping_models",
+ ),
+ pytest.param(
+ ["a"],
+ ["a", "b"],
+ [("a", "b")],
+ {"a"},
+ {"a": "Being built by another run", "b": "Upstream Trino materialization unavailable"},
+ id="downstream_of_a_model_another_run_builds",
+ ),
+ ],
+ )
+ async def test_two_trino_runs_never_build_the_same_model_at_once(
+ self,
+ winner_nodes: list[str],
+ loser_nodes: list[str],
+ edges: list[tuple[str, str]],
+ built_side_by_side: set[str],
+ expected_loser_skips: dict[str, str],
+ ) -> None:
+ dag_id = str(uuid.uuid4())
+ winner_id = f"execute-dag-trino-{dag_id}:3600"
+ loser_id = f"execute-dag-trino-{dag_id}-manual"
+
+ @temporal_activity.defn(name="get_dag_structure_activity")
+ async def stub_get_dag_structure(_: GetDAGStructureInputs) -> DAGPlan:
+ return DAGPlan(nodes=["a", "b", "c"], executable_nodes=["a", "b", "c"], edges=edges)
+
+ def held_by(parent_workflow_id: str) -> list[str]:
+ return [
+ child_id for child_id, held in _held_children.items() if held.parent_workflow_id == parent_workflow_id
+ ]
+
+ async with await WorkflowEnvironment.start_time_skipping() as env:
+ async with temporalio.worker.Worker(
+ env.client,
+ task_queue="test-queue",
+ workflows=[ExecuteDAGWorkflow, HeldMockMaterializeViewWorkflow],
+ activities=[
+ stub_preempt_dag_run,
+ stub_check_team_shadow_eligibility,
+ stub_get_dag_structure,
+ stub_record_skipped_data_modeling_jobs,
+ stub_notify_dag_materialization_failures,
+ ],
+ workflow_runner=temporalio.worker.UnsandboxedWorkflowRunner(),
+ ):
+
+ async def start(workflow_id: str, node_ids: list[str]) -> temporalio.client.WorkflowHandle:
+ return await env.client.start_workflow(
+ ExecuteDAGWorkflow.run,
+ ExecuteDAGInputs(
+ team_id=1, dag_id=dag_id, node_ids=node_ids, managed_warehouse_only=True, shadow=True
+ ),
+ id=workflow_id,
+ task_queue="test-queue",
+ execution_timeout=dt.timedelta(hours=1),
+ )
+
+ async def release(child_ids: list[str]) -> None:
+ for child_id in child_ids:
+ await env.client.get_workflow_handle(child_id).signal(HeldMockMaterializeViewWorkflow.release)
+
+ # With time skipping on, the server jumps to the winner's timeout while children wait for a signal.
+ with env.auto_time_skipping_disabled():
+ winner = await start(winner_id, winner_nodes)
+ await _wait_until(lambda: len(held_by(winner_id)) == len(winner_nodes))
+ loser = await start(loser_id, loser_nodes)
+ await _wait_until(lambda: {held.node_id for held in _held_children.values()} == built_side_by_side)
+
+ await release(held_by(loser_id))
+ loser_result: ExecuteDAGResult = await asyncio.wait_for(loser.result(), timeout=20)
+ await release(held_by(winner_id))
+ winner_result: ExecuteDAGResult = await asyncio.wait_for(winner.result(), timeout=20)
+
+ assert max(_peak_builds_per_node.values()) == 1
+ assert winner_result.successful_nodes == len(winner_nodes)
+ assert loser_result.failed_nodes == 0
+ assert {r.node_id: r.skip_reason for r in loser_result.node_results if r.skipped} == expected_loser_skips
+ assert [child.node_id for child in _child_starts if child.parent_workflow_id == loser_id] == sorted(
+ set(loser_nodes) - set(expected_loser_skips)
+ )
+ [recorded] = _recorded_skipped_nodes
+ assert recorded.engine == DataModelingJobEngine.MANAGED_WAREHOUSE.value
+ assert sorted(skipped.node_id for skipped in recorded.skipped_nodes) == sorted(expected_loser_skips)
+
+ async def test_a_history_from_before_the_lock_replays(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ original_patched = temporal_workflow.patched
+
+ def legacy_patched(patch_id: str) -> bool:
+ return False if patch_id == TRINO_CHILD_LOCK_PATCH else original_patched(patch_id)
+
+ @temporal_activity.defn(name="get_dag_structure_activity")
+ async def stub_get_dag_structure(_: GetDAGStructureInputs) -> DAGPlan:
+ return DAGPlan(nodes=["a", "b"], executable_nodes=["a", "b"], edges=[("a", "b")])
+
+ workflow_id = f"execute-dag-trino-{uuid.uuid4()}"
+ async with await WorkflowEnvironment.start_time_skipping() as env:
+ async with temporalio.worker.Worker(
+ env.client,
+ task_queue="test-queue",
+ workflows=[ExecuteDAGWorkflow, MockMaterializeViewWorkflow],
+ activities=[
+ stub_preempt_dag_run,
+ stub_check_team_shadow_eligibility,
+ stub_get_dag_structure,
+ stub_record_skipped_data_modeling_jobs,
+ stub_notify_dag_materialization_failures,
+ ],
+ workflow_runner=temporalio.worker.UnsandboxedWorkflowRunner(),
+ ):
+ monkeypatch.setattr(temporal_workflow, "patched", legacy_patched)
+ await env.client.execute_workflow(
+ ExecuteDAGWorkflow.run,
+ ExecuteDAGInputs(team_id=1, dag_id="dag", managed_warehouse_only=True, shadow=True),
+ id=workflow_id,
+ task_queue="test-queue",
+ execution_timeout=dt.timedelta(minutes=5),
+ )
+ history = await env.client.get_workflow_handle(workflow_id).fetch_history()
+ monkeypatch.setattr(temporal_workflow, "patched", original_patched)
+
+ child_ids = [
+ event.start_child_workflow_execution_initiated_event_attributes.workflow_id
+ for event in history.events
+ if event.event_type == EventType.EVENT_TYPE_START_CHILD_WORKFLOW_EXECUTION_INITIATED
+ ]
+ assert len(child_ids) == 2
+ assert not any(child_id.endswith("-trino") for child_id in child_ids)
+ await temporalio.worker.Replayer(
+ workflows=[ExecuteDAGWorkflow],
+ workflow_runner=temporalio.worker.UnsandboxedWorkflowRunner(),
+ ).replay_workflow(history)
diff --git a/posthog/temporal/tests/data_modeling/test_preempt_dag_run.py b/posthog/temporal/tests/data_modeling/test_preempt_dag_run.py
index 0b877f9aa6f9..f0cc80c488b0 100644
--- a/posthog/temporal/tests/data_modeling/test_preempt_dag_run.py
+++ b/posthog/temporal/tests/data_modeling/test_preempt_dag_run.py
@@ -73,20 +73,23 @@ async def _run_activity(
) -> list[str]:
"""Run the activity against a stubbed Temporal client, returning the workflows it cancelled.
- `workflow_status` maps a workflow id to what describe() does: a status to report, or an
- exception to raise. Anything unlisted is reported as still RUNNING.
+ `workflow_status` maps a workflow id, or `{workflow_id}:{run_id}` for one run of it, to what
+ describe() does: a status to report, or an exception to raise. Anything unlisted is reported
+ as still RUNNING.
"""
cancelled: list[str] = []
statuses: dict[str, WorkflowExecutionStatus | Exception] = workflow_status or {}
- def get_workflow_handle(workflow_id: str) -> unittest.mock.MagicMock:
+ def get_workflow_handle(workflow_id: str, run_id: str | None = None) -> unittest.mock.MagicMock:
handle = unittest.mock.MagicMock()
async def cancel() -> None:
cancelled.append(workflow_id)
async def describe() -> unittest.mock.MagicMock:
- outcome = statuses.get(workflow_id, WorkflowExecutionStatus.RUNNING)
+ outcome = statuses.get(
+ f"{workflow_id}:{run_id}", statuses.get(workflow_id, WorkflowExecutionStatus.RUNNING)
+ )
if isinstance(outcome, Exception):
raise outcome
description = unittest.mock.MagicMock()
@@ -123,7 +126,7 @@ async def _run_activity(
(DataModelingJobEngine.CLICKHOUSE, DataModelingJobEngine.MANAGED_WAREHOUSE),
],
)
-async def test_preempt_only_cancels_the_runs_own_engine(
+async def test_preempt_only_touches_the_runs_own_engine(
preempting_engine: DataModelingJobEngine, spared_engine: DataModelingJobEngine, ateam
) -> None:
jobs = {
@@ -137,15 +140,78 @@ async def test_preempt_only_cancels_the_runs_own_engine(
for engine in (DataModelingJobEngine.CLICKHOUSE, DataModelingJobEngine.MANAGED_WAREHOUSE)
}
- cancelled = await _run_activity(ateam.pk, [NODE_HOURLY], engine=preempting_engine)
+ await _run_activity(
+ ateam.pk,
+ [NODE_HOURLY],
+ workflow_status={job.workflow_id or "": WorkflowExecutionStatus.COMPLETED for job in jobs.values()},
+ engine=preempting_engine,
+ )
for job in jobs.values():
await database_sync_to_async(job.refresh_from_db)()
- assert cancelled == [jobs[preempting_engine].workflow_id]
assert jobs[preempting_engine].status == DataModelingJobStatus.FAILED
assert jobs[spared_engine].status == DataModelingJobStatus.RUNNING
+ "engine,workflow_status,expect_cancelled,expected_status,expected_error",
+ [
+ pytest.param(
+ DataModelingJobEngine.CLICKHOUSE,
+ WorkflowExecutionStatus.RUNNING,
+ True,
+ DataModelingJobStatus.FAILED,
+ PREEMPTED_ERROR,
+ id="clickhouse_build_is_preempted",
+ ),
+ # Cancelling a Trino build does not stop its SQL.
+ pytest.param(
+ DataModelingJobEngine.MANAGED_WAREHOUSE,
+ WorkflowExecutionStatus.RUNNING,
+ False,
+ DataModelingJobStatus.RUNNING,
+ None,
+ id="running_trino_build_is_left_alone",
+ ),
+ pytest.param(
+ DataModelingJobEngine.MANAGED_WAREHOUSE,
+ WorkflowExecutionStatus.TERMINATED,
+ False,
+ DataModelingJobStatus.FAILED,
+ ABANDONED_ERROR,
+ # A later build runs under the same fixed id, so only the row's own run shows it ended.
+ id="abandoned_trino_build_is_reaped",
+ ),
+ ],
+)
+async def test_an_owned_build_is_preempted_only_on_clickhouse(
+ engine: DataModelingJobEngine,
+ workflow_status: WorkflowExecutionStatus,
+ expect_cancelled: bool,
+ expected_status: DataModelingJobStatus,
+ expected_error: str | None,
+ ateam,
+) -> None:
+ workflow_id = f"materialize-view-{DAG_ID}-{NODE_HOURLY}-trino"
+ job = await database_sync_to_async(DataModelingJob.objects.create)(
+ team=ateam,
+ status=DataModelingJobStatus.RUNNING,
+ engine=engine,
+ workflow_id=workflow_id,
+ workflow_run_id="run-1",
+ parent_workflow_id=PARENT_HOURLY,
+ )
+
+ cancelled = await _run_activity(
+ ateam.pk, [NODE_HOURLY], workflow_status={f"{workflow_id}:run-1": workflow_status}, engine=engine
+ )
+
+ await database_sync_to_async(job.refresh_from_db)()
+ assert cancelled == ([workflow_id] if expect_cancelled else [])
+ assert job.status == expected_status
+ assert job.error == expected_error
+
+
@pytest.mark.parametrize(
"node_ids,expected_preempted",
[
diff --git a/posthog/temporal/tests/data_modeling/test_record_skipped_jobs.py b/posthog/temporal/tests/data_modeling/test_record_skipped_jobs.py
index aba5432dea79..1bdbada73af4 100644
--- a/posthog/temporal/tests/data_modeling/test_record_skipped_jobs.py
+++ b/posthog/temporal/tests/data_modeling/test_record_skipped_jobs.py
@@ -1,3 +1,4 @@
+import datetime as dt
import dataclasses
import pytest
@@ -37,7 +38,7 @@ async def _make_node(team, dag, name):
return node, saved_query
-async def _run(team, dag, skipped_nodes):
+async def _run(team, dag, skipped_nodes, run_started_at=None):
env = ActivityEnvironment()
env.info = dataclasses.replace(env.info, workflow_id=WORKFLOW_ID, workflow_run_id="run-1")
await env.run(
@@ -47,6 +48,7 @@ async def _run(team, dag, skipped_nodes):
dag_id=str(dag.id),
engine=DataModelingJobEngine.CLICKHOUSE,
skipped_nodes=skipped_nodes,
+ run_started_at=run_started_at,
),
)
@@ -59,7 +61,16 @@ async def adag_local(ateam):
class TestRecordSkippedDataModelingJobs:
- async def test_writes_a_skip_row_naming_every_blocking_upstream(self, ateam, adag_local):
+ @pytest.mark.parametrize(
+ "built_by_another_run,expected_error",
+ [
+ (False, "Skipped because upstream views orders_daily and customers_daily are failing."),
+ (True, "Skipped because another run is already building this view."),
+ ],
+ )
+ async def test_writes_a_skip_row_naming_why_it_was_skipped(
+ self, ateam, adag_local, built_by_another_run: bool, expected_error: str
+ ):
blocked, blocked_query = await _make_node(ateam, adag_local, "rollup_daily")
parent_a, _ = await _make_node(ateam, adag_local, "orders_daily")
parent_b, _ = await _make_node(ateam, adag_local, "customers_daily")
@@ -68,7 +79,9 @@ class TestRecordSkippedDataModelingJobs:
ateam,
adag_local,
[
- SkippedDataModelingNode(
+ SkippedDataModelingNode(node_id=str(blocked.id), built_by_another_run=True)
+ if built_by_another_run
+ else SkippedDataModelingNode(
node_id=str(blocked.id),
failed_upstream_node_ids=[str(parent_a.id), str(parent_b.id)],
failed_upstream_total=2,
@@ -83,7 +96,7 @@ class TestRecordSkippedDataModelingJobs:
assert job.status == DataModelingJobStatus.SKIPPED
assert job.rows_materialized == 0
assert job.parent_workflow_id == WORKFLOW_ID
- assert job.error == "Skipped because upstream views orders_daily and customers_daily are failing."
+ assert job.error == expected_error
async def test_counts_the_upstreams_it_cannot_name(self, ateam, adag_local):
blocked, blocked_query = await _make_node(ateam, adag_local, "rollup_daily")
@@ -125,3 +138,30 @@ class TestRecordSkippedDataModelingJobs:
lambda: DataModelingJob.objects.filter(saved_query_id=blocked_query.id).count()
)()
assert count == 1
+
+ async def test_records_a_skip_when_an_earlier_run_left_a_job_under_the_same_parent(self, ateam, adag_local):
+ blocked, blocked_query = await _make_node(ateam, adag_local, "rollup_daily")
+ earlier = await database_sync_to_async(DataModelingJob.objects.create)(
+ team=ateam,
+ saved_query=blocked_query,
+ status=DataModelingJobStatus.COMPLETED,
+ engine=DataModelingJobEngine.CLICKHOUSE,
+ workflow_id="child",
+ workflow_run_id="child-run",
+ parent_workflow_id=WORKFLOW_ID,
+ )
+ run_started_at = (earlier.created_at + dt.timedelta(minutes=1)).isoformat()
+
+ await _run(
+ ateam,
+ adag_local,
+ [SkippedDataModelingNode(node_id=str(blocked.id), built_by_another_run=True)],
+ run_started_at,
+ )
+
+ statuses = await database_sync_to_async(
+ lambda: sorted(
+ DataModelingJob.objects.filter(saved_query_id=blocked_query.id).values_list("status", flat=True)
+ )
+ )()
+ assert statuses == sorted([DataModelingJobStatus.COMPLETED, DataModelingJobStatus.SKIPPED])