Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-modeling): give trino shadow runs room to finish the daily dag (#123154)

Jovan Sakovic727be7bd3f9d
diff --git a/posthog/temporal/data_modeling/metrics.py b/posthog/temporal/data_modeling/metrics.py
index b2c5312a0013..174e5453d874 100644
--- a/posthog/temporal/data_modeling/metrics.py
+++ b/posthog/temporal/data_modeling/metrics.py
@@ -51,7 +51,8 @@ DATA_MODELING_DURATION_HISTOGRAM_BUCKETS = [
7_200.0,
14_400.0,
21_600.0,
- 43_200.0, # TRINO_SHADOW_RUN_TIMEOUT
+ 43_200.0,
+ 82_800.0, # TRINO_SHADOW_RUN_TIMEOUT
]
AD_HOC_TIER = "ad_hoc"
diff --git a/posthog/temporal/data_modeling/workflows/execute_dag.py b/posthog/temporal/data_modeling/workflows/execute_dag.py
index d1b20fac353e..5911586de7f8 100644
--- a/posthog/temporal/data_modeling/workflows/execute_dag.py
+++ b/posthog/temporal/data_modeling/workflows/execute_dag.py
@@ -45,9 +45,13 @@ from products.data_quality.backend.facade.contracts import CHECK_SUITE_WORKFLOW_
from products.data_quality.backend.facade.enums import SuiteRunTrigger
MAX_CONCURRENT_CHILDREN = 10
+# A Trino model runs for hours, so at the ClickHouse limit a shadow run cannot get through its DAG before its timeout.
+TRINO_SHADOW_MAX_CONCURRENT_CHILDREN = 25
+TRINO_SHADOW_CONCURRENCY_PATCH = "data-modeling-trino-shadow-concurrency"
TRINO_SHADOW_DAG_RUN_PATCH = "data-modeling-trino-shadow-dag-run"
-TRINO_SHADOW_RUN_TIMEOUT = dt.timedelta(hours=12)
+# Under a day, so a run that uses its whole budget still ends before the next daily shadow starts.
+TRINO_SHADOW_RUN_TIMEOUT = dt.timedelta(hours=23)
def trino_shadow_workflow_id(workflow_id: str, dag_id: str) -> str:
@@ -329,7 +333,10 @@ class ExecuteDAGWorkflow(PostHogWorkflow):
# execute child workflows with bounded concurrency using a sliding window;
# the semaphore limits how many child workflows run simultaneously across
# all levels to be a friendlier neighbor to managed warehouse and ClickHouse infrastructure
- semaphore = asyncio.Semaphore(MAX_CONCURRENT_CHILDREN)
+ max_concurrent_children = MAX_CONCURRENT_CHILDREN
+ if managed_warehouse_only and temporalio.workflow.patched(TRINO_SHADOW_CONCURRENCY_PATCH):
+ max_concurrent_children = TRINO_SHADOW_MAX_CONCURRENT_CHILDREN
+ semaphore = asyncio.Semaphore(max_concurrent_children)
for i, level in enumerate(levels):
temporalio.workflow.logger.info(
f"Executing level {i + 1}/{len(levels)}",
diff --git a/posthog/temporal/tests/data_modeling/test_execute_dag_workflow.py b/posthog/temporal/tests/data_modeling/test_execute_dag_workflow.py
index 835bd269b93e..b0036cd83893 100644
--- a/posthog/temporal/tests/data_modeling/test_execute_dag_workflow.py
+++ b/posthog/temporal/tests/data_modeling/test_execute_dag_workflow.py
@@ -1130,7 +1130,7 @@ class TestTrinoShadowRun:
)
assert shadow.kwargs["id"] == expected_shadow_id
assert shadow.kwargs["parent_close_policy"] == ParentClosePolicy.ABANDON
- assert shadow.kwargs["execution_timeout"] == dt.timedelta(hours=12)
+ assert shadow.kwargs["execution_timeout"] == dt.timedelta(hours=23)
assert outcome.result.successful_nodes == 2
@pytest.mark.parametrize(