:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(data-modeling): give trino shadow runs room to finish the daily dag (#123154)
Jovan Sakovic727be7bd3f9d
diff --git a/posthog/temporal/data_modeling/metrics.py b/posthog/temporal/data_modeling/metrics.pyindex b2c5312a0013..174e5453d874 100644--- a/posthog/temporal/data_modeling/metrics.py+++ b/posthog/temporal/data_modeling/metrics.py@@ -51,7 +51,8 @@ DATA_MODELING_DURATION_HISTOGRAM_BUCKETS = [7_200.0,14_400.0,21_600.0,- 43_200.0, # TRINO_SHADOW_RUN_TIMEOUT+ 43_200.0,+ 82_800.0, # TRINO_SHADOW_RUN_TIMEOUT]AD_HOC_TIER = "ad_hoc"diff --git a/posthog/temporal/data_modeling/workflows/execute_dag.py b/posthog/temporal/data_modeling/workflows/execute_dag.pyindex d1b20fac353e..5911586de7f8 100644--- a/posthog/temporal/data_modeling/workflows/execute_dag.py+++ b/posthog/temporal/data_modeling/workflows/execute_dag.py@@ -45,9 +45,13 @@ from products.data_quality.backend.facade.contracts import CHECK_SUITE_WORKFLOW_from products.data_quality.backend.facade.enums import SuiteRunTriggerMAX_CONCURRENT_CHILDREN = 10+# A Trino model runs for hours, so at the ClickHouse limit a shadow run cannot get through its DAG before its timeout.+TRINO_SHADOW_MAX_CONCURRENT_CHILDREN = 25+TRINO_SHADOW_CONCURRENCY_PATCH = "data-modeling-trino-shadow-concurrency"TRINO_SHADOW_DAG_RUN_PATCH = "data-modeling-trino-shadow-dag-run"-TRINO_SHADOW_RUN_TIMEOUT = dt.timedelta(hours=12)+# Under a day, so a run that uses its whole budget still ends before the next daily shadow starts.+TRINO_SHADOW_RUN_TIMEOUT = dt.timedelta(hours=23)def trino_shadow_workflow_id(workflow_id: str, dag_id: str) -> str:@@ -329,7 +333,10 @@ class ExecuteDAGWorkflow(PostHogWorkflow):# execute child workflows with bounded concurrency using a sliding window;# the semaphore limits how many child workflows run simultaneously across# all levels to be a friendlier neighbor to managed warehouse and ClickHouse infrastructure- semaphore = asyncio.Semaphore(MAX_CONCURRENT_CHILDREN)+ max_concurrent_children = MAX_CONCURRENT_CHILDREN+ if managed_warehouse_only and temporalio.workflow.patched(TRINO_SHADOW_CONCURRENCY_PATCH):+ max_concurrent_children = TRINO_SHADOW_MAX_CONCURRENT_CHILDREN+ semaphore = asyncio.Semaphore(max_concurrent_children)for i, level in enumerate(levels):temporalio.workflow.logger.info(f"Executing level {i + 1}/{len(levels)}",diff --git a/posthog/temporal/tests/data_modeling/test_execute_dag_workflow.py b/posthog/temporal/tests/data_modeling/test_execute_dag_workflow.pyindex 835bd269b93e..b0036cd83893 100644--- a/posthog/temporal/tests/data_modeling/test_execute_dag_workflow.py+++ b/posthog/temporal/tests/data_modeling/test_execute_dag_workflow.py@@ -1130,7 +1130,7 @@ class TestTrinoShadowRun:)assert shadow.kwargs["id"] == expected_shadow_idassert shadow.kwargs["parent_close_policy"] == ParentClosePolicy.ABANDON- assert shadow.kwargs["execution_timeout"] == dt.timedelta(hours=12)+ assert shadow.kwargs["execution_timeout"] == dt.timedelta(hours=23)assert outcome.result.successful_nodes == 2@pytest.mark.parametrize(