:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-modeling): label run metrics by tier, run mode and error class (#121148)
Jovan Sakovic8baf0145a46d
diff --git a/posthog/temporal/common/worker.py b/posthog/temporal/common/worker.pyindex 8d854bebba2e..22f327d22371 100644--- a/posthog/temporal/common/worker.py+++ b/posthog/temporal/common/worker.py@@ -43,6 +43,8 @@ from posthog.temporal.common.posthog_client import PostHogClientInterceptorfrom posthog.temporal.common.slo_interceptor import SloInterceptorfrom posthog.temporal.common.utils import configure_asyncify_executor, shutdown_asyncify_executorfrom posthog.temporal.data_modeling.metrics import (+ DATA_MODELING_DURATION_HISTOGRAM_BUCKETS,+ DATA_MODELING_DURATION_HISTOGRAM_METRICS,DATA_MODELING_LATENCY_HISTOGRAM_BUCKETS,DATA_MODELING_LATENCY_HISTOGRAM_METRICS,)@@ -442,6 +444,12 @@ async def create_worker(itertools.repeat(DATA_MODELING_LATENCY_HISTOGRAM_BUCKETS),))+ histogram_bucket_overrides |= dict(+ zip(+ DATA_MODELING_DURATION_HISTOGRAM_METRICS,+ itertools.repeat(DATA_MODELING_DURATION_HISTOGRAM_BUCKETS),+ )+ )if task_queue == settings.DATA_WAREHOUSE_TASK_QUEUE:# Both metrics hold small counts. The default buckets are for latencies in milliseconds,# which puts every count into the first bucket.diff --git a/posthog/temporal/data_modeling/activities/materialize_view_managed_warehouse.py b/posthog/temporal/data_modeling/activities/materialize_view_managed_warehouse.pyindex 6251334e39b3..9976e02a5fce 100644--- a/posthog/temporal/data_modeling/activities/materialize_view_managed_warehouse.py+++ b/posthog/temporal/data_modeling/activities/materialize_view_managed_warehouse.py@@ -24,6 +24,7 @@ from products.data_modeling.backend.facade.api import ()from products.data_modeling.backend.facade.models import (DataModelingJob,+ DataModelingJobRunMode,DataModelingJobStatus,DataWarehouseSavedQuery,Node,@@ -45,7 +46,7 @@ from products.managed_warehouse.backend.facade.contracts import ()from products.managed_warehouse.backend.facade.feature_flags import DATA_MODELING_SHADOW_FLAG-from ..metrics import get_node_suspended_metric+from ..metrics import UNKNOWN_RUN_MODE, get_node_suspended_metricfrom .utils import (CONSECUTIVE_FAILURES_TO_SUSPEND,bind_data_modeling_log_context,@@ -84,6 +85,10 @@ class ManagedWarehouseShadowInputs:}+INCREMENTAL_RUN_MODE = DataModelingJobRunMode.INCREMENTAL.value+FULL_REFRESH_RUN_MODE = DataModelingJobRunMode.FULL_REFRESH.value++@dataclasses.dataclass(frozen=False)class ManagedWarehouseShadowResult:row_count: int@@ -93,6 +98,15 @@ class ManagedWarehouseShadowResult:error: str | None = Nonefile_size_bytes: int = 0file_size_delta_bytes: int = 0+ run_mode: str = UNKNOWN_RUN_MODE+++def _failed_run_mode(plan: WritePlan | None, use_trino: bool) -> str:+ if not use_trino:+ return FULL_REFRESH_RUN_MODE+ if plan is None:+ return UNKNOWN_RUN_MODE+ return INCREMENTAL_RUN_MODE if plan.incremental else FULL_REFRESH_RUN_MODE@frozen@@ -324,6 +338,7 @@ async def _materialize_view_managed_warehouse(table_name=result.table_name,file_size_bytes=result.file_size_bytes,file_size_delta_bytes=result.file_size_delta_bytes,+ run_mode=INCREMENTAL_RUN_MODE if inputs.use_trino and result.merged else FULL_REFRESH_RUN_MODE,)job_engine = await _resolve_managed_warehouse_job(inputs.job_id, shadow_result)await clear_node_suspension_for_engine(@@ -348,6 +363,7 @@ async def _materialize_view_managed_warehouse(schema_name=schema_name,table_name=table_name,error=str(e),+ run_mode=_failed_run_mode(plan, inputs.use_trino),)if plan is not None and plan.incremental:# The merge may have stopped partway or failed on a schema change. Rebuilding next rundiff --git a/posthog/temporal/data_modeling/metrics.py b/posthog/temporal/data_modeling/metrics.pyindex b33c2d5fbc39..61344104ec58 100644--- a/posthog/temporal/data_modeling/metrics.py+++ b/posthog/temporal/data_modeling/metrics.py@@ -1,7 +1,10 @@+import re+from temporalio import activity, workflowfrom temporalio.common import MetricCounter, MetricHistogramFloatfrom posthog.dataclasses import frozen+from posthog.temporal.data_modeling.activities.utils import is_externally_aborted# custom latency histogram buckets,# since we lose some important granularity with default max at 60s@@ -29,6 +32,41 @@ DATA_MODELING_LATENCY_HISTOGRAM_BUCKETS = [3_600_000.0, # 1h (run_dag_activity start_to_close_timeout)]+# The SDK defaults (50s to 10,000s) leave both fast ClickHouse nodes and long Trino runs unreadable.+DATA_MODELING_DURATION_HISTOGRAM_METRICS = (+ "data_modeling_node_duration_seconds",+ "data_modeling_dag_duration_seconds",+)+DATA_MODELING_DURATION_HISTOGRAM_BUCKETS = [+ 5.0,+ 15.0,+ 30.0,+ 60.0,+ 120.0,+ 300.0,+ 600.0,+ 1_200.0,+ 1_800.0,+ 3_600.0,+ 7_200.0,+ 14_400.0,+ 21_600.0,+ 43_200.0, # TRINO_SHADOW_RUN_TIMEOUT+]++AD_HOC_TIER = "ad_hoc"+UNKNOWN_RUN_MODE = "unknown"++# Tier schedule runs are `execute-dag-{dag_id}:{seconds}[:{anchor}]-{timestamp}`, and their Trino+# shadows are `execute-dag-trino-{dag_id}:{seconds}[:{anchor}]`.+_TIER_WORKFLOW_ID = re.compile(r"^execute-dag-(?:trino-)?[0-9a-f-]{36}:(\d+)")+++def tier_label(dag_workflow_id: str | None) -> str:+ """The cadence tier in seconds of the DAG run with this workflow id, or `ad_hoc` for any other run."""+ match = _TIER_WORKFLOW_ID.match(dag_workflow_id or "")+ return match.group(1) if match else AD_HOC_TIER+def get_node_suspended_metric(engine: str) -> MetricCounter:return (@@ -132,10 +170,10 @@ def get_clickhouse_materialization_duration_metric() -> MetricHistogramFloat:# DAG-level metrics (v2 ExecuteDAGWorkflow)-def get_dag_finished_metric(status: str, engine: str) -> MetricCounter:+def get_dag_finished_metric(status: str, engine: str, *, tier: str) -> MetricCounter:return (workflow.metric_meter()- .with_additional_attributes({"status": status, "engine": engine})+ .with_additional_attributes({"status": status, "engine": engine, "tier_seconds": tier}).create_counter("data_modeling_dag_finished","Number of DAG executions finished. Status is completed, partial_failure, skipped, or failed.",@@ -143,10 +181,10 @@ def get_dag_finished_metric(status: str, engine: str) -> MetricCounter:)-def get_dag_duration_metric(engine: str) -> MetricHistogramFloat:+def get_dag_duration_metric(engine: str, *, tier: str) -> MetricHistogramFloat:return (workflow.metric_meter()- .with_additional_attributes({"engine": engine})+ .with_additional_attributes({"engine": engine, "tier_seconds": tier}).create_histogram_float("data_modeling_dag_duration_seconds","Total wall-clock duration of a DAG execution.",@@ -180,21 +218,30 @@ def get_trino_shadow_start_metric(outcome: str) -> MetricCounter:)-def get_node_finished_metric(status: str, engine: str) -> MetricCounter:+def get_node_finished_metric(+ status: str, engine: str, *, tier: str, run_mode: str, error_class: str = "none"+) -> MetricCounter:return (workflow.metric_meter()- .with_additional_attributes({"status": status, "engine": engine})+ .with_additional_attributes(+ {"status": status, "engine": engine, "tier_seconds": tier, "run_mode": run_mode, "error_class": error_class}+ ).create_counter("data_modeling_node_finished",- "Number of node materializations finished. Status is completed, failed, cancelled, or quality_blocked.",+ "Number of node materializations finished. Status is completed, failed, cancelled, or quality_blocked. "+ "A failure's error_class is infra when it does not count toward suspension, otherwise other.",))-def get_node_duration_metric(engine: str) -> MetricHistogramFloat:+def error_class(error: str) -> str:+ return "infra" if is_externally_aborted(error) else "other"+++def get_node_duration_metric(engine: str, *, run_mode: str) -> MetricHistogramFloat:return (workflow.metric_meter()- .with_additional_attributes({"engine": engine})+ .with_additional_attributes({"engine": engine, "run_mode": run_mode}).create_histogram_float("data_modeling_node_duration_seconds","Wall-clock duration of a single node materialization.",diff --git a/posthog/temporal/data_modeling/workflows/execute_dag.py b/posthog/temporal/data_modeling/workflows/execute_dag.pyindex 13a779f6750e..d1b20fac353e 100644--- a/posthog/temporal/data_modeling/workflows/execute_dag.py+++ b/posthog/temporal/data_modeling/workflows/execute_dag.py@@ -31,6 +31,7 @@ from posthog.temporal.data_modeling.metrics import (get_dag_finished_metric,get_dag_node_count_metric,get_trino_shadow_start_metric,+ tier_label,)from posthog.temporal.data_modeling.workflows.materialize_view import (MaterializeViewWorkflow,@@ -565,8 +566,9 @@ class ExecuteDAGWorkflow(PostHogWorkflow):else:dag_status = "partial_failure"engine = _serving_engine(inputs)- get_dag_finished_metric(dag_status, engine).add(1)- get_dag_duration_metric(engine).record(duration_seconds)+ tier = tier_label(temporalio.workflow.info().workflow_id)+ get_dag_finished_metric(dag_status, engine, tier=tier).add(1)+ get_dag_duration_metric(engine, tier=tier).record(duration_seconds)get_dag_node_count_metric("successful", engine).record(successful_nodes)get_dag_node_count_metric("failed", engine).record(failed_nodes)get_dag_node_count_metric("skipped", engine).record(skipped_nodes)diff --git a/posthog/temporal/data_modeling/workflows/materialize_view.py b/posthog/temporal/data_modeling/workflows/materialize_view.pyindex b6f563217217..a9b1e5868b2d 100644--- a/posthog/temporal/data_modeling/workflows/materialize_view.py+++ b/posthog/temporal/data_modeling/workflows/materialize_view.py@@ -44,6 +44,8 @@ from posthog.temporal.data_modeling.activities import ()from posthog.temporal.data_modeling.activities.enrich_view_semantics import EnrichViewSemanticsInputsfrom posthog.temporal.data_modeling.metrics import (+ UNKNOWN_RUN_MODE,+ error_class,get_clickhouse_materialization_duration_metric,get_managed_warehouse_shadow_duration_metrics,get_managed_warehouse_shadow_finished_metrics,@@ -56,12 +58,13 @@ from posthog.temporal.data_modeling.metrics import (get_node_rows_materialized_metric,get_node_storage_delta_mib_metric,get_node_total_storage_mib_metric,+ tier_label,)from posthog.temporal.data_modeling.workflows.enrich_view_semantics import EnrichViewSemanticsWorkflowfrom posthog.temporal.utils import CDPProducerWorkflowInputsfrom products.customer_analytics.backend.facade.temporal_contracts import StageAccountPropertySyncInput-from products.data_modeling.backend.facade.models import DataModelingJobEngine+from products.data_modeling.backend.facade.models import DataModelingJobEngine, DataModelingJobRunModefrom products.data_quality.backend.facade.contracts import (CHECK_SUITE_WORKFLOW_NAME,QUALITY_AUDIT_GATE,@@ -165,12 +168,26 @@ CLICKHOUSE_ENGINE = DataModelingJobEngine.CLICKHOUSE.valueMANAGED_WAREHOUSE_ENGINE = DataModelingJobEngine.MANAGED_WAREHOUSE.value-def _record_managed_warehouse_node_metrics(result: ManagedWarehouseShadowResult) -> None:+def _run_mode(materialize_result: MaterializeViewResult | None) -> str:+ if materialize_result is None:+ return UNKNOWN_RUN_MODE+ if materialize_result.incremental:+ return DataModelingJobRunMode.INCREMENTAL.value+ return DataModelingJobRunMode.FULL_REFRESH.value+++def _record_managed_warehouse_node_metrics(result: ManagedWarehouseShadowResult, tier: str) -> None:if result.error is not None:- get_node_finished_metric("failed", MANAGED_WAREHOUSE_ENGINE).add(1)+ get_node_finished_metric(+ "failed",+ MANAGED_WAREHOUSE_ENGINE,+ tier=tier,+ run_mode=result.run_mode,+ error_class=error_class(result.error),+ ).add(1)return- get_node_finished_metric("completed", MANAGED_WAREHOUSE_ENGINE).add(1)- get_node_duration_metric(MANAGED_WAREHOUSE_ENGINE).record(result.duration_seconds)+ get_node_finished_metric("completed", MANAGED_WAREHOUSE_ENGINE, tier=tier, run_mode=result.run_mode).add(1)+ get_node_duration_metric(MANAGED_WAREHOUSE_ENGINE, run_mode=result.run_mode).record(result.duration_seconds)get_node_rows_materialized_metric(MANAGED_WAREHOUSE_ENGINE).record(result.row_count)if result.file_size_bytes > 0:get_node_total_storage_mib_metric(MANAGED_WAREHOUSE_ENGINE).record(result.file_size_bytes / (1024 * 1024))@@ -207,6 +224,7 @@ class MaterializeViewWorkflow(PostHogWorkflow):start_time = temporalio.workflow.now()parent_info = temporalio.workflow.info().parentparent_workflow_id = parent_info.workflow_id if parent_info else None+ tier = tier_label(parent_workflow_id)job_id = Nonemanaged_warehouse_job_id = Nonemanaged_warehouse_only = inputs.managed_warehouse_only@@ -348,7 +366,9 @@ class MaterializeViewWorkflow(PostHogWorkflow):start_to_close_timeout=dt.timedelta(minutes=5),retry_policy=temporalio.common.RetryPolicy(maximum_attempts=3),)- get_node_finished_metric("quality_blocked", CLICKHOUSE_ENGINE).add(1)+ get_node_finished_metric(+ "quality_blocked", CLICKHOUSE_ENGINE, tier=tier, run_mode=_run_mode(materialize_result)+ ).add(1)await self._discard_staged_cdp_rows(inputs, job_id, materialize_result)end_time = temporalio.workflow.now()blocked_duration_seconds = (end_time - start_time).total_seconds()@@ -448,8 +468,9 @@ class MaterializeViewWorkflow(PostHogWorkflow):)# node-level metrics- get_node_finished_metric("completed", CLICKHOUSE_ENGINE).add(1)- get_node_duration_metric(CLICKHOUSE_ENGINE).record(duration_seconds)+ run_mode = _run_mode(materialize_result)+ get_node_finished_metric("completed", CLICKHOUSE_ENGINE, tier=tier, run_mode=run_mode).add(1)+ get_node_duration_metric(CLICKHOUSE_ENGINE, run_mode=run_mode).record(duration_seconds)get_node_rows_materialized_metric(CLICKHOUSE_ENGINE).record(materialize_result.row_count)if storage_result.storage_delta_mib is not None and storage_result.storage_delta_mib >= 0:get_node_storage_delta_mib_metric(CLICKHOUSE_ENGINE).record(storage_result.storage_delta_mib)@@ -506,7 +527,13 @@ class MaterializeViewWorkflow(PostHogWorkflow):f"Failed to mark job as failed: {str(fail_err)}",extra=inputs.properties_to_log,)- get_node_finished_metric("cancelled" if cancelled else "failed", CLICKHOUSE_ENGINE).add(1)+ get_node_finished_metric(+ "cancelled" if cancelled else "failed",+ CLICKHOUSE_ENGINE,+ tier=tier,+ run_mode=_run_mode(materialize_result),+ error_class="none" if cancelled else error_class(error_message),+ ).add(1)raise# Keep the parent workflow's concurrency slot until managed warehouse finishes the query.@@ -514,10 +541,15 @@ class MaterializeViewWorkflow(PostHogWorkflow):if managed_warehouse_shadow_handle is not None:try:result = await managed_warehouse_shadow_handle- _record_managed_warehouse_node_metrics(result)+ _record_managed_warehouse_node_metrics(result, tier)except Exception as shadow_err:+ shadow_cancelled = _is_cancellation(shadow_err)get_node_finished_metric(- "cancelled" if _is_cancellation(shadow_err) else "failed", MANAGED_WAREHOUSE_ENGINE+ "cancelled" if shadow_cancelled else "failed",+ MANAGED_WAREHOUSE_ENGINE,+ tier=tier,+ run_mode=UNKNOWN_RUN_MODE,+ error_class="none" if shadow_cancelled else error_class(str(shadow_err)),).add(1)await self._finalize_orphaned_managed_warehouse_job(managed_warehouse_job_id, inputs, str(shadow_err))temporalio.workflow.logger.warning(diff --git a/posthog/temporal/tests/data_modeling/test_execute_dag_workflow.py b/posthog/temporal/tests/data_modeling/test_execute_dag_workflow.pyindex 3281332a7312..835bd269b93e 100644--- a/posthog/temporal/tests/data_modeling/test_execute_dag_workflow.py+++ b/posthog/temporal/tests/data_modeling/test_execute_dag_workflow.py@@ -1010,6 +1010,7 @@ class TestTrinoDependencyOutcomes:return Noneinfo = MagicMock()+ info.workflow_id = "execute-dag-0190a1b2-c3d4-7e5f-8a9b-0c1d2e3f4a5b:3600-2026-10-21T10:00:00Z"with (patch.object(temporal_workflow, "execute_activity", new=AsyncMock(side_effect=execute_activity)diff --git a/posthog/temporal/tests/data_modeling/test_materialize_view_workflow.py b/posthog/temporal/tests/data_modeling/test_materialize_view_workflow.pyindex 7324422a94b2..ff195d80715a 100644--- a/posthog/temporal/tests/data_modeling/test_materialize_view_workflow.py+++ b/posthog/temporal/tests/data_modeling/test_materialize_view_workflow.py@@ -151,6 +151,7 @@ class TestQualityGateBranching:schema_name="models",table_name="orders",error=shadow_error,+ run_mode="incremental",))with patch.object(temporalio.workflow, "start_activity", return_value=shadow):@@ -168,6 +169,8 @@ class TestQualityGateBranching:finished = self.metrics["get_node_finished_metric"]assert finished.call_args.args == ("failed" if shadow_error else "completed", "managed_warehouse")+ assert finished.call_args.kwargs.get("error_class", "none") == ("other" if shadow_error else "none")+ assert finished.call_args.kwargs["run_mode"] == "incremental"assert self.metrics["get_node_duration_metric"].called is (shadow_error is None)async def test_a_clickhouse_run_does_not_wait_for_a_slow_trino_shadow(self) -> None:diff --git a/posthog/temporal/tests/data_modeling/test_metrics.py b/posthog/temporal/tests/data_modeling/test_metrics.pynew file mode 100644index 000000000000..9a338b4a7eef--- /dev/null+++ b/posthog/temporal/tests/data_modeling/test_metrics.py@@ -0,0 +1,17 @@+from parameterized import parameterized++from posthog.temporal.data_modeling.metrics import tier_label+++ [+ ("execute-dag-0190a1b2-c3d4-7e5f-8a9b-0c1d2e3f4a5b:3600-2026-10-21T10:00:00Z", "3600"),+ ("execute-dag-0190a1b2-c3d4-7e5f-8a9b-0c1d2e3f4a5b:2592000:10800-2026-10-21T03:00:00Z", "2592000"),+ ("execute-dag-trino-0190a1b2-c3d4-7e5f-8a9b-0c1d2e3f4a5b:900", "900"),+ ("execute-dag-0190a1b2-c3d4-7e5f-8a9b-0c1d2e3f4a5b-2026-10-21T10:00:00Z", "ad_hoc"),+ ("materialize-view-manual-run", "ad_hoc"),+ (None, "ad_hoc"),+ ]+)+def test_tier_label(workflow_id: str | None, expected: str) -> None:+ assert tier_label(workflow_id) == expecteddiff --git a/products/data_modeling/backend/facade/models.py b/products/data_modeling/backend/facade/models.pyindex 471adc2d7d15..e59d0b217d84 100644--- a/products/data_modeling/backend/facade/models.py+++ b/products/data_modeling/backend/facade/models.py@@ -13,6 +13,7 @@ from products.data_modeling.backend.models.dag import DAG, DEFAULT_DAG_NAMEfrom products.data_modeling.backend.models.data_modeling_job import (DataModelingJob,DataModelingJobEngine,+ DataModelingJobRunMode,DataModelingJobStatus,)from products.data_modeling.backend.models.datawarehouse_managed_viewset import DataWarehouseManagedViewSet@@ -37,6 +38,7 @@ __all__ = ["SAVED_QUERY_NODE_TYPES","DataModelingJob","DataModelingJobEngine",+ "DataModelingJobRunMode","DataModelingJobStatus","DataWarehouseManagedViewSet","DataWarehouseSavedQuery",