Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-modeling): label run metrics by tier, run mode and error class (#121148)

Jovan Sakovic8baf0145a46d
diff --git a/posthog/temporal/common/worker.py b/posthog/temporal/common/worker.py
index 8d854bebba2e..22f327d22371 100644
--- a/posthog/temporal/common/worker.py
+++ b/posthog/temporal/common/worker.py
@@ -43,6 +43,8 @@ from posthog.temporal.common.posthog_client import PostHogClientInterceptor
from posthog.temporal.common.slo_interceptor import SloInterceptor
from posthog.temporal.common.utils import configure_asyncify_executor, shutdown_asyncify_executor
from posthog.temporal.data_modeling.metrics import (
+ DATA_MODELING_DURATION_HISTOGRAM_BUCKETS,
+ DATA_MODELING_DURATION_HISTOGRAM_METRICS,
DATA_MODELING_LATENCY_HISTOGRAM_BUCKETS,
DATA_MODELING_LATENCY_HISTOGRAM_METRICS,
)
@@ -442,6 +444,12 @@ async def create_worker(
itertools.repeat(DATA_MODELING_LATENCY_HISTOGRAM_BUCKETS),
)
)
+ histogram_bucket_overrides |= dict(
+ zip(
+ DATA_MODELING_DURATION_HISTOGRAM_METRICS,
+ itertools.repeat(DATA_MODELING_DURATION_HISTOGRAM_BUCKETS),
+ )
+ )
if task_queue == settings.DATA_WAREHOUSE_TASK_QUEUE:
# Both metrics hold small counts. The default buckets are for latencies in milliseconds,
# which puts every count into the first bucket.
diff --git a/posthog/temporal/data_modeling/activities/materialize_view_managed_warehouse.py b/posthog/temporal/data_modeling/activities/materialize_view_managed_warehouse.py
index 6251334e39b3..9976e02a5fce 100644
--- a/posthog/temporal/data_modeling/activities/materialize_view_managed_warehouse.py
+++ b/posthog/temporal/data_modeling/activities/materialize_view_managed_warehouse.py
@@ -24,6 +24,7 @@ from products.data_modeling.backend.facade.api import (
)
from products.data_modeling.backend.facade.models import (
DataModelingJob,
+ DataModelingJobRunMode,
DataModelingJobStatus,
DataWarehouseSavedQuery,
Node,
@@ -45,7 +46,7 @@ from products.managed_warehouse.backend.facade.contracts import (
)
from products.managed_warehouse.backend.facade.feature_flags import DATA_MODELING_SHADOW_FLAG
-from ..metrics import get_node_suspended_metric
+from ..metrics import UNKNOWN_RUN_MODE, get_node_suspended_metric
from .utils import (
CONSECUTIVE_FAILURES_TO_SUSPEND,
bind_data_modeling_log_context,
@@ -84,6 +85,10 @@ class ManagedWarehouseShadowInputs:
}
+INCREMENTAL_RUN_MODE = DataModelingJobRunMode.INCREMENTAL.value
+FULL_REFRESH_RUN_MODE = DataModelingJobRunMode.FULL_REFRESH.value
+
+
@dataclasses.dataclass(frozen=False)
class ManagedWarehouseShadowResult:
row_count: int
@@ -93,6 +98,15 @@ class ManagedWarehouseShadowResult:
error: str | None = None
file_size_bytes: int = 0
file_size_delta_bytes: int = 0
+ run_mode: str = UNKNOWN_RUN_MODE
+
+
+def _failed_run_mode(plan: WritePlan | None, use_trino: bool) -> str:
+ if not use_trino:
+ return FULL_REFRESH_RUN_MODE
+ if plan is None:
+ return UNKNOWN_RUN_MODE
+ return INCREMENTAL_RUN_MODE if plan.incremental else FULL_REFRESH_RUN_MODE
@frozen
@@ -324,6 +338,7 @@ async def _materialize_view_managed_warehouse(
table_name=result.table_name,
file_size_bytes=result.file_size_bytes,
file_size_delta_bytes=result.file_size_delta_bytes,
+ run_mode=INCREMENTAL_RUN_MODE if inputs.use_trino and result.merged else FULL_REFRESH_RUN_MODE,
)
job_engine = await _resolve_managed_warehouse_job(inputs.job_id, shadow_result)
await clear_node_suspension_for_engine(
@@ -348,6 +363,7 @@ async def _materialize_view_managed_warehouse(
schema_name=schema_name,
table_name=table_name,
error=str(e),
+ run_mode=_failed_run_mode(plan, inputs.use_trino),
)
if plan is not None and plan.incremental:
# The merge may have stopped partway or failed on a schema change. Rebuilding next run
diff --git a/posthog/temporal/data_modeling/metrics.py b/posthog/temporal/data_modeling/metrics.py
index b33c2d5fbc39..61344104ec58 100644
--- a/posthog/temporal/data_modeling/metrics.py
+++ b/posthog/temporal/data_modeling/metrics.py
@@ -1,7 +1,10 @@
+import re
+
from temporalio import activity, workflow
from temporalio.common import MetricCounter, MetricHistogramFloat
from posthog.dataclasses import frozen
+from posthog.temporal.data_modeling.activities.utils import is_externally_aborted
# custom latency histogram buckets,
# since we lose some important granularity with default max at 60s
@@ -29,6 +32,41 @@ DATA_MODELING_LATENCY_HISTOGRAM_BUCKETS = [
3_600_000.0, # 1h (run_dag_activity start_to_close_timeout)
]
+# The SDK defaults (50s to 10,000s) leave both fast ClickHouse nodes and long Trino runs unreadable.
+DATA_MODELING_DURATION_HISTOGRAM_METRICS = (
+ "data_modeling_node_duration_seconds",
+ "data_modeling_dag_duration_seconds",
+)
+DATA_MODELING_DURATION_HISTOGRAM_BUCKETS = [
+ 5.0,
+ 15.0,
+ 30.0,
+ 60.0,
+ 120.0,
+ 300.0,
+ 600.0,
+ 1_200.0,
+ 1_800.0,
+ 3_600.0,
+ 7_200.0,
+ 14_400.0,
+ 21_600.0,
+ 43_200.0, # TRINO_SHADOW_RUN_TIMEOUT
+]
+
+AD_HOC_TIER = "ad_hoc"
+UNKNOWN_RUN_MODE = "unknown"
+
+# Tier schedule runs are `execute-dag-{dag_id}:{seconds}[:{anchor}]-{timestamp}`, and their Trino
+# shadows are `execute-dag-trino-{dag_id}:{seconds}[:{anchor}]`.
+_TIER_WORKFLOW_ID = re.compile(r"^execute-dag-(?:trino-)?[0-9a-f-]{36}:(\d+)")
+
+
+def tier_label(dag_workflow_id: str | None) -> str:
+ """The cadence tier in seconds of the DAG run with this workflow id, or `ad_hoc` for any other run."""
+ match = _TIER_WORKFLOW_ID.match(dag_workflow_id or "")
+ return match.group(1) if match else AD_HOC_TIER
+
def get_node_suspended_metric(engine: str) -> MetricCounter:
return (
@@ -132,10 +170,10 @@ def get_clickhouse_materialization_duration_metric() -> MetricHistogramFloat:
# DAG-level metrics (v2 ExecuteDAGWorkflow)
-def get_dag_finished_metric(status: str, engine: str) -> MetricCounter:
+def get_dag_finished_metric(status: str, engine: str, *, tier: str) -> MetricCounter:
return (
workflow.metric_meter()
- .with_additional_attributes({"status": status, "engine": engine})
+ .with_additional_attributes({"status": status, "engine": engine, "tier_seconds": tier})
.create_counter(
"data_modeling_dag_finished",
"Number of DAG executions finished. Status is completed, partial_failure, skipped, or failed.",
@@ -143,10 +181,10 @@ def get_dag_finished_metric(status: str, engine: str) -> MetricCounter:
)
-def get_dag_duration_metric(engine: str) -> MetricHistogramFloat:
+def get_dag_duration_metric(engine: str, *, tier: str) -> MetricHistogramFloat:
return (
workflow.metric_meter()
- .with_additional_attributes({"engine": engine})
+ .with_additional_attributes({"engine": engine, "tier_seconds": tier})
.create_histogram_float(
"data_modeling_dag_duration_seconds",
"Total wall-clock duration of a DAG execution.",
@@ -180,21 +218,30 @@ def get_trino_shadow_start_metric(outcome: str) -> MetricCounter:
)
-def get_node_finished_metric(status: str, engine: str) -> MetricCounter:
+def get_node_finished_metric(
+ status: str, engine: str, *, tier: str, run_mode: str, error_class: str = "none"
+) -> MetricCounter:
return (
workflow.metric_meter()
- .with_additional_attributes({"status": status, "engine": engine})
+ .with_additional_attributes(
+ {"status": status, "engine": engine, "tier_seconds": tier, "run_mode": run_mode, "error_class": error_class}
+ )
.create_counter(
"data_modeling_node_finished",
- "Number of node materializations finished. Status is completed, failed, cancelled, or quality_blocked.",
+ "Number of node materializations finished. Status is completed, failed, cancelled, or quality_blocked. "
+ "A failure's error_class is infra when it does not count toward suspension, otherwise other.",
)
)
-def get_node_duration_metric(engine: str) -> MetricHistogramFloat:
+def error_class(error: str) -> str:
+ return "infra" if is_externally_aborted(error) else "other"
+
+
+def get_node_duration_metric(engine: str, *, run_mode: str) -> MetricHistogramFloat:
return (
workflow.metric_meter()
- .with_additional_attributes({"engine": engine})
+ .with_additional_attributes({"engine": engine, "run_mode": run_mode})
.create_histogram_float(
"data_modeling_node_duration_seconds",
"Wall-clock duration of a single node materialization.",
diff --git a/posthog/temporal/data_modeling/workflows/execute_dag.py b/posthog/temporal/data_modeling/workflows/execute_dag.py
index 13a779f6750e..d1b20fac353e 100644
--- a/posthog/temporal/data_modeling/workflows/execute_dag.py
+++ b/posthog/temporal/data_modeling/workflows/execute_dag.py
@@ -31,6 +31,7 @@ from posthog.temporal.data_modeling.metrics import (
get_dag_finished_metric,
get_dag_node_count_metric,
get_trino_shadow_start_metric,
+ tier_label,
)
from posthog.temporal.data_modeling.workflows.materialize_view import (
MaterializeViewWorkflow,
@@ -565,8 +566,9 @@ class ExecuteDAGWorkflow(PostHogWorkflow):
else:
dag_status = "partial_failure"
engine = _serving_engine(inputs)
- get_dag_finished_metric(dag_status, engine).add(1)
- get_dag_duration_metric(engine).record(duration_seconds)
+ tier = tier_label(temporalio.workflow.info().workflow_id)
+ get_dag_finished_metric(dag_status, engine, tier=tier).add(1)
+ get_dag_duration_metric(engine, tier=tier).record(duration_seconds)
get_dag_node_count_metric("successful", engine).record(successful_nodes)
get_dag_node_count_metric("failed", engine).record(failed_nodes)
get_dag_node_count_metric("skipped", engine).record(skipped_nodes)
diff --git a/posthog/temporal/data_modeling/workflows/materialize_view.py b/posthog/temporal/data_modeling/workflows/materialize_view.py
index b6f563217217..a9b1e5868b2d 100644
--- a/posthog/temporal/data_modeling/workflows/materialize_view.py
+++ b/posthog/temporal/data_modeling/workflows/materialize_view.py
@@ -44,6 +44,8 @@ from posthog.temporal.data_modeling.activities import (
)
from posthog.temporal.data_modeling.activities.enrich_view_semantics import EnrichViewSemanticsInputs
from posthog.temporal.data_modeling.metrics import (
+ UNKNOWN_RUN_MODE,
+ error_class,
get_clickhouse_materialization_duration_metric,
get_managed_warehouse_shadow_duration_metrics,
get_managed_warehouse_shadow_finished_metrics,
@@ -56,12 +58,13 @@ from posthog.temporal.data_modeling.metrics import (
get_node_rows_materialized_metric,
get_node_storage_delta_mib_metric,
get_node_total_storage_mib_metric,
+ tier_label,
)
from posthog.temporal.data_modeling.workflows.enrich_view_semantics import EnrichViewSemanticsWorkflow
from posthog.temporal.utils import CDPProducerWorkflowInputs
from products.customer_analytics.backend.facade.temporal_contracts import StageAccountPropertySyncInput
-from products.data_modeling.backend.facade.models import DataModelingJobEngine
+from products.data_modeling.backend.facade.models import DataModelingJobEngine, DataModelingJobRunMode
from products.data_quality.backend.facade.contracts import (
CHECK_SUITE_WORKFLOW_NAME,
QUALITY_AUDIT_GATE,
@@ -165,12 +168,26 @@ CLICKHOUSE_ENGINE = DataModelingJobEngine.CLICKHOUSE.value
MANAGED_WAREHOUSE_ENGINE = DataModelingJobEngine.MANAGED_WAREHOUSE.value
-def _record_managed_warehouse_node_metrics(result: ManagedWarehouseShadowResult) -> None:
+def _run_mode(materialize_result: MaterializeViewResult | None) -> str:
+ if materialize_result is None:
+ return UNKNOWN_RUN_MODE
+ if materialize_result.incremental:
+ return DataModelingJobRunMode.INCREMENTAL.value
+ return DataModelingJobRunMode.FULL_REFRESH.value
+
+
+def _record_managed_warehouse_node_metrics(result: ManagedWarehouseShadowResult, tier: str) -> None:
if result.error is not None:
- get_node_finished_metric("failed", MANAGED_WAREHOUSE_ENGINE).add(1)
+ get_node_finished_metric(
+ "failed",
+ MANAGED_WAREHOUSE_ENGINE,
+ tier=tier,
+ run_mode=result.run_mode,
+ error_class=error_class(result.error),
+ ).add(1)
return
- get_node_finished_metric("completed", MANAGED_WAREHOUSE_ENGINE).add(1)
- get_node_duration_metric(MANAGED_WAREHOUSE_ENGINE).record(result.duration_seconds)
+ get_node_finished_metric("completed", MANAGED_WAREHOUSE_ENGINE, tier=tier, run_mode=result.run_mode).add(1)
+ get_node_duration_metric(MANAGED_WAREHOUSE_ENGINE, run_mode=result.run_mode).record(result.duration_seconds)
get_node_rows_materialized_metric(MANAGED_WAREHOUSE_ENGINE).record(result.row_count)
if result.file_size_bytes > 0:
get_node_total_storage_mib_metric(MANAGED_WAREHOUSE_ENGINE).record(result.file_size_bytes / (1024 * 1024))
@@ -207,6 +224,7 @@ class MaterializeViewWorkflow(PostHogWorkflow):
start_time = temporalio.workflow.now()
parent_info = temporalio.workflow.info().parent
parent_workflow_id = parent_info.workflow_id if parent_info else None
+ tier = tier_label(parent_workflow_id)
job_id = None
managed_warehouse_job_id = None
managed_warehouse_only = inputs.managed_warehouse_only
@@ -348,7 +366,9 @@ class MaterializeViewWorkflow(PostHogWorkflow):
start_to_close_timeout=dt.timedelta(minutes=5),
retry_policy=temporalio.common.RetryPolicy(maximum_attempts=3),
)
- get_node_finished_metric("quality_blocked", CLICKHOUSE_ENGINE).add(1)
+ get_node_finished_metric(
+ "quality_blocked", CLICKHOUSE_ENGINE, tier=tier, run_mode=_run_mode(materialize_result)
+ ).add(1)
await self._discard_staged_cdp_rows(inputs, job_id, materialize_result)
end_time = temporalio.workflow.now()
blocked_duration_seconds = (end_time - start_time).total_seconds()
@@ -448,8 +468,9 @@ class MaterializeViewWorkflow(PostHogWorkflow):
)
# node-level metrics
- get_node_finished_metric("completed", CLICKHOUSE_ENGINE).add(1)
- get_node_duration_metric(CLICKHOUSE_ENGINE).record(duration_seconds)
+ run_mode = _run_mode(materialize_result)
+ get_node_finished_metric("completed", CLICKHOUSE_ENGINE, tier=tier, run_mode=run_mode).add(1)
+ get_node_duration_metric(CLICKHOUSE_ENGINE, run_mode=run_mode).record(duration_seconds)
get_node_rows_materialized_metric(CLICKHOUSE_ENGINE).record(materialize_result.row_count)
if storage_result.storage_delta_mib is not None and storage_result.storage_delta_mib >= 0:
get_node_storage_delta_mib_metric(CLICKHOUSE_ENGINE).record(storage_result.storage_delta_mib)
@@ -506,7 +527,13 @@ class MaterializeViewWorkflow(PostHogWorkflow):
f"Failed to mark job as failed: {str(fail_err)}",
extra=inputs.properties_to_log,
)
- get_node_finished_metric("cancelled" if cancelled else "failed", CLICKHOUSE_ENGINE).add(1)
+ get_node_finished_metric(
+ "cancelled" if cancelled else "failed",
+ CLICKHOUSE_ENGINE,
+ tier=tier,
+ run_mode=_run_mode(materialize_result),
+ error_class="none" if cancelled else error_class(error_message),
+ ).add(1)
raise
# Keep the parent workflow's concurrency slot until managed warehouse finishes the query.
@@ -514,10 +541,15 @@ class MaterializeViewWorkflow(PostHogWorkflow):
if managed_warehouse_shadow_handle is not None:
try:
result = await managed_warehouse_shadow_handle
- _record_managed_warehouse_node_metrics(result)
+ _record_managed_warehouse_node_metrics(result, tier)
except Exception as shadow_err:
+ shadow_cancelled = _is_cancellation(shadow_err)
get_node_finished_metric(
- "cancelled" if _is_cancellation(shadow_err) else "failed", MANAGED_WAREHOUSE_ENGINE
+ "cancelled" if shadow_cancelled else "failed",
+ MANAGED_WAREHOUSE_ENGINE,
+ tier=tier,
+ run_mode=UNKNOWN_RUN_MODE,
+ error_class="none" if shadow_cancelled else error_class(str(shadow_err)),
).add(1)
await self._finalize_orphaned_managed_warehouse_job(managed_warehouse_job_id, inputs, str(shadow_err))
temporalio.workflow.logger.warning(
diff --git a/posthog/temporal/tests/data_modeling/test_execute_dag_workflow.py b/posthog/temporal/tests/data_modeling/test_execute_dag_workflow.py
index 3281332a7312..835bd269b93e 100644
--- a/posthog/temporal/tests/data_modeling/test_execute_dag_workflow.py
+++ b/posthog/temporal/tests/data_modeling/test_execute_dag_workflow.py
@@ -1010,6 +1010,7 @@ class TestTrinoDependencyOutcomes:
return None
info = MagicMock()
+ info.workflow_id = "execute-dag-0190a1b2-c3d4-7e5f-8a9b-0c1d2e3f4a5b:3600-2026-10-21T10:00:00Z"
with (
patch.object(
temporal_workflow, "execute_activity", new=AsyncMock(side_effect=execute_activity)
diff --git a/posthog/temporal/tests/data_modeling/test_materialize_view_workflow.py b/posthog/temporal/tests/data_modeling/test_materialize_view_workflow.py
index 7324422a94b2..ff195d80715a 100644
--- a/posthog/temporal/tests/data_modeling/test_materialize_view_workflow.py
+++ b/posthog/temporal/tests/data_modeling/test_materialize_view_workflow.py
@@ -151,6 +151,7 @@ class TestQualityGateBranching:
schema_name="models",
table_name="orders",
error=shadow_error,
+ run_mode="incremental",
)
)
with patch.object(temporalio.workflow, "start_activity", return_value=shadow):
@@ -168,6 +169,8 @@ class TestQualityGateBranching:
finished = self.metrics["get_node_finished_metric"]
assert finished.call_args.args == ("failed" if shadow_error else "completed", "managed_warehouse")
+ assert finished.call_args.kwargs.get("error_class", "none") == ("other" if shadow_error else "none")
+ assert finished.call_args.kwargs["run_mode"] == "incremental"
assert self.metrics["get_node_duration_metric"].called is (shadow_error is None)
async def test_a_clickhouse_run_does_not_wait_for_a_slow_trino_shadow(self) -> None:
diff --git a/posthog/temporal/tests/data_modeling/test_metrics.py b/posthog/temporal/tests/data_modeling/test_metrics.py
new file mode 100644
index 000000000000..9a338b4a7eef
--- /dev/null
+++ b/posthog/temporal/tests/data_modeling/test_metrics.py
@@ -0,0 +1,17 @@
+from parameterized import parameterized
+
+from posthog.temporal.data_modeling.metrics import tier_label
+
+
+ [
+ ("execute-dag-0190a1b2-c3d4-7e5f-8a9b-0c1d2e3f4a5b:3600-2026-10-21T10:00:00Z", "3600"),
+ ("execute-dag-0190a1b2-c3d4-7e5f-8a9b-0c1d2e3f4a5b:2592000:10800-2026-10-21T03:00:00Z", "2592000"),
+ ("execute-dag-trino-0190a1b2-c3d4-7e5f-8a9b-0c1d2e3f4a5b:900", "900"),
+ ("execute-dag-0190a1b2-c3d4-7e5f-8a9b-0c1d2e3f4a5b-2026-10-21T10:00:00Z", "ad_hoc"),
+ ("materialize-view-manual-run", "ad_hoc"),
+ (None, "ad_hoc"),
+ ]
+)
+def test_tier_label(workflow_id: str | None, expected: str) -> None:
+ assert tier_label(workflow_id) == expected
diff --git a/products/data_modeling/backend/facade/models.py b/products/data_modeling/backend/facade/models.py
index 471adc2d7d15..e59d0b217d84 100644
--- a/products/data_modeling/backend/facade/models.py
+++ b/products/data_modeling/backend/facade/models.py
@@ -13,6 +13,7 @@ from products.data_modeling.backend.models.dag import DAG, DEFAULT_DAG_NAME
from products.data_modeling.backend.models.data_modeling_job import (
DataModelingJob,
DataModelingJobEngine,
+ DataModelingJobRunMode,
DataModelingJobStatus,
)
from products.data_modeling.backend.models.datawarehouse_managed_viewset import DataWarehouseManagedViewSet
@@ -37,6 +38,7 @@ __all__ = [
"SAVED_QUERY_NODE_TYPES",
"DataModelingJob",
"DataModelingJobEngine",
+ "DataModelingJobRunMode",
"DataModelingJobStatus",
"DataWarehouseManagedViewSet",
"DataWarehouseSavedQuery",