Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

chore(data-modeling): delete the in-workflow trino shadow comparison (#128353)

Jovan Sakovic97beacd7ea20
diff --git a/posthog/temporal/data_modeling/metrics.py b/posthog/temporal/data_modeling/metrics.py
index 174e5453d874..4ccbe61726fe 100644
--- a/posthog/temporal/data_modeling/metrics.py
+++ b/posthog/temporal/data_modeling/metrics.py
@@ -3,7 +3,6 @@ import re
from temporalio import activity, workflow
from temporalio.common import MetricCounter, MetricHistogramFloat
-from posthog.dataclasses import frozen
from posthog.temporal.data_modeling.activities.utils import is_externally_aborted
# custom latency histogram buckets,
@@ -102,94 +101,6 @@ def get_managed_warehouse_trino_missing_object_metric(error_name: str) -> Metric
)
-@frozen
-class _DualMetricCounters:
- managed_warehouse: MetricCounter
- legacy_duckgres: MetricCounter
-
- def add(self, value: int) -> None:
- self.managed_warehouse.add(value)
- self.legacy_duckgres.add(value)
-
-
-@frozen
-class _DualMetricHistograms:
- managed_warehouse: MetricHistogramFloat
- legacy_duckgres: MetricHistogramFloat
-
- def record(self, value: float) -> None:
- self.managed_warehouse.record(value)
- self.legacy_duckgres.record(value)
-
-
-def get_managed_warehouse_shadow_finished_metrics(status: str) -> _DualMetricCounters:
- meter = workflow.metric_meter().with_additional_attributes({"status": status})
- description = "Number of managed warehouse shadow materialization activities finished, by status."
- return _DualMetricCounters(
- managed_warehouse=meter.create_counter("managed_warehouse_shadow_materialization_finished", description),
- legacy_duckgres=meter.create_counter("duckgres_shadow_materialization_finished", description),
- )
-
-
-def get_managed_warehouse_shadow_row_count_match_metrics(matched: bool) -> _DualMetricCounters:
- meter = workflow.metric_meter().with_additional_attributes({"matched": str(matched).lower()})
- description = "Row count comparison between ClickHouse and managed warehouse shadow materializations."
- return _DualMetricCounters(
- managed_warehouse=meter.create_counter("managed_warehouse_shadow_row_count_comparison", description),
- legacy_duckgres=meter.create_counter("duckgres_shadow_row_count_comparison", description),
- )
-
-
-def get_managed_warehouse_shadow_duration_metrics() -> _DualMetricHistograms:
- meter = workflow.metric_meter()
- description = "Duration of managed warehouse shadow materialization in seconds."
- return _DualMetricHistograms(
- managed_warehouse=meter.create_histogram_float(
- "managed_warehouse_shadow_materialization_duration_seconds", description, "s"
- ),
- legacy_duckgres=meter.create_histogram_float(
- "duckgres_shadow_materialization_duration_seconds", description, "s"
- ),
- )
-
-
-def get_managed_warehouse_shadow_rows_materialized_metrics() -> _DualMetricHistograms:
- meter = workflow.metric_meter()
- description = "Number of rows materialized per managed warehouse shadow materialization."
- return _DualMetricHistograms(
- managed_warehouse=meter.create_histogram_float("managed_warehouse_shadow_rows_materialized", description),
- legacy_duckgres=meter.create_histogram_float("duckgres_shadow_rows_materialized", description),
- )
-
-
-def get_managed_warehouse_shadow_storage_mib_metrics() -> _DualMetricHistograms:
- meter = workflow.metric_meter()
- description = "Total DuckLake storage size (MiB) after a managed warehouse shadow materialization."
- return _DualMetricHistograms(
- managed_warehouse=meter.create_histogram_float("managed_warehouse_shadow_storage_mib", description, "MiB"),
- legacy_duckgres=meter.create_histogram_float("duckgres_shadow_storage_mib", description, "MiB"),
- )
-
-
-def get_managed_warehouse_shadow_storage_delta_mib_metrics() -> _DualMetricHistograms:
- meter = workflow.metric_meter()
- description = "Change in DuckLake storage size (MiB) after a managed warehouse shadow materialization."
- return _DualMetricHistograms(
- managed_warehouse=meter.create_histogram_float(
- "managed_warehouse_shadow_storage_delta_mib", description, "MiB"
- ),
- legacy_duckgres=meter.create_histogram_float("duckgres_shadow_storage_delta_mib", description, "MiB"),
- )
-
-
-def get_clickhouse_materialization_duration_metric() -> MetricHistogramFloat:
- return workflow.metric_meter().create_histogram_float(
- "clickhouse_materialization_duration_seconds",
- "Duration of ClickHouse materialization in seconds.",
- "s",
- )
-
-
# DAG-level metrics (v2 ExecuteDAGWorkflow)
diff --git a/posthog/temporal/data_modeling/workflows/materialize_view.py b/posthog/temporal/data_modeling/workflows/materialize_view.py
index 92eba4c9a5eb..80aae7377792 100644
--- a/posthog/temporal/data_modeling/workflows/materialize_view.py
+++ b/posthog/temporal/data_modeling/workflows/materialize_view.py
@@ -46,13 +46,6 @@ from posthog.temporal.data_modeling.activities.enrich_view_semantics import Enri
from posthog.temporal.data_modeling.metrics import (
UNKNOWN_RUN_MODE,
error_class,
- get_clickhouse_materialization_duration_metric,
- get_managed_warehouse_shadow_duration_metrics,
- get_managed_warehouse_shadow_finished_metrics,
- get_managed_warehouse_shadow_row_count_match_metrics,
- get_managed_warehouse_shadow_rows_materialized_metrics,
- get_managed_warehouse_shadow_storage_delta_mib_metrics,
- get_managed_warehouse_shadow_storage_mib_metrics,
get_node_duration_metric,
get_node_finished_metric,
get_node_rows_materialized_metric,
@@ -232,8 +225,9 @@ class MaterializeViewWorkflow(PostHogWorkflow):
managed_warehouse_job_id = None
managed_warehouse_only = inputs.managed_warehouse_only
- # Histories recorded without the patch marker replay the in-workflow shadow below.
- if not managed_warehouse_only and temporalio.workflow.patched(CLICKHOUSE_RUN_WITHOUT_TRINO_SHADOW_PATCH):
+ if not managed_warehouse_only:
+ # Histories that carry the patch marker replay with deprecate_patch; delete it once those runs drain.
+ temporalio.workflow.deprecate_patch(CLICKHOUSE_RUN_WITHOUT_TRINO_SHADOW_PATCH)
managed_warehouse_enabled = False
else:
managed_warehouse_enabled = await temporalio.workflow.execute_activity(
@@ -375,16 +369,6 @@ class MaterializeViewWorkflow(PostHogWorkflow):
await self._discard_staged_cdp_rows(inputs, job_id, materialize_result)
end_time = temporalio.workflow.now()
blocked_duration_seconds = (end_time - start_time).total_seconds()
- if managed_warehouse_shadow_handle is not None:
- shadow_succeeded = await self._collect_shadow_comparison(
- managed_warehouse_shadow_handle,
- managed_warehouse_job_id,
- materialize_result.row_count,
- blocked_duration_seconds,
- inputs,
- )
- if use_trino:
- trino_materialized = shadow_succeeded
return MaterializeViewWorkflowResult(
job_id=job_id,
node_id=inputs.node_id,
@@ -449,18 +433,6 @@ class MaterializeViewWorkflow(PostHogWorkflow):
await self._maybe_stage_account_properties(inputs, materialize_result, job_id)
- # after the main workflow succeeds, collect shadow stats for comparison
- if managed_warehouse_shadow_handle is not None:
- shadow_succeeded = await self._collect_shadow_comparison(
- managed_warehouse_shadow_handle,
- managed_warehouse_job_id,
- materialize_result.row_count,
- duration_seconds,
- inputs,
- )
- if use_trino:
- trino_materialized = shadow_succeeded
-
temporalio.workflow.logger.info(
"MaterializeViewWorkflow completed successfully",
extra={
@@ -879,70 +851,6 @@ class MaterializeViewWorkflow(PostHogWorkflow):
extra={"saved_query_id": saved_query_id, "error": str(e)},
)
- async def _collect_shadow_comparison(
- self,
- shadow_handle: temporalio.workflow.ActivityHandle[ManagedWarehouseShadowResult],
- managed_warehouse_job_id: str | None,
- clickhouse_row_count: int,
- clickhouse_duration_seconds: float,
- inputs: MaterializeViewWorkflowInputs,
- ) -> bool:
- """Await the managed warehouse shadow activity and emit comparison metrics.
-
- The activity itself is responsible for updating its job to a terminal state.
- This is best-effort — any failure is swallowed so it never affects the workflow result.
- """
- try:
- shadow_result: ManagedWarehouseShadowResult = await shadow_handle
-
- row_count_matched = clickhouse_row_count == shadow_result.row_count
- if shadow_result.skipped:
- status = "skipped"
- elif shadow_result.error is None:
- status = "completed"
- else:
- status = "failed"
-
- get_managed_warehouse_shadow_finished_metrics(status).add(1)
- get_clickhouse_materialization_duration_metric().record(clickhouse_duration_seconds)
- if shadow_result.error is None:
- get_managed_warehouse_shadow_duration_metrics().record(shadow_result.duration_seconds)
- get_managed_warehouse_shadow_rows_materialized_metrics().record(shadow_result.row_count)
- get_managed_warehouse_shadow_row_count_match_metrics(row_count_matched).add(1)
- if shadow_result.file_size_bytes > 0:
- get_managed_warehouse_shadow_storage_mib_metrics().record(
- shadow_result.file_size_bytes / (1024 * 1024)
- )
- if shadow_result.file_size_delta_bytes >= 0:
- get_managed_warehouse_shadow_storage_delta_mib_metrics().record(
- shadow_result.file_size_delta_bytes / (1024 * 1024)
- )
-
- temporalio.workflow.logger.info(
- "managed_warehouse_shadow_comparison",
- extra={
- "clickhouse_rows": clickhouse_row_count,
- "clickhouse_duration_seconds": round(clickhouse_duration_seconds, 2),
- "managed_warehouse_rows": shadow_result.row_count,
- "managed_warehouse_duration_seconds": round(shadow_result.duration_seconds, 2),
- "managed_warehouse_schema": shadow_result.schema_name,
- "managed_warehouse_table": shadow_result.table_name,
- "managed_warehouse_error": shadow_result.error,
- "row_count_match": row_count_matched,
- **inputs.properties_to_log,
- },
- )
- return shadow_result.error is None
- except Exception as shadow_err:
- get_managed_warehouse_shadow_finished_metrics("error").add(1)
- await self._finalize_orphaned_managed_warehouse_job(managed_warehouse_job_id, inputs, shadow_err)
- temporalio.workflow.logger.warning(
- f"Managed warehouse shadow comparison failed: {str(shadow_err)}",
- extra=inputs.properties_to_log,
- )
- capture_exception(shadow_err)
- return False
-
async def _finalize_orphaned_managed_warehouse_job(
self,
managed_warehouse_job_id: str | None,
diff --git a/posthog/temporal/tests/data_modeling/test_materialize_view_workflow.py b/posthog/temporal/tests/data_modeling/test_materialize_view_workflow.py
index 7d6237d1d7da..ec362b46abfc 100644
--- a/posthog/temporal/tests/data_modeling/test_materialize_view_workflow.py
+++ b/posthog/temporal/tests/data_modeling/test_materialize_view_workflow.py
@@ -26,7 +26,6 @@ from posthog.temporal.data_modeling.workflows.materialize_view import (
)
from products.customer_analytics.backend.facade.temporal_contracts import StageAccountPropertySyncInput
-from products.data_modeling.backend.facade.models import DataModelingJobEngine
from products.data_quality.backend.facade.contracts import CHECK_SUITE_WORKFLOW_NAME, QualityAuditMode
from products.warehouse_sources.backend.facade.hooks import PersonPropertySyncActivityInputs
@@ -62,11 +61,9 @@ class TestQualityGateBranching:
activity_results: list,
child_result: dict | Exception,
*,
- shadow_handle: AsyncMock | None = None,
start_child: AsyncMock | None = None,
inputs: MaterializeViewWorkflowInputs | None = None,
execute_activity: AsyncMock | None = None,
- patched: bool = True,
) -> tuple:
execute_activity = execute_activity or AsyncMock(side_effect=activity_results)
child = (
@@ -85,76 +82,19 @@ class TestQualityGateBranching:
stack.enter_context(patch.object(temporalio.workflow, "info", return_value=info))
stack.enter_context(patch.object(temporalio.workflow, "now", return_value=dt.datetime(2026, 8, 1)))
stack.enter_context(patch.object(temporalio.workflow, "logger"))
- stack.enter_context(patch.object(temporalio.workflow, "patched", return_value=patched))
+ stack.enter_context(patch.object(temporalio.workflow, "deprecate_patch"))
stack.enter_context(patch(f"{WORKFLOW_MODULE}.capture_exception"))
- if shadow_handle is not None:
- stack.enter_context(patch.object(temporalio.workflow, "start_activity", return_value=shadow_handle))
for metric in (
"get_node_finished_metric",
"get_node_duration_metric",
"get_node_rows_materialized_metric",
"get_node_storage_delta_mib_metric",
"get_node_total_storage_mib_metric",
- "get_managed_warehouse_shadow_finished_metrics",
- "get_clickhouse_materialization_duration_metric",
- "get_managed_warehouse_shadow_duration_metrics",
- "get_managed_warehouse_shadow_rows_materialized_metrics",
- "get_managed_warehouse_shadow_row_count_match_metrics",
- "get_managed_warehouse_shadow_storage_mib_metrics",
- "get_managed_warehouse_shadow_storage_delta_mib_metrics",
):
self.metrics[metric] = stack.enter_context(patch(f"{WORKFLOW_MODULE}.{metric}"))
result = await MaterializeViewWorkflow().run(inputs or _inputs())
return result, execute_activity
- @pytest.mark.parametrize(
- "shadow_error,skipped,blocked,expected_status",
- [
- (None, False, False, "completed"),
- ("Upstream write failed", False, False, "failed"),
- ("Upstream table missing", True, False, "skipped"),
- (None, False, True, None),
- ],
- )
- async def test_a_history_without_the_patch_replays_the_in_workflow_shadow(
- self, shadow_error: str | None, skipped: bool, blocked: bool, expected_status: str | None
- ) -> None:
- shadow = asyncio.get_running_loop().create_future()
- shadow.set_result(
- ManagedWarehouseShadowResult(
- row_count=10,
- duration_seconds=1,
- schema_name="models",
- table_name="orders",
- error=shadow_error,
- skipped=skipped,
- )
- )
- activity_results = [
- True,
- "shadow-job",
- "job-1",
- _materialize_result("skip"),
- PrepareQueryableTableResult(storage_delta_mib=None, total_storage_mib=None),
- None,
- None,
- ]
- if blocked:
- activity_results.pop(1)
- with patch.object(temporalio.workflow, "start_activity", return_value=shadow) as start_shadow:
- result, _ = await self._run(
- activity_results, {}, inputs=dataclasses.replace(_inputs(), skip_trino=blocked), patched=False
- )
- assert start_shadow.called is not blocked
- assert result.job_id == "job-1"
- assert result.rows_materialized == 10
- assert result.trino_materialized is (not blocked and shadow_error is None)
- shadow_finished = self.metrics["get_managed_warehouse_shadow_finished_metrics"]
- if expected_status is None:
- shadow_finished.assert_not_called()
- else:
- shadow_finished.assert_called_once_with(expected_status)
-
@pytest.mark.parametrize(
"shadow_error,skipped,expected_status,expected_error_class",
[
@@ -343,31 +283,6 @@ class TestQualityGateBranching:
assert result.quality_audited is expected_audited
assert result.quality_blocking_failures is None
- async def test_a_replayed_blocked_publish_still_settles_the_managed_warehouse_shadow(self):
- # In a history recorded before the patch, the blocked branch returns early. Leaving the
- # shadow activity unawaited there orphans the shadow job.
- shadow_handle = AsyncMock()
- shadow_handle.__await__ = lambda self=None: iter([])
- activity_results = [
- True, # managed warehouse shadow enabled
- "managed-warehouse-job-1",
- "job-1", # create clickhouse job
- _materialize_result("gate"),
- StageQueryableFilesResult(staged_folder_path="staged_1"),
- None, # quality_block_materialization
- ]
-
- with (
- patch.object(MaterializeViewWorkflow, "_collect_shadow_comparison", new=AsyncMock()) as collect,
- patch.object(temporalio.workflow, "start_activity", return_value=shadow_handle) as start_shadow,
- ):
- result, execute_activity = await self._run(activity_results, {"checks_failed_blocking": 1}, patched=False)
-
- assert result.quality_blocking_failures == 1
- assert execute_activity.await_args_list[1].args[1].engine == DataModelingJobEngine.MANAGED_WAREHOUSE
- collect.assert_awaited_once()
- assert start_shadow.call_args.args[1].use_trino is True
-
def _cancelled_child_error() -> ChildWorkflowError:
error = ChildWorkflowError(
@@ -563,30 +478,6 @@ class TestMaybeEnrichViewSemantics:
await workflow._maybe_enrich_view_semantics(_inputs(), result)
-class TestCollectShadowComparison:
- async def test_finalizes_orphaned_job_when_shadow_handle_errors(self):
- workflow = MaterializeViewWorkflow()
-
- async def dead_handle():
- raise RuntimeError("shadow activity died before finalizing")
-
- with (
- patch.object(temporalio.workflow, "execute_activity", new=AsyncMock()) as execute_activity,
- patch.object(temporalio.workflow, "logger"),
- patch(f"{WORKFLOW_MODULE}.capture_exception"),
- patch(f"{WORKFLOW_MODULE}.get_managed_warehouse_shadow_finished_metrics"),
- ):
- await workflow._collect_shadow_comparison(dead_handle(), "job-123", 5, 1.0, _inputs())
-
- # the activity died without self-finalizing, so the workflow must back it up
- execute_activity.assert_awaited_once()
- assert execute_activity.await_args is not None
- activity, payload = execute_activity.await_args.args
- assert activity is fail_materialization_activity
- assert payload.job_id == "job-123"
- assert payload.update_node is False
-
-
def _person_sync_result(**overrides) -> MaterializeViewResult:
kwargs: dict = {
"node_id": "node-1",