:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
chore(data-modeling): delete the in-workflow trino shadow comparison (#128353)
Jovan Sakovic97beacd7ea20
diff --git a/posthog/temporal/data_modeling/metrics.py b/posthog/temporal/data_modeling/metrics.pyindex 174e5453d874..4ccbe61726fe 100644--- a/posthog/temporal/data_modeling/metrics.py+++ b/posthog/temporal/data_modeling/metrics.py@@ -3,7 +3,6 @@ import refrom temporalio import activity, workflowfrom temporalio.common import MetricCounter, MetricHistogramFloat-from posthog.dataclasses import frozenfrom posthog.temporal.data_modeling.activities.utils import is_externally_aborted# custom latency histogram buckets,@@ -102,94 +101,6 @@ def get_managed_warehouse_trino_missing_object_metric(error_name: str) -> Metric)-@frozen-class _DualMetricCounters:- managed_warehouse: MetricCounter- legacy_duckgres: MetricCounter-- def add(self, value: int) -> None:- self.managed_warehouse.add(value)- self.legacy_duckgres.add(value)---@frozen-class _DualMetricHistograms:- managed_warehouse: MetricHistogramFloat- legacy_duckgres: MetricHistogramFloat-- def record(self, value: float) -> None:- self.managed_warehouse.record(value)- self.legacy_duckgres.record(value)---def get_managed_warehouse_shadow_finished_metrics(status: str) -> _DualMetricCounters:- meter = workflow.metric_meter().with_additional_attributes({"status": status})- description = "Number of managed warehouse shadow materialization activities finished, by status."- return _DualMetricCounters(- managed_warehouse=meter.create_counter("managed_warehouse_shadow_materialization_finished", description),- legacy_duckgres=meter.create_counter("duckgres_shadow_materialization_finished", description),- )---def get_managed_warehouse_shadow_row_count_match_metrics(matched: bool) -> _DualMetricCounters:- meter = workflow.metric_meter().with_additional_attributes({"matched": str(matched).lower()})- description = "Row count comparison between ClickHouse and managed warehouse shadow materializations."- return _DualMetricCounters(- managed_warehouse=meter.create_counter("managed_warehouse_shadow_row_count_comparison", description),- legacy_duckgres=meter.create_counter("duckgres_shadow_row_count_comparison", description),- )---def get_managed_warehouse_shadow_duration_metrics() -> _DualMetricHistograms:- meter = workflow.metric_meter()- description = "Duration of managed warehouse shadow materialization in seconds."- return _DualMetricHistograms(- managed_warehouse=meter.create_histogram_float(- "managed_warehouse_shadow_materialization_duration_seconds", description, "s"- ),- legacy_duckgres=meter.create_histogram_float(- "duckgres_shadow_materialization_duration_seconds", description, "s"- ),- )---def get_managed_warehouse_shadow_rows_materialized_metrics() -> _DualMetricHistograms:- meter = workflow.metric_meter()- description = "Number of rows materialized per managed warehouse shadow materialization."- return _DualMetricHistograms(- managed_warehouse=meter.create_histogram_float("managed_warehouse_shadow_rows_materialized", description),- legacy_duckgres=meter.create_histogram_float("duckgres_shadow_rows_materialized", description),- )---def get_managed_warehouse_shadow_storage_mib_metrics() -> _DualMetricHistograms:- meter = workflow.metric_meter()- description = "Total DuckLake storage size (MiB) after a managed warehouse shadow materialization."- return _DualMetricHistograms(- managed_warehouse=meter.create_histogram_float("managed_warehouse_shadow_storage_mib", description, "MiB"),- legacy_duckgres=meter.create_histogram_float("duckgres_shadow_storage_mib", description, "MiB"),- )---def get_managed_warehouse_shadow_storage_delta_mib_metrics() -> _DualMetricHistograms:- meter = workflow.metric_meter()- description = "Change in DuckLake storage size (MiB) after a managed warehouse shadow materialization."- return _DualMetricHistograms(- managed_warehouse=meter.create_histogram_float(- "managed_warehouse_shadow_storage_delta_mib", description, "MiB"- ),- legacy_duckgres=meter.create_histogram_float("duckgres_shadow_storage_delta_mib", description, "MiB"),- )---def get_clickhouse_materialization_duration_metric() -> MetricHistogramFloat:- return workflow.metric_meter().create_histogram_float(- "clickhouse_materialization_duration_seconds",- "Duration of ClickHouse materialization in seconds.",- "s",- )--# DAG-level metrics (v2 ExecuteDAGWorkflow)diff --git a/posthog/temporal/data_modeling/workflows/materialize_view.py b/posthog/temporal/data_modeling/workflows/materialize_view.pyindex 92eba4c9a5eb..80aae7377792 100644--- a/posthog/temporal/data_modeling/workflows/materialize_view.py+++ b/posthog/temporal/data_modeling/workflows/materialize_view.py@@ -46,13 +46,6 @@ from posthog.temporal.data_modeling.activities.enrich_view_semantics import Enrifrom posthog.temporal.data_modeling.metrics import (UNKNOWN_RUN_MODE,error_class,- get_clickhouse_materialization_duration_metric,- get_managed_warehouse_shadow_duration_metrics,- get_managed_warehouse_shadow_finished_metrics,- get_managed_warehouse_shadow_row_count_match_metrics,- get_managed_warehouse_shadow_rows_materialized_metrics,- get_managed_warehouse_shadow_storage_delta_mib_metrics,- get_managed_warehouse_shadow_storage_mib_metrics,get_node_duration_metric,get_node_finished_metric,get_node_rows_materialized_metric,@@ -232,8 +225,9 @@ class MaterializeViewWorkflow(PostHogWorkflow):managed_warehouse_job_id = Nonemanaged_warehouse_only = inputs.managed_warehouse_only- # Histories recorded without the patch marker replay the in-workflow shadow below.- if not managed_warehouse_only and temporalio.workflow.patched(CLICKHOUSE_RUN_WITHOUT_TRINO_SHADOW_PATCH):+ if not managed_warehouse_only:+ # Histories that carry the patch marker replay with deprecate_patch; delete it once those runs drain.+ temporalio.workflow.deprecate_patch(CLICKHOUSE_RUN_WITHOUT_TRINO_SHADOW_PATCH)managed_warehouse_enabled = Falseelse:managed_warehouse_enabled = await temporalio.workflow.execute_activity(@@ -375,16 +369,6 @@ class MaterializeViewWorkflow(PostHogWorkflow):await self._discard_staged_cdp_rows(inputs, job_id, materialize_result)end_time = temporalio.workflow.now()blocked_duration_seconds = (end_time - start_time).total_seconds()- if managed_warehouse_shadow_handle is not None:- shadow_succeeded = await self._collect_shadow_comparison(- managed_warehouse_shadow_handle,- managed_warehouse_job_id,- materialize_result.row_count,- blocked_duration_seconds,- inputs,- )- if use_trino:- trino_materialized = shadow_succeededreturn MaterializeViewWorkflowResult(job_id=job_id,node_id=inputs.node_id,@@ -449,18 +433,6 @@ class MaterializeViewWorkflow(PostHogWorkflow):await self._maybe_stage_account_properties(inputs, materialize_result, job_id)- # after the main workflow succeeds, collect shadow stats for comparison- if managed_warehouse_shadow_handle is not None:- shadow_succeeded = await self._collect_shadow_comparison(- managed_warehouse_shadow_handle,- managed_warehouse_job_id,- materialize_result.row_count,- duration_seconds,- inputs,- )- if use_trino:- trino_materialized = shadow_succeeded-temporalio.workflow.logger.info("MaterializeViewWorkflow completed successfully",extra={@@ -879,70 +851,6 @@ class MaterializeViewWorkflow(PostHogWorkflow):extra={"saved_query_id": saved_query_id, "error": str(e)},)- async def _collect_shadow_comparison(- self,- shadow_handle: temporalio.workflow.ActivityHandle[ManagedWarehouseShadowResult],- managed_warehouse_job_id: str | None,- clickhouse_row_count: int,- clickhouse_duration_seconds: float,- inputs: MaterializeViewWorkflowInputs,- ) -> bool:- """Await the managed warehouse shadow activity and emit comparison metrics.-- The activity itself is responsible for updating its job to a terminal state.- This is best-effort — any failure is swallowed so it never affects the workflow result.- """- try:- shadow_result: ManagedWarehouseShadowResult = await shadow_handle-- row_count_matched = clickhouse_row_count == shadow_result.row_count- if shadow_result.skipped:- status = "skipped"- elif shadow_result.error is None:- status = "completed"- else:- status = "failed"-- get_managed_warehouse_shadow_finished_metrics(status).add(1)- get_clickhouse_materialization_duration_metric().record(clickhouse_duration_seconds)- if shadow_result.error is None:- get_managed_warehouse_shadow_duration_metrics().record(shadow_result.duration_seconds)- get_managed_warehouse_shadow_rows_materialized_metrics().record(shadow_result.row_count)- get_managed_warehouse_shadow_row_count_match_metrics(row_count_matched).add(1)- if shadow_result.file_size_bytes > 0:- get_managed_warehouse_shadow_storage_mib_metrics().record(- shadow_result.file_size_bytes / (1024 * 1024)- )- if shadow_result.file_size_delta_bytes >= 0:- get_managed_warehouse_shadow_storage_delta_mib_metrics().record(- shadow_result.file_size_delta_bytes / (1024 * 1024)- )-- temporalio.workflow.logger.info(- "managed_warehouse_shadow_comparison",- extra={- "clickhouse_rows": clickhouse_row_count,- "clickhouse_duration_seconds": round(clickhouse_duration_seconds, 2),- "managed_warehouse_rows": shadow_result.row_count,- "managed_warehouse_duration_seconds": round(shadow_result.duration_seconds, 2),- "managed_warehouse_schema": shadow_result.schema_name,- "managed_warehouse_table": shadow_result.table_name,- "managed_warehouse_error": shadow_result.error,- "row_count_match": row_count_matched,- **inputs.properties_to_log,- },- )- return shadow_result.error is None- except Exception as shadow_err:- get_managed_warehouse_shadow_finished_metrics("error").add(1)- await self._finalize_orphaned_managed_warehouse_job(managed_warehouse_job_id, inputs, shadow_err)- temporalio.workflow.logger.warning(- f"Managed warehouse shadow comparison failed: {str(shadow_err)}",- extra=inputs.properties_to_log,- )- capture_exception(shadow_err)- return False-async def _finalize_orphaned_managed_warehouse_job(self,managed_warehouse_job_id: str | None,diff --git a/posthog/temporal/tests/data_modeling/test_materialize_view_workflow.py b/posthog/temporal/tests/data_modeling/test_materialize_view_workflow.pyindex 7d6237d1d7da..ec362b46abfc 100644--- a/posthog/temporal/tests/data_modeling/test_materialize_view_workflow.py+++ b/posthog/temporal/tests/data_modeling/test_materialize_view_workflow.py@@ -26,7 +26,6 @@ from posthog.temporal.data_modeling.workflows.materialize_view import ()from products.customer_analytics.backend.facade.temporal_contracts import StageAccountPropertySyncInput-from products.data_modeling.backend.facade.models import DataModelingJobEnginefrom products.data_quality.backend.facade.contracts import CHECK_SUITE_WORKFLOW_NAME, QualityAuditModefrom products.warehouse_sources.backend.facade.hooks import PersonPropertySyncActivityInputs@@ -62,11 +61,9 @@ class TestQualityGateBranching:activity_results: list,child_result: dict | Exception,*,- shadow_handle: AsyncMock | None = None,start_child: AsyncMock | None = None,inputs: MaterializeViewWorkflowInputs | None = None,execute_activity: AsyncMock | None = None,- patched: bool = True,) -> tuple:execute_activity = execute_activity or AsyncMock(side_effect=activity_results)child = (@@ -85,76 +82,19 @@ class TestQualityGateBranching:stack.enter_context(patch.object(temporalio.workflow, "info", return_value=info))stack.enter_context(patch.object(temporalio.workflow, "now", return_value=dt.datetime(2026, 8, 1)))stack.enter_context(patch.object(temporalio.workflow, "logger"))- stack.enter_context(patch.object(temporalio.workflow, "patched", return_value=patched))+ stack.enter_context(patch.object(temporalio.workflow, "deprecate_patch"))stack.enter_context(patch(f"{WORKFLOW_MODULE}.capture_exception"))- if shadow_handle is not None:- stack.enter_context(patch.object(temporalio.workflow, "start_activity", return_value=shadow_handle))for metric in ("get_node_finished_metric","get_node_duration_metric","get_node_rows_materialized_metric","get_node_storage_delta_mib_metric","get_node_total_storage_mib_metric",- "get_managed_warehouse_shadow_finished_metrics",- "get_clickhouse_materialization_duration_metric",- "get_managed_warehouse_shadow_duration_metrics",- "get_managed_warehouse_shadow_rows_materialized_metrics",- "get_managed_warehouse_shadow_row_count_match_metrics",- "get_managed_warehouse_shadow_storage_mib_metrics",- "get_managed_warehouse_shadow_storage_delta_mib_metrics",):self.metrics[metric] = stack.enter_context(patch(f"{WORKFLOW_MODULE}.{metric}"))result = await MaterializeViewWorkflow().run(inputs or _inputs())return result, execute_activity- @pytest.mark.parametrize(- "shadow_error,skipped,blocked,expected_status",- [- (None, False, False, "completed"),- ("Upstream write failed", False, False, "failed"),- ("Upstream table missing", True, False, "skipped"),- (None, False, True, None),- ],- )- async def test_a_history_without_the_patch_replays_the_in_workflow_shadow(- self, shadow_error: str | None, skipped: bool, blocked: bool, expected_status: str | None- ) -> None:- shadow = asyncio.get_running_loop().create_future()- shadow.set_result(- ManagedWarehouseShadowResult(- row_count=10,- duration_seconds=1,- schema_name="models",- table_name="orders",- error=shadow_error,- skipped=skipped,- )- )- activity_results = [- True,- "shadow-job",- "job-1",- _materialize_result("skip"),- PrepareQueryableTableResult(storage_delta_mib=None, total_storage_mib=None),- None,- None,- ]- if blocked:- activity_results.pop(1)- with patch.object(temporalio.workflow, "start_activity", return_value=shadow) as start_shadow:- result, _ = await self._run(- activity_results, {}, inputs=dataclasses.replace(_inputs(), skip_trino=blocked), patched=False- )- assert start_shadow.called is not blocked- assert result.job_id == "job-1"- assert result.rows_materialized == 10- assert result.trino_materialized is (not blocked and shadow_error is None)- shadow_finished = self.metrics["get_managed_warehouse_shadow_finished_metrics"]- if expected_status is None:- shadow_finished.assert_not_called()- else:- shadow_finished.assert_called_once_with(expected_status)-@pytest.mark.parametrize("shadow_error,skipped,expected_status,expected_error_class",[@@ -343,31 +283,6 @@ class TestQualityGateBranching:assert result.quality_audited is expected_auditedassert result.quality_blocking_failures is None- async def test_a_replayed_blocked_publish_still_settles_the_managed_warehouse_shadow(self):- # In a history recorded before the patch, the blocked branch returns early. Leaving the- # shadow activity unawaited there orphans the shadow job.- shadow_handle = AsyncMock()- shadow_handle.__await__ = lambda self=None: iter([])- activity_results = [- True, # managed warehouse shadow enabled- "managed-warehouse-job-1",- "job-1", # create clickhouse job- _materialize_result("gate"),- StageQueryableFilesResult(staged_folder_path="staged_1"),- None, # quality_block_materialization- ]-- with (- patch.object(MaterializeViewWorkflow, "_collect_shadow_comparison", new=AsyncMock()) as collect,- patch.object(temporalio.workflow, "start_activity", return_value=shadow_handle) as start_shadow,- ):- result, execute_activity = await self._run(activity_results, {"checks_failed_blocking": 1}, patched=False)-- assert result.quality_blocking_failures == 1- assert execute_activity.await_args_list[1].args[1].engine == DataModelingJobEngine.MANAGED_WAREHOUSE- collect.assert_awaited_once()- assert start_shadow.call_args.args[1].use_trino is True-def _cancelled_child_error() -> ChildWorkflowError:error = ChildWorkflowError(@@ -563,30 +478,6 @@ class TestMaybeEnrichViewSemantics:await workflow._maybe_enrich_view_semantics(_inputs(), result)-class TestCollectShadowComparison:- async def test_finalizes_orphaned_job_when_shadow_handle_errors(self):- workflow = MaterializeViewWorkflow()-- async def dead_handle():- raise RuntimeError("shadow activity died before finalizing")-- with (- patch.object(temporalio.workflow, "execute_activity", new=AsyncMock()) as execute_activity,- patch.object(temporalio.workflow, "logger"),- patch(f"{WORKFLOW_MODULE}.capture_exception"),- patch(f"{WORKFLOW_MODULE}.get_managed_warehouse_shadow_finished_metrics"),- ):- await workflow._collect_shadow_comparison(dead_handle(), "job-123", 5, 1.0, _inputs())-- # the activity died without self-finalizing, so the workflow must back it up- execute_activity.assert_awaited_once()- assert execute_activity.await_args is not None- activity, payload = execute_activity.await_args.args- assert activity is fail_materialization_activity- assert payload.job_id == "job-123"- assert payload.update_node is False--def _person_sync_result(**overrides) -> MaterializeViewResult:kwargs: dict = {"node_id": "node-1",