:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(data-warehouse): keep earliest value after a full reload of a desc source (#125560)
Daniel RC7cdbf4c0ebf5
diff --git a/products/warehouse_sources/backend/models/external_data_schema.py b/products/warehouse_sources/backend/models/external_data_schema.pyindex a41c1427fb22..4bee659b1784 100644--- a/products/warehouse_sources/backend/models/external_data_schema.py+++ b/products/warehouse_sources/backend/models/external_data_schema.py@@ -1232,11 +1232,18 @@ class ExternalDataSchema( # nosemgrep: semgrep.rules.security.prefer-uuid7-djanreturnfound = Truefield_type = config.get("incremental_field_type")+ # A stored watermark with no earliest value means the table already holds every older row.+ # A run from that watermark read only newer rows, so its oldest row must not become the+ # earliest value: the next run would read every row below it again.+ watermark_without_earliest = (+ config.get("incremental_field_last_value") is not None+ and config.get("incremental_field_earliest_value") is None+ )if "last_value" in staged:_advance_promoted_cursor(config, "incremental_field_last_value", staged["last_value"], "last", field_type)- if "earliest_value" in staged:+ if "earliest_value" in staged and not watermark_without_earliest:_advance_promoted_cursor(config, "incremental_field_earliest_value", staged["earliest_value"], "earliest", field_type)diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.pyindex c33ebbf4097f..453834bd82a1 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.py@@ -931,6 +931,11 @@ class PipelineV3(Generic[ResumableData]):self._logger,webhook_only=self._resource.webhook_only,)+ # A reset drops the earliest value of the table it deletes. No reloaded row is older than+ # that value, so a run that kept it would never stage an earliest value of its own.+ self._earliest_incremental_field_value = process_incremental_value(+ self._schema.incremental_field_earliest_value, self._schema.incremental_field_type+ )is_fresh_sync = self._delta_table_ref.is_first_sync or self._schema.table is Noneif not is_fresh_sync and self._attempt > 1 and not should_resume:diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.pyindex 9237242f11d0..bbc55b25c6ce 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.py@@ -1643,6 +1643,56 @@ class TestDescWatermarkAcrossResumedAttempts:assert schema.sync_type_config["incremental_field_last_value"] == self._NEWEST.isoformat()+ @pytest.mark.asyncio+ async def test_a_reset_run_promotes_the_oldest_row_it_loaded_as_the_earliest_value(self) -> None:+ # The deleted table held the same rows, so no row of the reload is older than its stored earliest value.+ oldest_row = self._NEWEST - timedelta(hours=5)+ schema = self._schema()+ schema.sync_type_config.update(+ incremental_field_last_value=self._NEWEST.isoformat(),+ incremental_field_earliest_value=oldest_row.isoformat(),+ )+ pipeline = self._attempt(schema, 1)+ pipeline._reset_pipeline = True+ pipeline._resumable_source_manager = None+ pipeline._delta_table_ref = MagicMock(is_first_sync=True, reset_table=AsyncMock())+ resource = pipeline._resource+ resource.items = MagicMock(return_value=iter([]))+ pipeline._sinks = MagicMock(clear=AsyncMock(), cdp_producer=MagicMock(should_run=AsyncMock(return_value=False)))+ table = pa.table({"created_at": pa.array([self._NEWEST, oldest_row], pa.timestamp("us", tz="UTC"))})++ with ExitStack() as stack:+ for name in (+ "reset_rows_synced_if_needed",+ "setup_row_tracking_with_billing_check",+ "persist_primary_keys",+ "handle_corrupted_delta_log",+ ):+ stack.enter_context(patch(f"{_PIPELINE}.{name}", new_callable=AsyncMock))+ stack.enter_context(patch(f"{_PIPELINE}.validate_incremental_sync"))+ stack.enter_context(patch(f"{_PIPELINE}.update_sync_type_config_keys"))+ stack.enter_context(patch(f"{_PIPELINE}.activity")).in_activity.return_value = False+ stack.enter_context(+ patch.object(+ schema_models, "update_sync_type_config_keys", side_effect=_in_memory_config_update(schema)+ )+ )+ await pipeline.run()+ run_uuid = cast(str, pipeline._run_uuid)+ await update_incremental_field_values(+ schema,+ table,+ resource,+ pipeline._last_incremental_field_value,+ pipeline._earliest_incremental_field_value,+ _make_logger(),+ staging_run_uuid=run_uuid,+ )+ assert schema.promote_staged_incremental_values(run_uuid)++ assert schema.sync_type_config["incremental_field_last_value"] == self._NEWEST.isoformat()+ assert schema.sync_type_config["incremental_field_earliest_value"] == oldest_row.isoformat()+class TestZeroBatchRunStampsTheFullRunMarker:@pytest.mark.asynciodiff --git a/products/warehouse_sources/backend/tests/test_models.py b/products/warehouse_sources/backend/tests/test_models.pyindex 6b70c1e37351..a4ee397ce22f 100644--- a/products/warehouse_sources/backend/tests/test_models.py+++ b/products/warehouse_sources/backend/tests/test_models.py@@ -1452,6 +1452,30 @@ class TestStagedIncrementalCursor:assert schema.promote_staged_incremental_values("run-1") is Trueassert schema.sync_type_config["incremental_field_earliest_value"] == 5+ @parameterized.expand(+ [+ # A run that read only above the stored watermark saw none of the older rows of the table.+ # Its oldest row as the earliest value makes the next run read the whole table below it again.+ ("a_forward_only_run_sets_no_earliest_value", {"incremental_field_last_value": 90}, None),+ ("the_first_run_sets_the_earliest_value", {}, 5),+ (+ "a_backfill_below_the_stored_earliest_value_advances_it",+ {"incremental_field_last_value": 90, "incremental_field_earliest_value": 40},+ 5,+ ),+ ]+ )+ def test_promote_sets_the_earliest_value_unless_the_watermark_has_none(+ self, _name: str, stored: dict[str, Any], expected_earliest: int | None+ ) -> None:+ schema = self._make_schema(+ **stored, incremental_staged={"run_uuid": "run-1", "last_value": 99, "earliest_value": 5}+ )+ with self._staged_in_memory(schema):+ assert schema.promote_staged_incremental_values("run-1") is True+ assert schema.sync_type_config["incremental_field_last_value"] == 99+ assert schema.sync_type_config.get("incremental_field_earliest_value") == expected_earliest+@parameterized.expand([("older_epoch_keeps_iso", IncrementalFieldType.DateTime, "2026-06-14T00:00:00+00:00", 1767225600),