Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-warehouse): keep earliest value after a full reload of a desc source (#125560)

Daniel RC7cdbf4c0ebf5
diff --git a/products/warehouse_sources/backend/models/external_data_schema.py b/products/warehouse_sources/backend/models/external_data_schema.py
index a41c1427fb22..4bee659b1784 100644
--- a/products/warehouse_sources/backend/models/external_data_schema.py
+++ b/products/warehouse_sources/backend/models/external_data_schema.py
@@ -1232,11 +1232,18 @@ class ExternalDataSchema( # nosemgrep: semgrep.rules.security.prefer-uuid7-djan
return
found = True
field_type = config.get("incremental_field_type")
+ # A stored watermark with no earliest value means the table already holds every older row.
+ # A run from that watermark read only newer rows, so its oldest row must not become the
+ # earliest value: the next run would read every row below it again.
+ watermark_without_earliest = (
+ config.get("incremental_field_last_value") is not None
+ and config.get("incremental_field_earliest_value") is None
+ )
if "last_value" in staged:
_advance_promoted_cursor(
config, "incremental_field_last_value", staged["last_value"], "last", field_type
)
- if "earliest_value" in staged:
+ if "earliest_value" in staged and not watermark_without_earliest:
_advance_promoted_cursor(
config, "incremental_field_earliest_value", staged["earliest_value"], "earliest", field_type
)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.py
index c33ebbf4097f..453834bd82a1 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.py
@@ -931,6 +931,11 @@ class PipelineV3(Generic[ResumableData]):
self._logger,
webhook_only=self._resource.webhook_only,
)
+ # A reset drops the earliest value of the table it deletes. No reloaded row is older than
+ # that value, so a run that kept it would never stage an earliest value of its own.
+ self._earliest_incremental_field_value = process_incremental_value(
+ self._schema.incremental_field_earliest_value, self._schema.incremental_field_type
+ )
is_fresh_sync = self._delta_table_ref.is_first_sync or self._schema.table is None
if not is_fresh_sync and self._attempt > 1 and not should_resume:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.py
index 9237242f11d0..bbc55b25c6ce 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.py
@@ -1643,6 +1643,56 @@ class TestDescWatermarkAcrossResumedAttempts:
assert schema.sync_type_config["incremental_field_last_value"] == self._NEWEST.isoformat()
+ @pytest.mark.asyncio
+ async def test_a_reset_run_promotes_the_oldest_row_it_loaded_as_the_earliest_value(self) -> None:
+ # The deleted table held the same rows, so no row of the reload is older than its stored earliest value.
+ oldest_row = self._NEWEST - timedelta(hours=5)
+ schema = self._schema()
+ schema.sync_type_config.update(
+ incremental_field_last_value=self._NEWEST.isoformat(),
+ incremental_field_earliest_value=oldest_row.isoformat(),
+ )
+ pipeline = self._attempt(schema, 1)
+ pipeline._reset_pipeline = True
+ pipeline._resumable_source_manager = None
+ pipeline._delta_table_ref = MagicMock(is_first_sync=True, reset_table=AsyncMock())
+ resource = pipeline._resource
+ resource.items = MagicMock(return_value=iter([]))
+ pipeline._sinks = MagicMock(clear=AsyncMock(), cdp_producer=MagicMock(should_run=AsyncMock(return_value=False)))
+ table = pa.table({"created_at": pa.array([self._NEWEST, oldest_row], pa.timestamp("us", tz="UTC"))})
+
+ with ExitStack() as stack:
+ for name in (
+ "reset_rows_synced_if_needed",
+ "setup_row_tracking_with_billing_check",
+ "persist_primary_keys",
+ "handle_corrupted_delta_log",
+ ):
+ stack.enter_context(patch(f"{_PIPELINE}.{name}", new_callable=AsyncMock))
+ stack.enter_context(patch(f"{_PIPELINE}.validate_incremental_sync"))
+ stack.enter_context(patch(f"{_PIPELINE}.update_sync_type_config_keys"))
+ stack.enter_context(patch(f"{_PIPELINE}.activity")).in_activity.return_value = False
+ stack.enter_context(
+ patch.object(
+ schema_models, "update_sync_type_config_keys", side_effect=_in_memory_config_update(schema)
+ )
+ )
+ await pipeline.run()
+ run_uuid = cast(str, pipeline._run_uuid)
+ await update_incremental_field_values(
+ schema,
+ table,
+ resource,
+ pipeline._last_incremental_field_value,
+ pipeline._earliest_incremental_field_value,
+ _make_logger(),
+ staging_run_uuid=run_uuid,
+ )
+ assert schema.promote_staged_incremental_values(run_uuid)
+
+ assert schema.sync_type_config["incremental_field_last_value"] == self._NEWEST.isoformat()
+ assert schema.sync_type_config["incremental_field_earliest_value"] == oldest_row.isoformat()
+
class TestZeroBatchRunStampsTheFullRunMarker:
@pytest.mark.asyncio
diff --git a/products/warehouse_sources/backend/tests/test_models.py b/products/warehouse_sources/backend/tests/test_models.py
index 6b70c1e37351..a4ee397ce22f 100644
--- a/products/warehouse_sources/backend/tests/test_models.py
+++ b/products/warehouse_sources/backend/tests/test_models.py
@@ -1452,6 +1452,30 @@ class TestStagedIncrementalCursor:
assert schema.promote_staged_incremental_values("run-1") is True
assert schema.sync_type_config["incremental_field_earliest_value"] == 5
+ @parameterized.expand(
+ [
+ # A run that read only above the stored watermark saw none of the older rows of the table.
+ # Its oldest row as the earliest value makes the next run read the whole table below it again.
+ ("a_forward_only_run_sets_no_earliest_value", {"incremental_field_last_value": 90}, None),
+ ("the_first_run_sets_the_earliest_value", {}, 5),
+ (
+ "a_backfill_below_the_stored_earliest_value_advances_it",
+ {"incremental_field_last_value": 90, "incremental_field_earliest_value": 40},
+ 5,
+ ),
+ ]
+ )
+ def test_promote_sets_the_earliest_value_unless_the_watermark_has_none(
+ self, _name: str, stored: dict[str, Any], expected_earliest: int | None
+ ) -> None:
+ schema = self._make_schema(
+ **stored, incremental_staged={"run_uuid": "run-1", "last_value": 99, "earliest_value": 5}
+ )
+ with self._staged_in_memory(schema):
+ assert schema.promote_staged_incremental_values("run-1") is True
+ assert schema.sync_type_config["incremental_field_last_value"] == 99
+ assert schema.sync_type_config.get("incremental_field_earliest_value") == expected_earliest
+
@parameterized.expand(
[
("older_epoch_keeps_iso", IncrementalFieldType.DateTime, "2026-06-14T00:00:00+00:00", 1767225600),