:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): promote the watermark over loaded batch prefixes (#119740)
Tom Owers986d093e5684
diff --git a/posthog/settings/temporal.py b/posthog/settings/temporal.pyindex 3d2bcc13fd4f..87b9472c2abd 100644--- a/posthog/settings/temporal.py+++ b/posthog/settings/temporal.py@@ -301,6 +301,14 @@ DATA_WAREHOUSE_IMPORT_FREE_HANDOFFS_ENABLED: bool = get_from_env(DATA_WAREHOUSE_IMPORT_WATERMARK_CARRY_OVER_ENABLED: bool = get_from_env("DATA_WAREHOUSE_IMPORT_WATERMARK_CARRY_OVER_ENABLED", False, type_cast=str_to_bool)+# An incremental merge import moves its watermark each time a contiguous prefix of its batches is+# loaded, so a run that fails keeps the progress of the batches that loaded. The import pods record+# the per-batch values and the load pods promote them, so set it on both. Enable it only when every+# source that says sort_mode="asc" reads its rows in that order, because the next run reads only+# the rows above the promoted value.+DATA_WAREHOUSE_IMPORT_PREFIX_WATERMARK_ENABLED: bool = get_from_env(+ "DATA_WAREHOUSE_IMPORT_PREFIX_WATERMARK_ENABLED", False, type_cast=str_to_bool+)# At a worker shutdown the import pipeline stops waiting for a source that neither yields an item# nor reaches a safe point, and hands the run to another worker. It applies only to runs whose# hand-offs are free (DATA_WAREHOUSE_IMPORT_FREE_HANDOFFS_ENABLED), because preemption makesdiff --git a/products/warehouse_sources/backend/models/external_data_schema.py b/products/warehouse_sources/backend/models/external_data_schema.pyindex e295586ab8ad..78807efa6ae3 100644--- a/products/warehouse_sources/backend/models/external_data_schema.py+++ b/products/warehouse_sources/backend/models/external_data_schema.py@@ -229,6 +229,11 @@ STAGED_RESUME_OWNER_KEY = "resume_owner_run_uuid"# drops the key too.APPEND_RUN_MARKER_KEY = "append_run_in_progress"+# The key for the newest watermark that a loaded prefix of batches moved, with the job that loaded+# them and the rows that job had synced through the prefix. A later attempt of the same job reads+# only the rows above that watermark, so it keeps that row count instead of counting from zero.+PREFIX_PROMOTION_KEY = "incremental_prefix_promotion"+class ExternalDataSchemaQuerySet(models.QuerySet["ExternalDataSchema"]):def update(self, **kwargs: Any) -> int:@@ -1241,6 +1246,50 @@ class ExternalDataSchema( # nosemgrep: semgrep.rules.security.prefer-uuid7-djanself.sync_type_config = update_sync_type_config_keys(self.id, self.team_id, mutate=mutate)return found+ def promote_loaded_prefix_value(+ self, *, run_uuid: str, job_id: str, field: str, value: Any, job_rows_synced: int+ ) -> bool:+ """Move `incremental_field_last_value` up to the value that a loaded prefix of `run_uuid` covers.++ Returns True when the watermark moved. It moves only forward, and only while the run still+ has a staged cursor at or above `value` for the same incremental field. A reset drops the+ staged cursors, so a run that loads late never moves the watermark of the table that replaced+ its own. A pair of values that cannot be compared keeps the current watermark.+ """+ moved = False++ def mutate(config: dict[str, Any]) -> None:+ nonlocal moved+ if config.get("incremental_field") != field:+ return+ staged = _staged_cursor_of_run(config, run_uuid)+ if staged is None or "last_value" not in staged:+ return+ field_type = config.get("incremental_field_type")+ staged_comparison = _compare_incremental_values(staged["last_value"], value, field_type)+ if staged_comparison is None or staged_comparison < 0:+ return+ current = config.get("incremental_field_last_value")+ if current is not None:+ comparison = _compare_incremental_values(current, value, field_type)+ if comparison is None or comparison >= 0:+ return+ config["incremental_field_last_value"] = value+ config[PREFIX_PROMOTION_KEY] = {+ "job_id": job_id,+ "run_uuid": run_uuid,+ "value": value,+ "rows_synced": job_rows_synced,+ }+ moved = True++ self.sync_type_config = update_sync_type_config_keys(self.id, self.team_id, mutate=mutate)+ return moved++ def serialize_incremental_value(self, value: Any) -> Any:+ """Encode a cursor value the way the staged and the promoted watermark values store it."""+ return self._serialize_incremental_value(value)+def _serialize_incremental_value(self, value: Any) -> Any:incremental_field_type = self.sync_type_config.get("incremental_field_type")if "numpy" in sys.modules:@@ -1321,6 +1370,7 @@ class ExternalDataSchema( # nosemgrep: semgrep.rules.security.prefer-uuid7-djan"incremental_field_earliest_value","incremental_staged","incremental_staged_pending",+ PREFIX_PROMOTION_KEY,APPEND_RUN_MARKER_KEY,"partitioning_enabled","partition_size",@@ -1538,6 +1588,31 @@ def _staged_attempts_of_run(config: dict[str, Any], workflow_run_id: str) -> lisreturn attempts+def _staged_cursor_of_run(config: dict[str, Any], run_uuid: str) -> dict[str, Any] | None:+ """The staged cursor of `run_uuid`, live or parked, without removing it."""+ for staged in (config.get("incremental_staged") or {}, *config.get("incremental_staged_pending", [])):+ if staged.get("run_uuid") == run_uuid:+ return staged+ return None+++def prefix_promotion_rows_synced(config: dict[str, Any], job_id: str) -> int | None:+ """The rows `job_id` synced through the loaded prefix that set the stored watermark, or None.++ None when the stored watermark did not come from a loaded prefix of this job, so a new attempt+ of the job reads the rows that the count would cover again.+ """+ promotion = config.get(PREFIX_PROMOTION_KEY)+ if not isinstance(promotion, dict) or promotion.get("job_id") != job_id:+ return None+ if "value" not in promotion or promotion["value"] != config.get("incremental_field_last_value"):+ return None+ rows_synced = promotion.get("rows_synced")+ if isinstance(rows_synced, bool) or not isinstance(rows_synced, int) or rows_synced < 0:+ return None+ return rows_synced++def _drop_parked_staged_cursor(config: dict[str, Any], run_uuid: str) -> dict[str, Any] | None:"""Remove the parked cursor of `run_uuid` and return it, or None when the run has none."""pending = config.get("incremental_staged_pending", [])diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/README.md b/products/warehouse_sources/backend/temporal/data_imports/pipelines/README.mdindex a9257e0f3218..5548bceab5af 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/README.md+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/README.md@@ -35,6 +35,18 @@ A later attempt of the same workflow run that also reads from the start loads thAn attempt that continues from a cursor merges, because it can read a row again that an earlier attempt already queued.The append keeps one row for each key in a batch, not in the run: a key that the source holds in two batches stays twice until a later run merges that key.+## Incremental runs that fail part way++The final batch of a run promotes the run's staged watermark, so a run that fails after many loaded batches would make the next run read all of them again.+With `DATA_WAREHOUSE_IMPORT_PREFIX_WATERMARK_ENABLED`, the watermark also moves each time batches `0..k` of a run are all loaded.++- Only an incremental merge on a primary key qualifies, from a source that says `sort_mode="asc"`. A first sync, a reset, a CDC table, a laned source, a resumable source, an attempt that continues an earlier one and a table with an active chunk sink (a warehouse-triggered workflow, a HogFunction or a person-property mapping) do not qualify. A retry clears the sink's staged files, so it would never stage the rows that it skips.+- A run with an external destination does not qualify. A file destination publishes the batches of a run only when the run completes, so a loaded warehouse prefix does not prove that the destination has the rows.+- The promoted value is safe only when the source really reads in ascending order, because the next run reads only the rows above it. The check for a batch out of order cannot protect rows that the run did not read yet. Enable the setting only where every source that says `sort_mode="asc"` proves that order.+- Each batch records in its queue metadata (`loaded_prefix_cursor`) the largest value below the newest value read so far. Rows that hold the newest value can still be in the next batch, so the next run reads them again and merges them. A batch out of ascending order stops the recording for the rest of the run.+- After each write, the loader checks that every earlier batch of the run has `succeeded` (`get_loaded_prefix_sync` in `jobs_db.py`). Then it promotes the value under the schema row lock. It never moves the watermark down, and it does nothing when the run has no staged cursor, because a reset deleted it.+- A later attempt of the same job reads only the rows above the promoted value, so it keeps the job's `rows_synced` through that prefix (`incremental_prefix_promotion`) instead of counting from zero.+## DestinationsA schema syncs to a set of destinations, of which the PostHog warehouse is one rather than a special case.diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/extract.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/extract.pyindex cf65c8b8edf1..9a01bfc42a5e 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/extract.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/extract.py@@ -351,15 +351,20 @@ async def handle_non_retryable_error(raise NonRetryableException() from error-async def reset_rows_synced_if_needed(job: "ExternalDataJob", should_resume: bool) -> None:+async def reset_rows_synced_if_needed(+ job: "ExternalDataJob", should_resume: bool, kept_rows_synced: int | None = None+) -> None:# Reset the rows_synced count - this may not be 0 if the job restarted due to a heartbeat timeout.## The incremental cursor is staged and only promoted on completion, so a retried attempt# re-extracts the whole window from batch 0, and keeping the previous attempt's count# double-counts every re-read row — `rows_synced` feeds billed usage via `Sum("rows_synced")`# in usage reports. A resumed attempt picks up the earlier attempt's staged batches instead.+ #+ # `kept_rows_synced` is the count through the loaded prefix that moved the watermark this+ # attempt reads from. This attempt does not read those rows again, so they stay counted.if job.rows_synced is not None and job.rows_synced != 0 and not should_resume:- job.rows_synced = 0+ job.rows_synced = min(job.rows_synced, kept_rows_synced or 0)await database_sync_to_async_pool(job.save)(update_fields=["rows_synced", "updated_at"])diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/test/test_extract.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/test/test_extract.pyindex 517912096550..6a88341d4e08 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/test/test_extract.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/test/test_extract.py@@ -777,29 +777,34 @@ class TestResetRowsSyncedIfNeeded:)@pytest.mark.parametrize(- "_name,should_resume,expect_reset",+ "_name,should_resume,kept_rows_synced,expected_rows_synced",[# A retry re-extracts the whole window from batch 0, so a leftover count from the# previous attempt would double-count every re-read row — and rows_synced feeds# billed usage. This is the regression case.- ("retry_resets", False, True),+ ("retry_resets", False, None, 0),# A resumable source picks up the previous attempt's staged batches, so its rows stay counted.- ("resumable_source_keeps_count", True, False),+ ("resumable_source_keeps_count", True, None, 1234),+ # The retry reads only the rows above a loaded prefix of the earlier attempt, so the+ # rows through that prefix stay billed.+ ("retry_after_a_loaded_prefix_keeps_its_rows", False, 1000, 1000),+ ("kept_count_never_raises_the_count", False, 5000, 1234),],)def test_reset_conditions(self,_name: str,should_resume: bool,- expect_reset: bool,+ kept_rows_synced: int | None,+ expected_rows_synced: int,team,) -> None:job = self._job_with_leftover_count(team)- async_to_sync(reset_rows_synced_if_needed)(job, should_resume)+ async_to_sync(reset_rows_synced_if_needed)(job, should_resume, kept_rows_synced=kept_rows_synced)job.refresh_from_db()- assert job.rows_synced == (0 if expect_reset else 1234)+ assert job.rows_synced == expected_rows_synced# transaction=True: the webhook-first branch clears the reset flag via update_sync_type_config_keys,diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/lanes.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/lanes.pyindex 21179889f12d..adddbb34ea5e 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/lanes.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/lanes.py@@ -94,6 +94,7 @@ class LanedPipelineV3(PipelineV3[ResumableData]):resumed_incremental_run_uuid: str | None = None,resumed_incremental_value: Any = None,preemption: PreemptionConfig | None = None,+ kept_rows_synced: int | None = None,) -> None:if not source_response.lanes:raise ValueError(f"{source_response.name} declares no lanes; run it on PipelineV3")@@ -112,6 +113,7 @@ class LanedPipelineV3(PipelineV3[ResumableData]):resumed_incremental_run_uuid=resumed_incremental_run_uuid,resumed_incremental_value=resumed_incremental_value,preemption=preemption,+ kept_rows_synced=kept_rows_synced,)# The base built the first lane; it shares the base's batch list so the two never disagree.diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/metrics.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/metrics.pyindex b361680ca6c5..ee1ab7eae587 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/metrics.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/metrics.py@@ -145,3 +145,12 @@ DELTALITE_GOVERNOR_RESERVED_MB = Gauge("warehouse_load_deltalite_governor_reserved_mb","Total memory (MB) currently reserved by in-flight deltalite upserts on this process",)++# One per run in a write that records a loaded-prefix watermark and does not complete in that write.+# outcome - promoted | kept (the guards kept the current watermark) | incomplete (an earlier+# batch of the run is not loaded) | error+LOADED_PREFIX_PROMOTION_TOTAL = Counter(+ "warehouse_load_loaded_prefix_promotion_total",+ "Watermark promotions after a loaded prefix of an incremental run, by outcome",+ labelnames=["outcome"],+)diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/processor.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/processor.pyindex 894839edfcde..b94d0ff6e6a2 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/processor.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/processor.py@@ -97,6 +97,7 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.pipelineDELTA_WRITE_DURATION_SECONDS,IDEMPOTENCY_HIT_TOTAL,INCOMPLETE_RUN_COMPLETIONS_REFUSED_TOTAL,+ LOADED_PREFIX_PROMOTION_TOTAL,PARQUET_READ_DURATION_SECONDS,)from products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.load.table_handles import (@@ -874,6 +875,78 @@ def _promote_staged_cursor(export_signal: ExportSignalMessage) -> None:)+def _promote_loaded_prefixes(+ signals: list[ExportSignalMessage], finals: list[ExportSignalMessage], schema: ExternalDataSchema+) -> None:+ """Move the watermark over the loaded prefix of each run in this write that does not complete here.++ A run that completes here promotes its full staged cursor in its completion instead. A failed+ promotion only leaves the watermark lower, so it never fails the batch, which already landed.+ """+ if not settings.DATA_WAREHOUSE_IMPORT_PREFIX_WATERMARK_ENABLED:+ return+ completed = {signal.run_uuid for signal in finals}+ signals_by_run: dict[str, list[ExportSignalMessage]] = {}+ for signal in signals:+ if signal.sync_type == "incremental" and signal.cdc_write_mode is None and not signal.is_first_ever_sync:+ signals_by_run.setdefault(signal.run_uuid, []).append(signal)+ for run_uuid, run_signals in signals_by_run.items():+ if run_uuid in completed:+ continue+ try:+ outcome = _promote_loaded_prefix(run_signals, schema)+ except Exception as e:+ outcome = "error"+ capture_exception(e)+ logger.warning(+ "loaded_prefix_promotion_failed",+ team_id=run_signals[0].team_id,+ external_data_schema_id=run_signals[0].schema_id,+ run_uuid=run_uuid,+ exc_info=True,+ )+ if outcome is not None:+ LOADED_PREFIX_PROMOTION_TOTAL.labels(outcome=outcome).inc()+++def _promote_loaded_prefix(run_signals: list[ExportSignalMessage], schema: ExternalDataSchema) -> str | None:+ """Promote the watermark that the last batch of this write recorded, when every earlier batch is loaded.++ Returns the metric outcome, or None for a batch that recorded no watermark.+ """+ head = run_signals[0]+ prefix = BatchQueue.get_loaded_prefix_sync(+ settings.WAREHOUSE_SOURCES_DATABASE_URL,+ team_id=head.team_id,+ schema_id=head.schema_id,+ run_uuid=head.run_uuid,+ first_batch_index=min(signal.batch_index for signal in run_signals),+ last_batch_index=max(signal.batch_index for signal in run_signals),+ )+ cursor = prefix.cursor+ if not isinstance(cursor, dict):+ return None+ field, value, job_rows_synced = cursor.get("field"), cursor.get("value"), cursor.get("job_rows_synced")+ if not isinstance(field, str) or value is None or not isinstance(job_rows_synced, int):+ return None+ if not prefix.is_complete:+ return "incomplete"+ moved = schema.promote_loaded_prefix_value(+ run_uuid=head.run_uuid, job_id=head.job_id, field=field, value=value, job_rows_synced=job_rows_synced+ )+ if not moved:+ return "kept"+ logger.info(+ "loaded_prefix_cursor_promoted",+ team_id=head.team_id,+ external_data_schema_id=head.schema_id,+ external_data_job_id=head.job_id,+ run_uuid=head.run_uuid,+ last_batch_index=max(signal.batch_index for signal in run_signals),+ )+ return "promoted"++def _mark_job_failed(export_signal: ExportSignalMessage, error: Exception) -> None:# Short-circuit if the job is already FAILED: redelivered DLQ'd messages# (the retry state stays in Redis until its 72h TTL) would otherwise spam@@ -1562,6 +1635,7 @@ def _process_message_reported(finals = [export_signal] if export_signal.is_final_batch else []if not finals and _rebuild_compaction_due(export_signal, schema, members):_compact_during_rebuild(delta_table_ref, verify_ownership, timer)+ _promote_loaded_prefixes(constituents if constituents is not None else [export_signal], finals, schema)for run_signal in finals:run_job = job if run_signal.job_id == export_signal.job_id else _load_job(run_signal.job_id)run_schema = run_job.schemadiff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/test_processor.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/test_processor.pyindex 258d0f773a92..7d5ef79f7edd 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/test_processor.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/test_processor.py@@ -2,7 +2,7 @@ import tempfilefrom collections.abc import Iteratorfrom contextlib import ExitStack, contextmanagerfrom pathlib import Path-from typing import Any+from typing import Any, NamedTupleimport pytestfrom unittest.mock import ANY, AsyncMock, MagicMock, patch@@ -44,6 +44,7 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_file_count_after_write,_get_write_type,_mark_job_completed,+ _promote_loaded_prefixes,_promote_staged_cursor,_read_existing_rows_by_first_pk,_rebuild_compaction_due,@@ -60,7 +61,7 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.pipelinefrom products.warehouse_sources.backend.temporal.data_imports.pipelines.test_mocks import mock_delta_tablefrom products.warehouse_sources.backend.types import ExternalDataSourceTypefrom products.warehouse_sources_queue.backend.core.batch_consumer import CoalescingDeclined-from products.warehouse_sources_queue.backend.core.jobs_db import UnfinishedEarlierBatches+from products.warehouse_sources_queue.backend.core.jobs_db import LoadedPrefix, UnfinishedEarlierBatches_COMPLETE_RUN = UnfinishedEarlierBatches(count=0, first_batch_index=None, states=())@@ -772,9 +773,10 @@ class TestStagedCursorWaitsForTheFinalBatch:# A descending source stages the newest value of its run from the first batch. Promoting it on a# batch before the final one moves the watermark above rows that are not loaded yet.- @parameterized.expand([("mid_run_batch", False, 0), ("final_batch", True, 1)])+ @parameterized.expand([("mid_run_batch", False, 0, 1), ("final_batch", True, 1, 0)])+ @override_settings(DATA_WAREHOUSE_IMPORT_PREFIX_WATERMARK_ENABLED=True)def test_a_staged_cursor_is_promoted_only_by_the_final_batch(- self, _name: str, is_final_batch: bool, promotions: int+ self, _name: str, is_final_batch: bool, promotions: int, prefix_promotions: int) -> None:delta_table = MagicMock()delta_table.schema.return_value = pa.schema([pa.field("id", pa.int64())])@@ -803,6 +805,12 @@ class TestStagedCursorWaitsForTheFinalBatch:mock_update = stack.enter_context(patch(f"{_PROCESSOR}.update_external_job_status"))mock_update.return_value.status = mock_job_cls.Status.COMPLETEDmock_schema_objects = stack.enter_context(patch(f"{_PROCESSOR}.ExternalDataSchema.objects"))+ stack.enter_context(+ patch(+ f"{_PROCESSOR}.BatchQueue.get_loaded_prefix_sync",+ return_value=LoadedPrefix(is_complete=True, cursor=_PREFIX_CURSOR),+ )+ )process_message(_message(is_final_batch=is_final_batch, run_uuid="wfrun-1-a2"))@@ -810,6 +818,95 @@ class TestStagedCursorWaitsForTheFinalBatch:assert promote.call_count == promotionsif promotions:promote.assert_called_with("wfrun-1-a2", merge_source_cursors=ANY)+ # The final batch promotes the full staged value, so only a batch before it promotes a prefix.+ promote_prefix = mock_job_cls.objects.prefetch_related.return_value.get.return_value.schema+ assert promote_prefix.promote_loaded_prefix_value.call_count == prefix_promotions+++_PREFIX_CURSOR = {"field": "id", "value": 30, "job_rows_synced": 7}+++class _PromoteMocks(NamedTuple):+ get_prefix: MagicMock+ promote: MagicMock+++class TestPromoteLoadedPrefixes:+ @staticmethod+ def _signals(*indexes: int, **overrides: Any) -> list[ExportSignalMessage]:+ return [ExportSignalMessage.from_dict(_message(batch_index=index, **overrides)) for index in indexes]++ @staticmethod+ def _promote(+ signals: list[ExportSignalMessage],+ prefix: LoadedPrefix | Exception,+ *,+ finals: list[ExportSignalMessage] | None = None,+ enabled: bool = True,+ ) -> _PromoteMocks:+ schema = MagicMock()+ with (+ override_settings(DATA_WAREHOUSE_IMPORT_PREFIX_WATERMARK_ENABLED=enabled),+ patch(f"{_PROCESSOR}.BatchQueue.get_loaded_prefix_sync", side_effect=[prefix]) as get_prefix,+ patch(f"{_PROCESSOR}.capture_exception"),+ ):+ _promote_loaded_prefixes(signals, finals or [], schema)+ return _PromoteMocks(get_prefix=get_prefix, promote=schema.promote_loaded_prefix_value)++ def test_a_loaded_prefix_promotes_the_value_of_the_last_batch_of_the_write(self) -> None:+ get_prefix, promote = self._promote(+ self._signals(3, 4, 5), LoadedPrefix(is_complete=True, cursor=_PREFIX_CURSOR)+ )++ assert get_prefix.call_args.kwargs == {+ "team_id": 1,+ "schema_id": "schema-1",+ "run_uuid": "run-1",+ "first_batch_index": 3,+ "last_batch_index": 5,+ }+ promote.assert_called_once_with(run_uuid="run-1", job_id="job-1", field="id", value=30, job_rows_synced=7)++ @parameterized.expand(+ [+ # An earlier batch is missing, aged out, failed or waits for a retry.+ ("gap_before_the_write", LoadedPrefix(is_complete=False, cursor=_PREFIX_CURSOR)),+ # A pod without the setting, or a run that cannot promote, queued the batch.+ ("batch_has_no_cursor", LoadedPrefix(is_complete=True, cursor=None)),+ ("cursor_in_an_unknown_shape", LoadedPrefix(is_complete=True, cursor={"value": 30})),+ # A queue read that fails must not fail a batch that already landed.+ ("queue_read_fails", RuntimeError("queue down")),+ ]+ )+ def test_no_promotion(self, _name: str, prefix: LoadedPrefix | Exception) -> None:+ _get_prefix, promote = self._promote(self._signals(2), prefix)++ promote.assert_not_called()++ @parameterized.expand(+ [+ ("setting_off", {}, False, False),+ ("final_batch_promotes_the_full_value", {}, True, True),+ ("append", {"sync_type": "append"}, False, True),+ ("full_refresh", {"sync_type": "full_refresh"}, False, True),+ ("cdc", {"cdc_write_mode": "scd2_append"}, False, True),+ ("first_ever_sync", {"is_first_ever_sync": True}, False, True),+ ]+ )+ def test_a_write_that_cannot_promote_a_prefix_reads_nothing(+ self, _name: str, overrides: dict[str, Any], completes_here: bool, enabled: bool+ ) -> None:+ signals = self._signals(2, **overrides)++ get_prefix, promote = self._promote(+ signals,+ LoadedPrefix(is_complete=True, cursor=_PREFIX_CURSOR),+ finals=signals if completes_here else None,+ enabled=enabled,+ )++ get_prefix.assert_not_called()+ promote.assert_not_called()def _set_messages(count: int, *, sync_type: str = "full_refresh", final: bool = True) -> list[dict[str, Any]]:diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.pyindex c9d576f867a8..5f2d977dfe7a 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.py@@ -1,10 +1,13 @@import timeimport asyncio+import decimalimport datetimeimport contextlibfrom collections.abc import AsyncGenerator, AsyncIteratorfrom typing import TYPE_CHECKING, Any, Generic+from django.conf import settings+import pyarrow as paimport posthoganalyticsfrom structlog.types import FilteringBoundLogger@@ -186,6 +189,10 @@ class PipelineV3(Generic[ResumableData]):_resumed_incremental_run_uuid: str | None = None_sent_resumed_run_finalization: bool = False_writes_staged_repartition_scheme: bool = False+ # Set only for a run whose batches record the watermark that each loaded prefix covers.+ _prefix_checkpoint: IncrementalHandoffCheckpoint | None = None+ _job_rows_synced_through_batch: int = 0+ _kept_rows_synced: int | None = Nonedef __init__(self,@@ -202,8 +209,10 @@ class PipelineV3(Generic[ResumableData]):resumed_incremental_run_uuid: str | None = None,resumed_incremental_value: Any = None,preemption: PreemptionConfig | None = None,+ kept_rows_synced: int | None = None,) -> None:self._resource = source_response+ self._kept_rows_synced = kept_rows_syncedself._source_cursor_manager = source_cursor_managerself._resource_name = source_response.name@@ -274,6 +283,22 @@ class PipelineV3(Generic[ResumableData]):if incremental_checkpoints_allowed and self._tracks_handoff_checkpoint(source_response, reset_pipeline):self._handoff_checkpoint = IncrementalHandoffCheckpoint(resumed_incremental_value)self._batch_range_reader = IncrementalBatchRangeReader(self._schema)+ # A continuing attempt or a source with its own cursor does not read from the watermark, and+ # a first sync appends, so the loaded batches of such a run say nothing about the watermark.+ # A file destination publishes the batches of a run only when the run completes, so a loaded+ # warehouse prefix does not prove that an external destination has the rows.+ if (+ settings.DATA_WAREHOUSE_IMPORT_PREFIX_WATERMARK_ENABLED+ and incremental_checkpoints_allowed+ and self._tracks_handoff_checkpoint(source_response, reset_pipeline)+ and not is_resume+ and source_cursor_manager is None+ and self._resource.primary_keys+ and not is_first_ever_sync+ and not models.external_destination_ids+ ):+ self._prefix_checkpoint = IncrementalHandoffCheckpoint()+ self._batch_range_reader = IncrementalBatchRangeReader(self._schema)# A queued repartition target is the scheme of the table when this attempt deletes the table# and loads it again. The reset then stages that target (see `handle_reset_or_full_refresh`),@@ -445,8 +470,11 @@ class PipelineV3(Generic[ResumableData]):# `hold_batch` only inserts the batch this attempt is already holding, not the one it is# about to hold. A row from this attempt exists in the queue only once this call flushes one.flushes_a_held_batch = self._pg_producer.has_held_batch+ loaded_prefix_cursor = self._loaded_prefix_cursor(pa_table)try:- self._pg_producer.hold_batch(batch_result, cumulative_row_count=row_count)+ self._pg_producer.hold_batch(+ batch_result, cumulative_row_count=row_count, loaded_prefix_cursor=loaded_prefix_cursor+ )except Exception:# The insert that failed belongs to the batch before this one. The checkpoint already# counts that batch, so no later value of it is safe. The value staged earlier stays valid.@@ -458,6 +486,42 @@ class PipelineV3(Generic[ResumableData]):self._queued_own_batch = Truereturn pa_table.num_rows+ def _loaded_prefix_cursor(self, pa_table: pa.Table) -> dict[str, Any] | None:+ """The watermark that this batch and every earlier batch of the run cover once loaded.++ The value is below the newest value seen so far, because rows that hold the newest value can+ still be in the next batch. A batch out of ascending order voids it for the rest of the run.+ """+ self._job_rows_synced_through_batch += pa_table.num_rows+ checkpoint = self._prefix_checkpoint+ if checkpoint is None or not self._schema.incremental_field:+ return None+ if pa_table.num_rows > 0:+ checkpoint.observe(self._batch_range_reader.read(pa_table))+ value = checkpoint.resume_value+ if value is None:+ return None+ serialized = self._schema.serialize_incremental_value(value)+ if isinstance(value, decimal.Decimal) and isinstance(serialized, int | float) and serialized > value:+ # Numeric values are stored as integers, so a negative decimal rounds up, past rows the+ # run has not read. Record nothing rather than a value above the loaded rows.+ return None+ return {+ "field": self._schema.incremental_field,+ "value": serialized,+ "job_rows_synced": self._job_rows_synced_through_batch,+ }++ async def _drop_prefix_checkpoint_for_active_sinks(self) -> None:+ """Stop recording prefix values for a table that has an active chunk sink.++ A retry after a prefix promotion skips rows that this job already staged for a sink, and+ clearing the sinks at the start of an attempt deletes those staged files. The skipped rows+ would never reach the sink, so such a table keeps its watermark until the run completes.+ """+ if self._prefix_checkpoint is not None and any([await sink.should_run() for sink in self._sinks.all]):+ self._prefix_checkpoint = None+def _total_batches(self) -> int:return len(self._batch_results)@@ -485,6 +549,7 @@ class PipelineV3(Generic[ResumableData]):def _mark_first_ever_sync(self) -> None:self._pg_producer.is_first_ever_sync = True+ self._prefix_checkpoint = Noneasync def _record_table_rebuild(self, run_uuid: str) -> None:"""Record that this attempt loads the table from empty (see `TableRebuildRun`).@@ -675,9 +740,13 @@ class PipelineV3(Generic[ResumableData]):try:await self._sinks.clear()+ await self._drop_prefix_checkpoint_for_active_sinks()+# v3 stages the incremental cursor until job completion, so a retried attempt- # re-extracts from batch 0 and the previous attempt's count must not be kept.- await reset_rows_synced_if_needed(self._job, should_resume)+ # re-extracts from batch 0 and the previous attempt's count must not be kept. Only the+ # rows below a watermark that a loaded prefix of this job moved stay counted.+ await reset_rows_synced_if_needed(self._job, should_resume, kept_rows_synced=self._kept_rows_synced)+ self._job_rows_synced_through_batch = self._job.rows_synced or 0validate_incremental_sync(self._is_incremental,diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/producer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/producer.pyindex 5382fee5b43f..3c0cdee59ff3 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/producer.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/producer.py@@ -30,7 +30,7 @@ from products.warehouse_sources.backend.temporal.data_imports.util import (PostHogInternalDatabaseError,is_transient_internal_db_error,)-from products.warehouse_sources_queue.backend.core.jobs_db import BATCH_TABLE, BatchQueue+from products.warehouse_sources_queue.backend.core.jobs_db import BATCH_TABLE, LOADED_PREFIX_CURSOR_KEY, BatchQueuelogger = structlog.get_logger(__name__)@@ -124,9 +124,10 @@ class PostgresProducer:with _queue_db_errors():self._conn = _connect_with_retry(database_url)self._batches_sent = 0- # The most recent staged batch and its cumulative row count, kept out of the queue until the- # next batch arrives or the run ends, so the run's last row can carry the final flag itself.- self._held: tuple[BatchWriteResult, int] | None = None+ # The most recent staged batch, its cumulative row count and its loaded-prefix cursor, kept out+ # of the queue until the next batch arrives or the run ends, so the run's last row can carry+ # the final flag itself.+ self._held: tuple[BatchWriteResult, int, dict[str, Any] | None] | None = None@propertydef sync_type(self) -> SyncTypeLiteral:@@ -144,7 +145,13 @@ class PostgresProducer:def has_held_batch(self) -> bool:return self._held is not None- def hold_batch(self, batch_result: BatchWriteResult, *, cumulative_row_count: int) -> None:+ def hold_batch(+ self,+ batch_result: BatchWriteResult,+ *,+ cumulative_row_count: int,+ loaded_prefix_cursor: dict[str, Any] | None = None,+ ) -> None:"""Stage a batch's queue row, inserting the previously held one as a non-final row.The parquet file is already durable when this is called; only the queue row waits. Holding@@ -157,9 +164,14 @@ class PostgresProducer:if batch_result.batch_index == 0 and not self._is_resume:self._supersede_other_runs()previous = self._held- self._held = (batch_result, cumulative_row_count)+ self._held = (batch_result, cumulative_row_count, loaded_prefix_cursor)if previous is not None:- self._insert(previous[0], is_final_batch=False, cumulative_row_count=previous[1])+ self._insert(+ previous[0],+ is_final_batch=False,+ cumulative_row_count=previous[1],+ loaded_prefix_cursor=previous[2],+ )def release_held_batch(self) -> bool:"""Insert the held batch as a non-final row now. Returns whether a row was inserted.@@ -171,7 +183,7 @@ class PostgresProducer:if held is None:return Falseself._held = None- self._insert(held[0], is_final_batch=False, cumulative_row_count=held[1])+ self._insert(held[0], is_final_batch=False, cumulative_row_count=held[1], loaded_prefix_cursor=held[2])return Truedef send_final_batch(@@ -287,6 +299,7 @@ class PostgresProducer:data_folder: Optional[str] = None,schema_path: Optional[str] = None,cumulative_row_count: int = 0,+ loaded_prefix_cursor: dict[str, Any] | None = None,) -> None:metadata: dict[str, Any] = {}if data_folder is not None:@@ -320,6 +333,8 @@ class PostgresProducer:metadata["arrow_bytes"] = batch_result.arrow_bytesif self._external_destination_ids is not None:metadata["external_destination_ids"] = self._external_destination_ids+ if loaded_prefix_cursor is not None:+ metadata[LOADED_PREFIX_CURSOR_KEY] = loaded_prefix_cursorwith _queue_db_errors():self._conn.execute(diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_jobs_db.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_jobs_db.pyindex 487def7575bb..0b8aa85dd1ea 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_jobs_db.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_jobs_db.py@@ -23,6 +23,7 @@ from products.warehouse_sources_queue.backend.core.jobs_db import (BATCH_TABLE,CLAIM_ELIGIBILITY_INTERVAL,LEASE_TABLE,+ LOADED_PREFIX_CURSOR_KEY,PARTITION_PRUNING_INTERVAL,STATUS_TABLE,TAKEOVER_STALE_THRESHOLD_SECONDS,@@ -30,6 +31,7 @@ from products.warehouse_sources_queue.backend.core.jobs_db import (ClaimCursor,ExecutingAttempt,InterruptionHistory,+ LoadedPrefix,PendingBatch,QueueDepth,_claim_window_sql,@@ -1678,6 +1680,81 @@ class TestUnfinishedEarlierBatchesSync:assert self._check(_db_url).count == 1[email protected]_db(transaction=True)+class TestLoadedPrefixSync:+ @pytest.fixture(autouse=True)+ def _fresh_pool(self):+ _SYNC_POOLS.clear()+ yield+ for pool in _SYNC_POOLS.values():+ for pooled in pool._idle:+ pooled.close()+ _SYNC_POOLS.clear()++ @staticmethod+ def _cursor(index: int) -> dict[str, Any]:+ return {"field": "id", "value": index * 10, "job_rows_synced": index}++ async def _insert_run(self, conn: psycopg.AsyncConnection[Any], states: list[str | None]) -> None:+ for index, state in enumerate(states):+ if state is None:+ continue+ bid = await _insert_batch(conn, batch_index=index, metadata={LOADED_PREFIX_CURSOR_KEY: self._cursor(index)})+ if state != "pending":+ await BatchQueue.update_status(conn, batch_id=bid, job_state=state, attempt=1)++ @pytest.mark.asyncio+ @pytest.mark.parametrize(+ "states,first,last,expected_complete",+ [+ (["executing"], 0, 0, True),+ (["succeeded", "succeeded", "executing"], 2, 2, True),+ # A coalesced write of batches 2 and 3.+ (["succeeded", "succeeded", "executing", "executing"], 2, 3, True),+ # Batch 1 aged out or waits for a retry while batch 2 loaded out of order.+ (["succeeded", "pending", "executing"], 2, 2, False),+ (["succeeded", "waiting_retry", "executing"], 2, 2, False),+ # A newer run superseded the rest of this run.+ (["succeeded", "failed", "executing"], 2, 2, False),+ # Batch 1 is gone, for example with a dropped partition.+ (["succeeded", None, "executing"], 2, 2, False),+ ],+ )+ async def test_a_prefix_is_complete_only_when_every_earlier_batch_succeeded(+ self, conn, _db_url, states, first, last, expected_complete+ ):+ await self._insert_run(conn, states)+ await _insert_batch(conn, run_uuid="run-other", batch_index=1)++ prefix = BatchQueue.get_loaded_prefix_sync(+ _db_url,+ team_id=1,+ schema_id="schema-1",+ run_uuid="run-1",+ first_batch_index=first,+ last_batch_index=last,+ connect_timeout_seconds=5,+ )++ assert prefix == LoadedPrefix(is_complete=expected_complete, cursor=self._cursor(last))++ @pytest.mark.asyncio+ async def test_a_batch_without_the_key_has_no_cursor(self, conn, _db_url):+ await _insert_batch(conn, batch_index=0)++ prefix = BatchQueue.get_loaded_prefix_sync(+ _db_url,+ team_id=1,+ schema_id="schema-1",+ run_uuid="run-1",+ first_batch_index=0,+ last_batch_index=0,+ connect_timeout_seconds=5,+ )++ assert prefix == LoadedPrefix(is_complete=True, cursor=None)++@pytest.mark.django_db(transaction=True)class TestClaimWindowSkipsForeignLeasedGroups:@pytest.mark.asynciodiff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.pyindex c93be7c72b8c..f70c854f1c90 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.py@@ -5,12 +5,15 @@ import threadingfrom contextlib import ExitStack, nullcontextfrom dataclasses import dataclassfrom datetime import UTC, datetime, timedelta+from decimal import Decimalfrom types import SimpleNamespacefrom typing import Any, castimport pytestfrom unittest.mock import AsyncMock, MagicMock, patch+from django.test import override_settings+import pyarrow as paimport deltalakefrom asgiref.sync import async_to_sync@@ -2072,6 +2075,208 @@ def _passthrough_pool(fn):return call+class TestLoadedPrefixCursor:+ @staticmethod+ def _pipeline(+ *,+ enabled: bool = True,+ sort_mode: str | None = "asc",+ sync_type: str = "incremental",+ reset_pipeline: bool = False,+ checkpoints_allowed: bool = True,+ can_resume: bool = False,+ own_cursor: bool = False,+ lanes: list[Any] | None = None,+ cdc_write_mode: str | None = None,+ primary_keys: list[str] | None = None,+ table_exists: bool = True,+ resumed_value: Any = None,+ external_destination_ids: list[str] | None = None,+ ) -> PipelineV3:+ schema = ExternalDataSchema(+ name="users",+ team_id=1,+ sync_type=sync_type,+ sync_type_config={"incremental_field": "n", "incremental_field_type": IncrementalFieldType.Integer},+ )+ resource = MagicMock(+ name="users",+ primary_keys=["id"] if primary_keys is None else primary_keys,+ partition_count=None,+ partition_size=None,+ partition_keys=["id"],+ partition_format=None,+ partition_mode=None,+ cdc_write_mode=cdc_write_mode,+ lanes=lanes,+ sort_mode=sort_mode,+ webhook_only=False,+ chunk_size=None,+ chunk_size_bytes=None,+ )+ manager = MagicMock(can_resume=MagicMock(return_value=True)) if can_resume else None+ with (+ override_settings(DATA_WAREHOUSE_IMPORT_PREFIX_WATERMARK_ENABLED=enabled),+ patch.object(ExternalDataSchema, "table", MagicMock() if table_exists else None),+ patch(f"{_PIPELINE}.current_import_attempt", return_value=1),+ patch(f"{_PIPELINE}.current_workflow_id", return_value="wf-1"),+ patch(f"{_PIPELINE}.current_workflow_run_id", return_value="wfrun-abc"),+ patch(f"{_PIPELINE}.S3BatchWriter"),+ patch(f"{_PIPELINE}.PostgresProducer"),+ patch(f"{_PIPELINE}.DeltaTableRef"),+ patch(f"{_PIPELINE}.resolve_resume_manager", return_value=manager),+ patch(+ f"{_PIPELINE}.resolve_primary_keys", return_value=primary_keys if primary_keys is not None else ["id"]+ ),+ ):+ return PipelineV3(+ source_response=resource,+ logger=_make_logger(),+ job_id="job-1",+ reset_pipeline=reset_pipeline,+ shutdown_monitor=MagicMock(),+ resumable_source_manager=manager,+ models=ImportJobModels(+ job=MagicMock(team_id=1, workflow_run_id="wfrun-abc", billable=False, id="job-1"),+ schema=schema,+ source=MagicMock(source_type="Postgres"),+ table=None,+ external_destination_ids=external_destination_ids or [],+ ),+ source_cursor_manager=MagicMock() if own_cursor else None,+ incremental_checkpoints_allowed=checkpoints_allowed,+ resumed_incremental_value=resumed_value,+ )++ @staticmethod+ async def _held_cursors(pipeline: PipelineV3, batches: list[list[int]]) -> list[Any]:+ row_count = 0+ for index, values in enumerate(batches):+ row_count += len(values)+ await pipeline._stage_batch(pa.table({"id": values, "n": values}), index, row_count)+ return [+ call.kwargs["loaded_prefix_cursor"]+ for call in cast(MagicMock, pipeline._pg_producer.hold_batch).call_args_list+ ]++ @pytest.mark.asyncio+ @pytest.mark.parametrize(+ "batches,expected_values",+ [+ ([[1, 2, 3], [4, 5, 6]], [2, 5]),+ # Rows that hold 3 continue into batch 1, so batch 0 covers only the rows below 3.+ ([[1, 2, 3], [3, 3, 4], [4, 5, 6]], [2, 3, 5]),+ # A batch that holds only the newest value adds nothing new below it.+ ([[1, 2, 3], [3, 3], [3, 4]], [2, 2, 3]),+ ([[7, 7]], [None]),+ # A row below a value already seen proves the read is not in order, so no later batch+ # covers anything.+ ([[1, 2, 3], [2, 9], [10, 11]], [2, None, None]),+ ([[3, 1]], [None]),+ ],+ )+ async def test_each_batch_records_what_the_prefix_through_it_covers(+ self, batches: list[list[int]], expected_values: list[Any]+ ) -> None:+ cursors = await self._held_cursors(self._pipeline(), batches)++ assert [cursor and cursor["value"] for cursor in cursors] == expected_values+ assert [cursor["field"] for cursor in cursors if cursor] == ["n"] * sum(v is not None for v in expected_values)++ @pytest.mark.asyncio+ async def test_the_cursor_carries_the_rows_the_job_synced_through_the_batch(self) -> None:+ pipeline = self._pipeline()+ pipeline._job_rows_synced_through_batch = 100++ cursors = await self._held_cursors(pipeline, [[1, 2, 3], [4, 5]])++ assert [cursor["job_rows_synced"] for cursor in cursors] == [103, 105]++ @pytest.mark.asyncio+ @pytest.mark.parametrize(+ "values,expected",+ [+ # Storing -1.4 as -1 would skip the unread rows between them.+ ([Decimal("-1.5"), Decimal("-1.4")], None),+ ([Decimal("1.5"), Decimal("2.5")], 1),+ ],+ )+ async def test_a_decimal_value_that_rounds_above_the_loaded_rows_records_nothing(+ self, values: list[Decimal], expected: Any+ ) -> None:+ pipeline = self._pipeline()+ pipeline._schema.sync_type_config["incremental_field_type"] = IncrementalFieldType.Numeric++ await pipeline._stage_batch(pa.table({"id": [1, 2], "n": pa.array(values, pa.decimal128(5, 1))}), 0, 2)++ cursor = cast(MagicMock, pipeline._pg_producer.hold_batch).call_args.kwargs["loaded_prefix_cursor"]+ assert (cursor and cursor["value"]) == expected++ @pytest.mark.asyncio+ @pytest.mark.parametrize("sink_gates,expected_values", [([False, False], [2, 5]), ([False, True], [None, None])])+ async def test_a_table_with_an_active_chunk_sink_records_nothing(+ self, sink_gates: list[bool], expected_values: list[Any]+ ) -> None:+ # A retry after a promotion skips rows whose staged sink files the next attempt deletes.+ pipeline = self._pipeline()+ pipeline._sinks = MagicMock(all=[MagicMock(should_run=AsyncMock(return_value=gate)) for gate in sink_gates])++ await pipeline._drop_prefix_checkpoint_for_active_sinks()+ cursors = await self._held_cursors(pipeline, [[1, 2, 3], [4, 5, 6]])++ assert [cursor and cursor["value"] for cursor in cursors] == expected_values++ @pytest.mark.asyncio+ @pytest.mark.parametrize(+ "run",+ [+ {"enabled": False},+ {"sort_mode": "desc"},+ {"sort_mode": None},+ {"sync_type": "append"},+ {"sync_type": "full_refresh"},+ {"cdc_write_mode": "scd2_append"},+ {"reset_pipeline": True},+ {"checkpoints_allowed": False},+ {"can_resume": True},+ {"own_cursor": True},+ {"lanes": [MagicMock()]},+ {"primary_keys": []},+ {"table_exists": False},+ {"resumed_value": 2},+ {"external_destination_ids": ["destination-1"]},+ ],+ ids=[+ "setting_off",+ "descending",+ "unordered",+ "append",+ "full_refresh",+ "cdc",+ "reset",+ "stored_cursors_ignored",+ "resumable_source",+ "source_reads_its_own_cursor",+ "laned_source",+ "no_primary_key",+ "first_ever_sync",+ "continues_an_earlier_attempt",+ "external_destination",+ ],+ )+ async def test_a_run_that_cannot_promote_a_prefix_records_no_cursor(self, run: dict[str, Any]) -> None:+ cursors = await self._held_cursors(self._pipeline(**run), [[1, 2, 3], [4, 5, 6]])++ assert cursors == [None, None]++ @pytest.mark.asyncio+ async def test_a_run_that_rebuilds_the_table_records_no_cursor(self) -> None:+ pipeline = self._pipeline()+ pipeline._mark_first_ever_sync()++ assert await self._held_cursors(pipeline, [[1, 2, 3], [4, 5, 6]]) == [None, None]++class TestIncrementalHandoffCheckpoint:def _pipeline(self, items, events: list[Any], *, resumed_from: int | None = None) -> PipelineV3:pipeline = _make_pipeline()diff --git a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/import_data_sync.py b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/import_data_sync.pyindex df8f143eeecc..86c2c7daf8fd 100644--- a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/import_data_sync.py+++ b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/import_data_sync.py@@ -37,6 +37,7 @@ from products.warehouse_sources.backend.models.external_data_schema import (UnsupportedSyncTypeError,apply_incremental_lookback,get_schema_if_exists,+ prefix_promotion_rows_synced,process_incremental_value,resolve_sync_type,staged_handoff_resume_point,@@ -513,6 +514,7 @@ async def _import_data_with_reporting(inputs: ImportDataActivityInputs, logger:processed_incremental_earliest_value = None# The cursor as stored, before the lookback shift below moves it back.incremental_last_value_before_lookback = None+ kept_rows_synced: int | None = None# A pending corrupt-delta revive rebuilds this table inside this run: handle_corrupted_delta_log# resets the Delta table before extraction, so the loader overwrites it from batch 0. The@@ -524,6 +526,8 @@ async def _import_data_with_reporting(inputs: ImportDataActivityInputs, logger:use_stored_cursors = reset_pipeline is not True and not delta_rebuild_pendingif use_stored_cursors:+ # Read from the same config as the watermark, so the count matches the rows this attempt skips.+ kept_rows_synced = prefix_promotion_rows_synced(schema.sync_type_config, str(model.id))processed_incremental_last_value = process_incremental_value(schema.sync_type_config.get("incremental_field_last_value"),schema.sync_type_config.get("incremental_field_type"),@@ -773,6 +777,7 @@ async def _import_data_with_reporting(inputs: ImportDataActivityInputs, logger:resumed_incremental_run_uuid=resumed_incremental_run_uuid,resumed_incremental_value=resumed_incremental_value,preemption=preemption,+ kept_rows_synced=kept_rows_synced,)else:raise ValueError(f"Source type {model.pipeline.source_type} not supported")@@ -1149,6 +1154,7 @@ async def _run(resumed_incremental_value: Any = None,preemption: PreemptionConfig | None = None,on_resumable_retry_budget: bool = False,+ kept_rows_synced: int | None = None,) -> PipelineResult:try:reset_pipeline = (@@ -1169,6 +1175,7 @@ async def _run(resumed_incremental_run_uuid=resumed_incremental_run_uuid,resumed_incremental_value=resumed_incremental_value,preemption=preemption,+ kept_rows_synced=kept_rows_synced,)result = await pipeline.run()diff --git a/products/warehouse_sources/backend/tests/test_models.py b/products/warehouse_sources/backend/tests/test_models.pyindex ec731c9947bc..6b70c1e37351 100644--- a/products/warehouse_sources/backend/tests/test_models.py+++ b/products/warehouse_sources/backend/tests/test_models.py@@ -28,12 +28,14 @@ from products.warehouse_sources.backend.models.external_data_schema import (DUPLICATE_PRIMARY_KEYS_RAW_ERROR,MISSING_PRIMARY_KEY_DISABLED_MESSAGE,MISSING_PRIMARY_KEYS_RAW_ERROR,+ PREFIX_PROMOTION_KEY,REPARTITION_HOLD_MAX_AGE,STAGED_CURSOR_PENDING_LIMIT,ExternalDataSchema,apply_incremental_lookback,mark_initial_sync_complete,mark_schema_running_unless_halted,+ prefix_promotion_rows_synced,process_incremental_value,staged_handoff_resume_point,staged_handoff_resume_value,@@ -1674,11 +1676,112 @@ class TestStagedIncrementalCursor:schema = self._make_schema(incremental_staged={"run_uuid": "run-1", "last_value": 42},incremental_staged_pending=[{"run_uuid": "run-0", "last_value": 1}],+ **{PREFIX_PROMOTION_KEY: {"job_id": "job-1", "run_uuid": "run-1", "value": 30, "rows_synced": 7}},)with _merge_in_memory(schema):schema.update_sync_type_config_for_reset_pipeline()assert "incremental_staged" not in schema.sync_type_configassert "incremental_staged_pending" not in schema.sync_type_config+ assert PREFIX_PROMOTION_KEY not in schema.sync_type_config++ def _promote_prefix(self, schema: ExternalDataSchema, value: Any, *, field: str = "id") -> bool:+ with self._staged_in_memory(schema):+ return schema.promote_loaded_prefix_value(+ run_uuid="run-1", job_id="job-1", field=field, value=value, job_rows_synced=7+ )++ @parameterized.expand(+ [+ ("no_watermark_yet", {}, 30),+ ("above_the_watermark", {"incremental_field_last_value": 10}, 30),+ (+ "parked_staged_cursor",+ {+ "incremental_staged": {"run_uuid": "run-2", "last_value": 5},+ "incremental_staged_pending": [{"run_uuid": "run-1", "last_value": 50}],+ },+ 30,+ ),+ ]+ )+ def test_promote_loaded_prefix_moves_the_watermark_up(self, _name: str, config: dict, value: Any) -> None:+ schema = self._make_schema(+ incremental_field="id", **{"incremental_staged": {"run_uuid": "run-1", "last_value": 50}, **config}+ )++ assert self._promote_prefix(schema, value) is True++ assert schema.sync_type_config["incremental_field_last_value"] == value+ assert schema.sync_type_config[PREFIX_PROMOTION_KEY] == {+ "job_id": "job-1",+ "run_uuid": "run-1",+ "value": value,+ "rows_synced": 7,+ }++ @parameterized.expand(+ [+ # A newer run already moved the watermark past this run's prefix.+ ("newer_run_is_ahead", {"incremental_field_last_value": 40}, "id", 30),+ ("equal_to_the_watermark", {"incremental_field_last_value": 30}, "id", 30),+ # A reset dropped the staged cursors, so the loaded rows belong to the deleted table.+ ("reset_dropped_the_staged_cursor", {"incremental_staged": None}, "id", 30),+ (+ "staged_cursor_of_another_run_only",+ {"incremental_staged": {"run_uuid": "run-2", "last_value": 50}},+ "id",+ 30,+ ),+ ("value_above_what_the_run_read", {}, "id", 60),+ ("incremental_field_changed", {}, "updated_at", 30),+ (+ "values_cannot_be_ordered",+ {"incremental_field_type": IncrementalFieldType.ObjectID, "incremental_field_last_value": "abc"},+ "id",+ "65f0c0c0a1b2c3d4e5f60001",+ ),+ ]+ )+ def test_promote_loaded_prefix_keeps_the_watermark(self, _name: str, config: dict, field: str, value: Any) -> None:+ schema = self._make_schema(+ incremental_field="id", **{"incremental_staged": {"run_uuid": "run-1", "last_value": 50}, **config}+ )+ if schema.sync_type_config["incremental_staged"] is None:+ del schema.sync_type_config["incremental_staged"]+ watermark_before = schema.sync_type_config.get("incremental_field_last_value")++ assert self._promote_prefix(schema, value, field=field) is False++ assert schema.sync_type_config.get("incremental_field_last_value") == watermark_before+ assert PREFIX_PROMOTION_KEY not in schema.sync_type_config++ def test_final_promotion_still_moves_the_watermark_to_the_full_value_after_a_prefix(self) -> None:+ schema = self._make_schema(incremental_field="id", incremental_staged={"run_uuid": "run-1", "last_value": 50})++ assert self._promote_prefix(schema, 30) is True+ with self._staged_in_memory(schema):+ assert schema.promote_staged_incremental_values("run-1") is True++ assert schema.sync_type_config["incremental_field_last_value"] == 50++ @parameterized.expand(+ [+ ("same_job_and_watermark", "job-1", 30, 7),+ ("another_job", "job-2", 30, None),+ # A later promotion moved the watermark, so this attempt does not start at the prefix.+ ("watermark_moved_since", "job-1", 45, None),+ ]+ )+ def test_prefix_promotion_rows_synced(self, _name: str, job_id: str, watermark: Any, expected: int | None) -> None:+ config = {+ "incremental_field_last_value": watermark,+ PREFIX_PROMOTION_KEY: {"job_id": "job-1", "run_uuid": "run-1", "value": 30, "rows_synced": 7},+ }++ assert prefix_promotion_rows_synced(config, job_id) == expected++ def test_prefix_promotion_rows_synced_without_a_promotion(self) -> None:+ assert prefix_promotion_rows_synced({"incremental_field_last_value": 30}, "job-1") is Noneclass TestStagedIncrementalCursorStaleWriters(BaseTest):diff --git a/products/warehouse_sources_queue/backend/core/jobs_db.py b/products/warehouse_sources_queue/backend/core/jobs_db.pyindex f693017e6de9..bb2faf6982d7 100644--- a/products/warehouse_sources_queue/backend/core/jobs_db.py+++ b/products/warehouse_sources_queue/backend/core/jobs_db.py@@ -1001,6 +1001,30 @@ _UNFINISHED_EARLIER_BATCHES_SQL = f""""""+# Batch metadata key for the watermark that the loaded batches of a run cover once this batch and+# every earlier batch of the run are loaded. Only the producer writes it (see `get_loaded_prefix_sync`).+LOADED_PREFIX_CURSOR_KEY = "loaded_prefix_cursor"++# Server-side ceiling for the loaded-prefix read. The promotion is best effort, so on a struggling+# queue DB it gives up and leaves the watermark where it is instead of holding the loader thread.+LOADED_PREFIX_STATEMENT_TIMEOUT_MS = 5_000++# Reads batches 0..last of one run through ``sb_run_uuid_bi_idx``. The final-only marker of a run can+# repeat a batch_index, so the batches are counted by distinct index.+_LOADED_PREFIX_SQL = f"""+ SELECT+ count(DISTINCT b.batch_index) FILTER (WHERE b.batch_index < %(first_batch_index)s),+ count(*) FILTER (WHERE b.batch_index < %(first_batch_index)s AND b.latest_state <> 'succeeded'),+ (array_agg(b.metadata -> %(cursor_key)s) FILTER (WHERE b.batch_index = %(last_batch_index)s))[1]+ FROM {BATCH_TABLE} b+ WHERE b.run_uuid = %(run_uuid)s+ AND b.batch_index <= %(last_batch_index)s+ AND b.team_id = %(team_id)s+ AND b.schema_id = %(schema_id)s+ AND b.created_at > now() - interval '{PARTITION_PRUNING_INTERVAL}'+"""++def _stale_executing_sql(scope_sql: str = "") -> str:"""Shared body of the stale-executing sweep (async consumer and its sync ops twin).@@ -1252,6 +1276,14 @@ class UnfinishedEarlierBatches:states: tuple[str, ...]+@dataclass(frozen=True, slots=True)+class LoadedPrefix:+ """Whether batches 0..last of a run are all loaded, and the cursor the last batch recorded."""++ is_complete: bool+ cursor: Any++@dataclass(frozen=True, slots=True)class OrphanedRunRef:"""A run holding a ``failed`` batch that still has non-terminal batches behind it.@@ -2099,6 +2131,56 @@ class BatchQueue:)raise AssertionError("unreachable")+ @staticmethod+ def get_loaded_prefix_sync(+ database_url: str,+ *,+ team_id: int,+ schema_id: str,+ run_uuid: str,+ first_batch_index: int,+ last_batch_index: int,+ connect_timeout_seconds: int = 10,+ statement_timeout_ms: int = LOADED_PREFIX_STATEMENT_TIMEOUT_MS,+ ) -> LoadedPrefix:+ """Whether a write of batches ``first..last`` of a run completes a loaded prefix of the run.++ The loader asks this after the write, while the batches of the write are still executing.+ The prefix is complete when every index below ``first_batch_index`` has a row and every such+ row reached 'succeeded'. Only a run that numbers its batches 0, 1, 2 and so on (no lanes)+ records ``LOADED_PREFIX_CURSOR_KEY``, so a missing index means the batch is gone, for example+ after the partition that held it was dropped.++ Raises ``psycopg.errors.QueryCanceled`` past ``statement_timeout_ms``.+ """+ pool = _sync_connection_pool(database_url, connect_timeout_seconds)+ params = {+ "team_id": team_id,+ "schema_id": schema_id,+ "run_uuid": run_uuid,+ "first_batch_index": first_batch_index,+ "last_batch_index": last_batch_index,+ "cursor_key": LOADED_PREFIX_CURSOR_KEY,+ }+ for attempt in (1, 2):+ with pool.connection() as conn:+ try:+ # The timeout is transaction-local, so it never leaks into the pooled session.+ with conn.transaction():+ conn.execute("SELECT set_config('statement_timeout', %s, true)", (str(statement_timeout_ms),))+ row = conn.execute(_LOADED_PREFIX_SQL, params).fetchone()+ except psycopg.errors.QueryCanceled:+ raise+ except psycopg.OperationalError:+ pool.discard(conn)+ if attempt == 2:+ raise+ continue+ assert row is not None+ earlier_batches, unfinished, cursor = row+ return LoadedPrefix(is_complete=earlier_batches == first_batch_index and unfinished == 0, cursor=cursor)+ raise AssertionError("unreachable")+@staticmethodasync def get_stale_executing(conn: psycopg.AsyncConnection[Any],