Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(warehouse-sources): skip pre-write compaction before an append restore (#119745)

Daniel RCb24fe5e8ced2
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/append_rollback.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/append_rollback.py
index 6d8f47130105..17a701d053a7 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/append_rollback.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/append_rollback.py
@@ -120,13 +120,13 @@ class UnfinishedAppendRuns:
return
delta_table = async_to_sync(self._delta_table_ref.get_delta_table)()
- if (
- not export_signal.is_resume
- and marker is not None
- and delta_table is not None
- and self._never_completed(marker)
- ):
- self._roll_back(marker, export_signal)
+ unfinished = (
+ None
+ if export_signal.is_resume or delta_table is None
+ else self._marker_to_roll_back(export_signal.run_uuid)
+ )
+ if unfinished is not None:
+ self._roll_back(unfinished, export_signal)
delta_table = async_to_sync(self._delta_table_ref.get_delta_table)()
# Written before the batch, so a crash between the two leaves a version that is still the
@@ -142,6 +142,24 @@ class UnfinishedAppendRuns:
self._schema.id, self._schema.team_id, updates={APPEND_RUN_MARKER_KEY: started.to_config()}
)
+ def restores_before_first_batch(self, run_uuid: str) -> bool:
+ """Whether `start_run` tries to restore the table at the first batch of the fresh attempt `run_uuid`.
+
+ The restore also adds back every file that a compaction after the recorded version removed.
+ A compaction before that batch therefore only makes the restore larger. `_roll_back` can still
+ skip the restore, for example when the unfinished run committed nothing.
+ """
+ if not settings.DATA_WAREHOUSE_APPEND_ROLLBACK_ENABLED:
+ return False
+ return self._marker_to_roll_back(run_uuid) is not None
+
+ def _marker_to_roll_back(self, run_uuid: str) -> AppendRunMarker | None:
+ """The recorded run that a fresh attempt `run_uuid` removes before it loads, or None."""
+ marker = self._marker()
+ if marker is None or marker.run_uuid == run_uuid or not self._never_completed(marker):
+ return None
+ return marker
+
def _never_completed(self, marker: AppendRunMarker) -> bool:
"""Whether every row written after the recorded version belongs to a run that did not complete.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/test_append_rollback.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/test_append_rollback.py
index a58b1162fe57..2907c5d88e23 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/test_append_rollback.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/test_append_rollback.py
@@ -21,6 +21,7 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.del
make_local_table_ref,
)
from products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.load.append_rollback import (
+ UnfinishedAppendRuns,
clear_append_run_marker,
)
from products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.load.processor import (
@@ -201,6 +202,18 @@ def _failed_mid_run(loader: _Loader) -> None:
loader.fail(job, "connection to the source was lost")
+def _old_marker_behind_a_completed_job(loader: _Loader) -> None:
+ failed_job = loader.new_job()
+ loader.load(loader.messages(failed_job, [[3, 4]], final=False))
+ old_marker = loader.config()[APPEND_RUN_MARKER_KEY]
+ loader.fail(failed_job, "failed")
+ loader.load(loader.messages(loader.new_job(), [[5, 6]], final=True))
+ # A loader from before the marker completes a run and leaves the old marker in place.
+ config = loader.config()
+ config[APPEND_RUN_MARKER_KEY] = old_marker
+ ExternalDataSchema.objects.filter(id=loader.schema.id).update(sync_type_config=config)
+
+
def _failed_twice(loader: _Loader) -> None:
_cancelled(loader)
_failed_mid_run(loader)
@@ -315,6 +328,32 @@ class TestUnfinishedAppendRun:
assert loader.ids() == _SEED_IDS + _WINDOW_IDS
assert "RESTORE" not in loader.operations()
+ @pytest.mark.parametrize(
+ "before, attempt, restores",
+ [
+ (lambda loader, job: _failed_mid_run(loader), 1, True),
+ (lambda loader, job: loader.load(loader.messages(job, [[3, 4]], final=False)), 2, True),
+ (lambda loader, job: loader.load(loader.messages(job, [[3, 4]], final=False)), 1, False),
+ (lambda loader, job: _old_marker_behind_a_completed_job(loader), 1, False),
+ ],
+ ids=["failed_job", "earlier_attempt", "same_attempt", "old_marker_behind_a_completed_job"],
+ )
+ def test_the_pre_load_check_predicts_the_restore(
+ self, loader: _Loader, before: Any, attempt: int, restores: bool
+ ) -> None:
+ loader.seed()
+ job = loader.new_job()
+ before(loader, job)
+ restores_so_far = loader.operations().count("RESTORE")
+
+ predicted = UnfinishedAppendRuns(
+ ExternalDataSchema.objects.get(id=loader.schema.id), make_local_table_ref(loader.table_path)
+ ).restores_before_first_batch(f"{job.workflow_run_id}-a{attempt}")
+ loader.load(loader.messages(job, [[5, 6]], attempt=attempt, final=False))
+
+ assert predicted is restores
+ assert loader.operations().count("RESTORE") == restores_so_far + int(restores)
+
def test_a_first_resumed_batch_records_the_job(self, loader: _Loader) -> None:
loader.seed()
job = loader.new_job()
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.py
index 0b28e1bb8a58..f41eb8de45a9 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.py
@@ -602,6 +602,29 @@ class PipelineV3(Generic[ResumableData]):
"""
return self._resource.cdc_write_mode == SCD2_APPEND_MODE
+ async def _first_batch_restores_table(self, should_resume: bool) -> bool:
+ """Whether the loader restores the table to undo an unfinished append run before this attempt loads.
+
+ The restore undoes a pre-write compaction too, so the compaction is wasted work that also makes the
+ restore commit larger. A pre-write vacuum can delete files that the restore needs, once they were
+ removed longer ago than `VACUUM_RETENTION`, and a failed restore leaves the unfinished run's rows in
+ the table. The post-load pass of a run that completes maintains the table instead.
+ """
+ if should_resume or self._run_uuid is None or self._pg_producer.sync_type != "append":
+ return False
+ # Deferred: the `load` package imports the load processor, which imports the facade that imports this module.
+ from products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.load.append_rollback import ( # noqa: PLC0415
+ UnfinishedAppendRuns,
+ )
+
+ append_runs = UnfinishedAppendRuns(self._schema, self._delta_table_ref)
+ try:
+ return await database_sync_to_async_pool(append_runs.restores_before_first_batch)(self._run_uuid)
+ except Exception:
+ # Pre-write maintenance never fails an attempt, so a failed read must not either.
+ await self._logger.awarning("V3 Pipeline: Could not check for a pending append restore", exc_info=True)
+ return False
+
def _close_producers(self) -> None:
self._pg_producer.close()
@@ -851,7 +874,7 @@ class PipelineV3(Generic[ResumableData]):
# Defensive pre-write compaction so a sync that arrived at a fragmented Delta
# target cleans up before adding more small files; see DeltaMaintenance.run_scheduled.
- if not is_fresh_sync:
+ if not is_fresh_sync and not await self._first_batch_restores_table(should_resume):
await DeltaMaintenance(self._delta_table_ref).run_scheduled(
self._schema,
is_cdc_companion=self._maintains_companion_table(),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.py
index 8cac34d88cc9..0a11de69837e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.py
@@ -463,6 +463,52 @@ class TestReplaceScopeHandOff:
assert "other" not in result.column_names
+class TestPreWriteMaintenanceBeforeAnAppendRestore:
+ @pytest.mark.parametrize(
+ "loader_restores, expected_maintained",
+ [(True, False), (False, True)],
+ ids=["restore_pending", "no_restore"],
+ )
+ @pytest.mark.asyncio
+ async def test_the_pre_write_pass_is_skipped_when_the_first_batch_restores_the_table(
+ self, loader_restores: bool, expected_maintained: bool
+ ) -> None:
+ pipeline = _make_pipeline()
+ pipeline._attempt = 2
+ pipeline._run_uuid = "run-abc-a2"
+ pipeline._schema.table = MagicMock()
+ pipeline._schema.sync_type_config = {}
+ pipeline._delta_table_ref = MagicMock(is_first_sync=False)
+ pipeline._pg_producer = MagicMock(sync_type="append", is_first_ever_sync=False)
+ pipeline._resource.items = MagicMock(return_value=iter([]))
+ pipeline._batcher.should_yield.return_value = False # type: ignore[attr-defined]
+
+ with ExitStack() as stack:
+ for name in (
+ "reset_rows_synced_if_needed",
+ "setup_row_tracking_with_billing_check",
+ "persist_primary_keys",
+ "handle_reset_or_full_refresh",
+ "handle_corrupted_delta_log",
+ ):
+ stack.enter_context(patch(f"{_PIPELINE}.{name}", new_callable=AsyncMock))
+ stack.enter_context(patch(f"{_PIPELINE}.validate_incremental_sync"))
+ maintenance = stack.enter_context(patch(f"{_PIPELINE}.DeltaMaintenance"))
+ maintenance.return_value.run_scheduled = AsyncMock()
+ append_runs = stack.enter_context(
+ patch(
+ "products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.load.append_rollback.UnfinishedAppendRuns"
+ )
+ )
+ append_runs.return_value.restores_before_first_batch.return_value = loader_restores
+ stack.enter_context(patch(f"{_PIPELINE}.activity")).in_activity.return_value = False
+
+ await pipeline.run()
+
+ append_runs.return_value.restores_before_first_batch.assert_called_once_with("run-abc-a2")
+ assert maintenance.return_value.run_scheduled.called is expected_maintained
+
+
class TestExtractionFailureDoesNotCleanupS3:
@pytest.mark.asyncio
async def test_s3_files_preserved_when_extraction_fails(self) -> None: