:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(warehouse-sources): skip pre-write compaction before an append restore (#119745)
Daniel RCb24fe5e8ced2
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/append_rollback.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/append_rollback.pyindex 6d8f47130105..17a701d053a7 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/append_rollback.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/append_rollback.py@@ -120,13 +120,13 @@ class UnfinishedAppendRuns:returndelta_table = async_to_sync(self._delta_table_ref.get_delta_table)()- if (- not export_signal.is_resume- and marker is not None- and delta_table is not None- and self._never_completed(marker)- ):- self._roll_back(marker, export_signal)+ unfinished = (+ None+ if export_signal.is_resume or delta_table is None+ else self._marker_to_roll_back(export_signal.run_uuid)+ )+ if unfinished is not None:+ self._roll_back(unfinished, export_signal)delta_table = async_to_sync(self._delta_table_ref.get_delta_table)()# Written before the batch, so a crash between the two leaves a version that is still the@@ -142,6 +142,24 @@ class UnfinishedAppendRuns:self._schema.id, self._schema.team_id, updates={APPEND_RUN_MARKER_KEY: started.to_config()})+ def restores_before_first_batch(self, run_uuid: str) -> bool:+ """Whether `start_run` tries to restore the table at the first batch of the fresh attempt `run_uuid`.++ The restore also adds back every file that a compaction after the recorded version removed.+ A compaction before that batch therefore only makes the restore larger. `_roll_back` can still+ skip the restore, for example when the unfinished run committed nothing.+ """+ if not settings.DATA_WAREHOUSE_APPEND_ROLLBACK_ENABLED:+ return False+ return self._marker_to_roll_back(run_uuid) is not None++ def _marker_to_roll_back(self, run_uuid: str) -> AppendRunMarker | None:+ """The recorded run that a fresh attempt `run_uuid` removes before it loads, or None."""+ marker = self._marker()+ if marker is None or marker.run_uuid == run_uuid or not self._never_completed(marker):+ return None+ return marker+def _never_completed(self, marker: AppendRunMarker) -> bool:"""Whether every row written after the recorded version belongs to a run that did not complete.diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/test_append_rollback.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/test_append_rollback.pyindex a58b1162fe57..2907c5d88e23 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/test_append_rollback.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/test_append_rollback.py@@ -21,6 +21,7 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delmake_local_table_ref,)from products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.load.append_rollback import (+ UnfinishedAppendRuns,clear_append_run_marker,)from products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.load.processor import (@@ -201,6 +202,18 @@ def _failed_mid_run(loader: _Loader) -> None:loader.fail(job, "connection to the source was lost")+def _old_marker_behind_a_completed_job(loader: _Loader) -> None:+ failed_job = loader.new_job()+ loader.load(loader.messages(failed_job, [[3, 4]], final=False))+ old_marker = loader.config()[APPEND_RUN_MARKER_KEY]+ loader.fail(failed_job, "failed")+ loader.load(loader.messages(loader.new_job(), [[5, 6]], final=True))+ # A loader from before the marker completes a run and leaves the old marker in place.+ config = loader.config()+ config[APPEND_RUN_MARKER_KEY] = old_marker+ ExternalDataSchema.objects.filter(id=loader.schema.id).update(sync_type_config=config)++def _failed_twice(loader: _Loader) -> None:_cancelled(loader)_failed_mid_run(loader)@@ -315,6 +328,32 @@ class TestUnfinishedAppendRun:assert loader.ids() == _SEED_IDS + _WINDOW_IDSassert "RESTORE" not in loader.operations()+ @pytest.mark.parametrize(+ "before, attempt, restores",+ [+ (lambda loader, job: _failed_mid_run(loader), 1, True),+ (lambda loader, job: loader.load(loader.messages(job, [[3, 4]], final=False)), 2, True),+ (lambda loader, job: loader.load(loader.messages(job, [[3, 4]], final=False)), 1, False),+ (lambda loader, job: _old_marker_behind_a_completed_job(loader), 1, False),+ ],+ ids=["failed_job", "earlier_attempt", "same_attempt", "old_marker_behind_a_completed_job"],+ )+ def test_the_pre_load_check_predicts_the_restore(+ self, loader: _Loader, before: Any, attempt: int, restores: bool+ ) -> None:+ loader.seed()+ job = loader.new_job()+ before(loader, job)+ restores_so_far = loader.operations().count("RESTORE")++ predicted = UnfinishedAppendRuns(+ ExternalDataSchema.objects.get(id=loader.schema.id), make_local_table_ref(loader.table_path)+ ).restores_before_first_batch(f"{job.workflow_run_id}-a{attempt}")+ loader.load(loader.messages(job, [[5, 6]], attempt=attempt, final=False))++ assert predicted is restores+ assert loader.operations().count("RESTORE") == restores_so_far + int(restores)+def test_a_first_resumed_batch_records_the_job(self, loader: _Loader) -> None:loader.seed()job = loader.new_job()diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.pyindex 0b28e1bb8a58..f41eb8de45a9 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.py@@ -602,6 +602,29 @@ class PipelineV3(Generic[ResumableData]):"""return self._resource.cdc_write_mode == SCD2_APPEND_MODE+ async def _first_batch_restores_table(self, should_resume: bool) -> bool:+ """Whether the loader restores the table to undo an unfinished append run before this attempt loads.++ The restore undoes a pre-write compaction too, so the compaction is wasted work that also makes the+ restore commit larger. A pre-write vacuum can delete files that the restore needs, once they were+ removed longer ago than `VACUUM_RETENTION`, and a failed restore leaves the unfinished run's rows in+ the table. The post-load pass of a run that completes maintains the table instead.+ """+ if should_resume or self._run_uuid is None or self._pg_producer.sync_type != "append":+ return False+ # Deferred: the `load` package imports the load processor, which imports the facade that imports this module.+ from products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.load.append_rollback import ( # noqa: PLC0415+ UnfinishedAppendRuns,+ )++ append_runs = UnfinishedAppendRuns(self._schema, self._delta_table_ref)+ try:+ return await database_sync_to_async_pool(append_runs.restores_before_first_batch)(self._run_uuid)+ except Exception:+ # Pre-write maintenance never fails an attempt, so a failed read must not either.+ await self._logger.awarning("V3 Pipeline: Could not check for a pending append restore", exc_info=True)+ return False+def _close_producers(self) -> None:self._pg_producer.close()@@ -851,7 +874,7 @@ class PipelineV3(Generic[ResumableData]):# Defensive pre-write compaction so a sync that arrived at a fragmented Delta# target cleans up before adding more small files; see DeltaMaintenance.run_scheduled.- if not is_fresh_sync:+ if not is_fresh_sync and not await self._first_batch_restores_table(should_resume):await DeltaMaintenance(self._delta_table_ref).run_scheduled(self._schema,is_cdc_companion=self._maintains_companion_table(),diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.pyindex 8cac34d88cc9..0a11de69837e 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.py@@ -463,6 +463,52 @@ class TestReplaceScopeHandOff:assert "other" not in result.column_names+class TestPreWriteMaintenanceBeforeAnAppendRestore:+ @pytest.mark.parametrize(+ "loader_restores, expected_maintained",+ [(True, False), (False, True)],+ ids=["restore_pending", "no_restore"],+ )+ @pytest.mark.asyncio+ async def test_the_pre_write_pass_is_skipped_when_the_first_batch_restores_the_table(+ self, loader_restores: bool, expected_maintained: bool+ ) -> None:+ pipeline = _make_pipeline()+ pipeline._attempt = 2+ pipeline._run_uuid = "run-abc-a2"+ pipeline._schema.table = MagicMock()+ pipeline._schema.sync_type_config = {}+ pipeline._delta_table_ref = MagicMock(is_first_sync=False)+ pipeline._pg_producer = MagicMock(sync_type="append", is_first_ever_sync=False)+ pipeline._resource.items = MagicMock(return_value=iter([]))+ pipeline._batcher.should_yield.return_value = False # type: ignore[attr-defined]++ with ExitStack() as stack:+ for name in (+ "reset_rows_synced_if_needed",+ "setup_row_tracking_with_billing_check",+ "persist_primary_keys",+ "handle_reset_or_full_refresh",+ "handle_corrupted_delta_log",+ ):+ stack.enter_context(patch(f"{_PIPELINE}.{name}", new_callable=AsyncMock))+ stack.enter_context(patch(f"{_PIPELINE}.validate_incremental_sync"))+ maintenance = stack.enter_context(patch(f"{_PIPELINE}.DeltaMaintenance"))+ maintenance.return_value.run_scheduled = AsyncMock()+ append_runs = stack.enter_context(+ patch(+ "products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.load.append_rollback.UnfinishedAppendRuns"+ )+ )+ append_runs.return_value.restores_before_first_batch.return_value = loader_restores+ stack.enter_context(patch(f"{_PIPELINE}.activity")).in_activity.return_value = False++ await pipeline.run()++ append_runs.return_value.restores_before_first_batch.assert_called_once_with("run-abc-a2")+ assert maintenance.return_value.run_scheduled.called is expected_maintained++class TestExtractionFailureDoesNotCleanupS3:@pytest.mark.asyncioasync def test_s3_files_preserved_when_extraction_fails(self) -> None: