:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(data-warehouse): compact rebuild runs between batches (#119723)
Daniel RCc248ffcaf94f
diff --git a/posthog/settings/data_warehouse.py b/posthog/settings/data_warehouse.pyindex 9e7388a1c89d..5fae06142dbe 100644--- a/posthog/settings/data_warehouse.py+++ b/posthog/settings/data_warehouse.py@@ -152,6 +152,11 @@ DATA_WAREHOUSE_FULL_VACUUM_INTERVAL_HOURS = get_from_env(# `optimize.compact`. deltalite streams each bin under the load slot's memory budget. A deltalite build# without `compact`, a table that deltalite refuses, or a deltalite failure falls back to delta-rs.DATA_WAREHOUSE_DELTALITE_COMPACTION = get_from_env("DATA_WAREHOUSE_DELTALITE_COMPACTION", False, type_cast=str_to_bool)+# A rebuild run (first sync, reset, full refresh) compacts its own files once a compaction would remove+# this many, so a long rebuild does not leave one post-load pass that outlasts the stuck-batch timeout.+DATA_WAREHOUSE_REBUILD_COMPACT_MIN_REMOVABLE_FILES = get_from_env(+ "DATA_WAREHOUSE_REBUILD_COMPACT_MIN_REMOVABLE_FILES", 20_000, type_cast=int+)# delta-rs merge spill-to-disk. A merge decompresses the target partition into an Arrow working set that# can exceed the 29 GB pod limit and OOM — killing every co-tenant activity on the pod. When set, delta-rsdiff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/maintenance.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/maintenance.pyindex 6fe69b21db21..dfff4a1f0756 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/maintenance.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/maintenance.py@@ -6,7 +6,7 @@ import datetime as dtimport resourcefrom collections import defaultdictfrom collections.abc import Callable, Iterable, Mapping-from typing import TYPE_CHECKING, Any, cast+from typing import TYPE_CHECKING, Any, Literal, castfrom django.conf import settings@@ -23,6 +23,7 @@ from posthog.sync import database_sync_to_async_poolfrom posthog.utils import get_machine_idfrom products.warehouse_sources.backend.models.external_data_schema import update_sync_type_config_keys+from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.consts import PARTITION_KEYfrom products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.deltalite_handles import (get_handle_cache,)@@ -41,6 +42,9 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delis_object_store_permission_denied,)from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.writer import _delta_table_identity+from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.partitioning import (+ partition_key_from_add_path,+)from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.post_load_phases import (note_post_load_phase,recorded_phase,@@ -104,6 +108,11 @@ COMPACT_MIN_TARGET_SIZE_BYTES = 8 * 1024 * 1024_COMPACT_TASK_MEMORY_FACTOR = 2# Footers of the largest files only: compacted files are the largest, and they set the worst case._COMPACT_RATIO_SAMPLE_FILES = 2+# deltalite commits after this many bins instead of once at the end. A pass that a deploy or the load+# worker's stuck-batch timeout stops then keeps the bins it finished, and the next pass plans only the rest.+COMPACT_MAX_BINS_PER_COMMIT = 16++RebuildCompactOutcome = Literal["compacted", "skipped", "failed"]@frozen@@ -212,6 +221,8 @@ def removable_file_count(file_sizes: Iterable[int], target_size: int = DEFAULT_C@frozenclass _PartitionFileStats:+ # The `_ph_partition_key` value, or None for an unpartitioned table and the null partition.+ key: str | Nonesize_bytes: intremovable_files: int@@ -226,13 +237,31 @@ def _partition_file_stats(sizes[path.rpartition("/")[0]].append(size or 0)return [_PartitionFileStats(+ key=partition_key_from_add_path(directory),size_bytes=sum(partition_sizes),removable_files=removable_file_count(partition_sizes, target_size),)- for partition_sizes in sizes.values()+ for directory, partition_sizes in sizes.items()]+def _partitions_up_to(partitions: list[_PartitionFileStats], removable_files: int) -> list[str] | None:+ """The partitions with the most removable files, until together they reach `removable_files`.++ None means the whole table: it has no partition column, or every partition is needed anyway.+ """+ selected: list[str] = []+ reached = 0+ for partition in sorted(partitions, key=lambda p: p.removable_files, reverse=True):+ if reached >= removable_files:+ break+ if partition.key is None:+ return None+ selected.append(partition.key)+ reached += partition.removable_files+ return selected if len(selected) < len(partitions) else None++@frozenclass _SmallFileFragmentation:fragmented: bool@@ -442,7 +471,9 @@ class DeltaMaintenance:return plan_compaction(ratio, get_governor().slot_budget_mb())@recorded_phase("compact")- async def _compact(self, table: deltalake.DeltaTable, plan: CompactionPlan | None = None) -> bool:+ async def _compact(+ self, table: deltalake.DeltaTable, plan: CompactionPlan | None = None, *, partitions: list[str] | None = None+ ) -> bool:plan = plan or await self._plan_compaction(table)target_size = plan.target_sizeif target_size is None:@@ -469,6 +500,12 @@ class DeltaMaintenance:while True:def _compact_op(size: int = target_size, tasks: int | None = max_concurrent_tasks) -> dict[str, Any]:+ if partitions:+ return table.optimize.compact(+ partition_filters=[(PARTITION_KEY, "in", partitions)],+ target_size=size,+ max_concurrent_tasks=tasks,+ )return table.optimize.compact(target_size=size, max_concurrent_tasks=tasks)try:@@ -507,7 +544,9 @@ class DeltaMaintenance:await self._logger.adebug(json.dumps(compact_stats))return True- async def _compact_with_deltalite(self, table: deltalake.DeltaTable) -> bool | None:+ async def _compact_with_deltalite(+ self, table: deltalake.DeltaTable, *, partitions: list[str] | None = None+ ) -> bool | None:"""Compact through deltalite's native compaction. None means delta-rs must compact instead.deltalite streams each bin under byte budgets and caps every output file by its decoded size,@@ -536,6 +575,8 @@ class DeltaMaintenance:# `has_batch_been_committed` would then read a compaction as a committed batch."commit_metadata": {"compact_engine": "deltalite", "job_id": str(self._table.job.id)},"min_partition_removable_files": 1,+ "max_bins_per_commit": COMPACT_MAX_BINS_PER_COMMIT,+ "partitions": partitions,}def _run() -> dict[str, Any]:@@ -564,11 +605,14 @@ class DeltaMaintenance:await self._log_compact_fallback("unsupported_table")return Noneexcept deltalite.DeltaLiteCommitConflictError:- # The retries ran out, or a concurrent writer changed the schema or the partitioning.- # Nothing was committed, and the next maintenance pass compacts the table again.+ # The retries ran out, or a concurrent writer changed the schema or the partitioning. The+ # chunks committed before the conflict stay, and the next maintenance pass compacts the rest.+ self._table.note_deltalite_commit(None)await self._logger.awarning("compact: deltalite lost to a concurrent commit, not reporting")return Falseexcept Exception as e:+ # A pass that commits in chunks can fail after some of them, so the handle is behind the log.+ self._table.note_deltalite_commit(None)# The object-store classifiers match the error text, but accept only OS and delta-rs errors.as_os_error = OSError(str(e))if is_object_store_permission_denied(as_os_error):@@ -577,8 +621,8 @@ class DeltaMaintenance:if is_transient_object_store_error(as_os_error):await self._logger.awarning("compact: transient object-store error, not reporting")raise TransientObjectStoreError(OBJECT_STORE_TRANSIENT_MESSAGE) from e- # A failed deltalite compaction commits nothing and deletes its output files, so delta-rs- # can compact the same table safely.+ # deltalite deletes the output files that no commit names, so delta-rs can compact the table+ # once the caller rereads the log.capture_exception(e)await self._log_compact_fallback("deltalite_error")return None@@ -749,29 +793,96 @@ class DeltaMaintenance:await self._logger.adebug(f"compact_if_fragmented: skipping ({stats})")return False+ return await self._compact_table(+ table,+ lambda planned: triggered(_small_file_fragmentation(planned, table_wide_small_files)),+ pass_name="compact_if_fragmented",+ stats=stats,+ )++ async def _compact_table(+ self,+ table: deltalake.DeltaTable,+ triggered: Callable[[list[_PartitionFileStats]], bool],+ *,+ pass_name: str,+ stats: str,+ select_partitions: Callable[[list[_PartitionFileStats]], list[str] | None] | None = None,+ ) -> bool:+ """Compact with deltalite when it is on, else with delta-rs.++ `select_partitions` limits the pass, from the partition stats at the bin size the engine uses.+ """+ partitions = None+ if select_partitions is not None:+ partitions = select_partitions(+ await asyncio.to_thread(_partition_file_stats, table, DEFAULT_COMPACT_TARGET_SIZE_BYTES)+ )if settings.DATA_WAREHOUSE_DELTALITE_COMPACTION:- await self._logger.ainfo(f"compact_if_fragmented: triggering compact ({stats})")- compacted = await self._compact_with_deltalite(table)+ await self._logger.ainfo(f"{pass_name}: triggering compact ({stats})")+ compacted = await self._compact_with_deltalite(table, partitions=partitions)if compacted is not None:return compacted+ reread = await self._table.get_delta_table()+ if reread is None:+ return False+ table = rereadplan = await self._plan_compaction(table)if plan.target_size is None:- await self._compact(table, plan)+ await self._compact(table, plan, partitions=partitions)return Falseif plan.target_size != DEFAULT_COMPACT_TARGET_SIZE_BYTES:# A smaller bin can merge fewer of the files, so the trigger must hold at the planned size too.- partitions = await asyncio.to_thread(_partition_file_stats, table, plan.target_size)- if not triggered(_small_file_fragmentation(partitions, table_wide_small_files)):+ planned = await asyncio.to_thread(_partition_file_stats, table, plan.target_size)+ if not triggered(planned):await self._logger.adebug(- f"compact_if_fragmented: skipping; no files are removable at planned target_size="- f"{plan.target_size} ({stats})"+ f"{pass_name}: skipping; no files are removable at planned target_size={plan.target_size} ({stats})")return False+ if select_partitions is not None:+ partitions = select_partitions(planned)- await self._logger.ainfo(f"compact_if_fragmented: triggering compact ({stats})")- return await self._compact(table, plan)+ await self._logger.ainfo(f"{pass_name}: triggering compact ({stats})")+ return await self._compact(table, plan, partitions=partitions)++ async def compact_during_rebuild(self, min_removable_files: int) -> RebuildCompactOutcome:+ """Compact a table that a rebuild run is still loading, once a compaction removes enough files.++ A rebuild run loads the table from empty, so every file here is one that the same run wrote.+ One pass compacts only the partitions with the most removable files, up to+ `min_removable_files`, so a backlog takes several passes instead of one long batch. Never+ raises an `Exception`: the batch is committed, and the next check or the post-load pass+ compacts the same files.+ """+ try:+ table = await self._table.get_delta_table()+ if table is None:+ return "skipped"++ partitions = await asyncio.to_thread(_partition_file_stats, table, DEFAULT_COMPACT_TARGET_SIZE_BYTES)+ removable_files = _small_file_fragmentation(partitions, table_wide_small_files=False).total_removable+ if removable_files < min_removable_files:+ return "skipped"++ compacted = await self._compact_table(+ table,+ lambda planned: (+ _small_file_fragmentation(planned, table_wide_small_files=False).total_removable+ >= min_removable_files+ ),+ pass_name="compact_during_rebuild",+ stats=(+ f"removable_files={removable_files}, partitions={len(partitions)}, "+ f"min_removable_files={min_removable_files}"+ ),+ select_partitions=lambda stats: _partitions_up_to(stats, min_removable_files),+ )+ return "compacted" if compacted else "skipped"+ except Exception as e:+ await self._log_maintenance_failure(e, step="compact_during_rebuild")+ return "failed"def _vacuumed_during_job(self, watermarks: VacuumWatermarks) -> bool:# The pre-write pass vacuums before it writes, and it never checks small files. The post-loaddiff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_maintenance.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_maintenance.pyindex af441ee92617..05809fb43526 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_maintenance.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_maintenance.py@@ -18,6 +18,7 @@ import deltaliteimport pyarrow.parquet as pqfrom parameterized import parameterized+from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.consts import PARTITION_KEYfrom products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.deltalite_handles import (DeltaLiteHandleCache,)@@ -26,6 +27,7 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.del)from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.maintenance import (_COMPACT_RATIO_SAMPLE_FILES,+ COMPACT_MAX_BINS_PER_COMMIT,COMPACT_OFFSET_OVERFLOW_RETRIES,DEFAULT_COMPACT_TARGET_SIZE_BYTES,DELTA_DELETED_FILE_RETENTION,@@ -41,7 +43,10 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delfrom products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.ops import (ObjectStorePermissionDeniedError,)-from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.test.helpers import make_logger+from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.test.helpers import (+ make_local_table_ref,+ make_logger,+)_MAINTENANCE_MODULE = "products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.maintenance"_MB = 1024 * 1024@@ -640,6 +645,8 @@ class TestDeltaliteCompaction:slot_budget_bytes=int(self._SLOT_MB * _MB),commit_metadata={"compact_engine": "deltalite", "job_id": "job-1"},min_partition_removable_files=1,+ max_bins_per_commit=COMPACT_MAX_BINS_PER_COMMIT,+ partitions=None,)assert [call.args[0] for call in table_ref.note_deltalite_commit.call_args_list] == expected_noted[done] = self._info_calls(table_ref, "compact: done")@@ -692,11 +699,15 @@ class TestDeltaliteCompaction:) -> None:compact = MagicMock(side_effect=error) if has_compact else Nonemaintenance, table_ref, fake_class = self._setup(compact)+ first = table_ref.get_delta_table.return_value+ refreshed = _mock_table({f"f{i}.parquet": _MB for i in range(300)})+ table_ref.get_delta_table = AsyncMock(side_effect=[first, refreshed])ran, delta_rs_compact, _, capture = await self._run(maintenance, fake_class, enabled=enabled)assert ran is True- delta_rs_compact.assert_awaited_once_with(table_ref.get_delta_table.return_value, _DEFAULT_PLAN)+ delta_rs_compact.assert_awaited_once_with(refreshed if enabled else first, _DEFAULT_PLAN, partitions=None)+ assert table_ref.note_deltalite_commit.called is (expected_reason == "deltalite_error")fallbacks = self._info_calls(table_ref, "compact: falling back to delta-rs")assert [call["compact_fallback_reason"] for call in fallbacks] == ([expected_reason] if expected_reason else [])assert capture.called is expect_capture@@ -713,7 +724,7 @@ class TestDeltaliteCompaction:assert ran is Falsedelta_rs_compact.assert_not_awaited()capture.assert_not_called()- table_ref.note_deltalite_commit.assert_not_called()+ table_ref.note_deltalite_commit.assert_called_once_with(None)@parameterized.expand([@@ -740,6 +751,122 @@ class TestDeltaliteCompaction:assert isinstance(raised.value.__cause__, deltalite.DeltaLiteError)+class TestCompactDuringRebuild:+ @staticmethod+ def _fill(path: str, partitions: int, files_per_partition: int) -> None:+ for part in range(partitions):+ for i in range(files_per_partition):+ deltalake.write_deltalake(+ path,+ pa.table({PARTITION_KEY: [str(part)], "id": [part * 1000 + i]}),+ mode="append",+ partition_by=[PARTITION_KEY],+ )++ @parameterized.expand(+ [+ # Seven tiny files per partition: a compaction removes six files from each.+ ("below_the_budget_delta_rs", False, 2, 13, "skipped", 14),+ ("at_the_budget_delta_rs", False, 2, 12, "compacted", 2),+ ("below_the_budget_deltalite", True, 2, 13, "skipped", 14),+ ("at_the_budget_deltalite", True, 2, 12, "compacted", 2),+ # A backlog over the budget compacts only the partitions that reach it.+ ("backlog_capped_delta_rs", False, 4, 12, "compacted", 2 + 2 * 7),+ ("backlog_capped_deltalite", True, 4, 12, "compacted", 2 + 2 * 7),+ ]+ )+ @pytest.mark.asyncio+ async def test_compacts_up_to_the_budget(+ self, _name: str, deltalite_engine: bool, partitions: int, budget: int, outcome: str, files_after: int+ ):+ with (+ tempfile.TemporaryDirectory() as path,+ override_settings(DATA_WAREHOUSE_DELTALITE_COMPACTION=deltalite_engine),+ ):+ self._fill(path, partitions=partitions, files_per_partition=7)+ rows_before = deltalake.DeltaTable(path).to_pyarrow_table().sort_by("id").to_pydict()++ result = await DeltaMaintenance(make_local_table_ref(path)).compact_during_rebuild(+ min_removable_files=budget+ )++ table = deltalake.DeltaTable(path)+ assert result == outcome+ assert len(table.file_uris()) == files_after+ assert table.to_pyarrow_table().sort_by("id").to_pydict() == rows_before++ @pytest.mark.asyncio+ async def test_a_smaller_planned_bin_reselects_the_partitions(self):+ # At 100 MiB every file is small. At the planned 300 KB, partition "a" holds only files of about+ # 160 KB, which a compaction cannot merge, so the pass must move to partition "b".+ with tempfile.TemporaryDirectory() as path:+ for i in range(8):+ deltalake.write_deltalake(+ path,+ pa.table({PARTITION_KEY: ["a"], "id": [i], "blob": [os.urandom(80_000).hex()]}),+ mode="append",+ partition_by=[PARTITION_KEY],+ )+ for i in range(7):+ deltalake.write_deltalake(+ path,+ pa.table({PARTITION_KEY: ["b"], "id": [100 + i], "blob": ["x"]}),+ mode="append",+ partition_by=[PARTITION_KEY],+ )+ maintenance = DeltaMaintenance(make_local_table_ref(path))+ plan = CompactionPlan(+ target_size=300_000, max_concurrent_tasks=1, compression_ratio=1.0, slot_budget_mb=1356.8+ )++ with patch.object(maintenance, "_plan_compaction", AsyncMock(return_value=plan)):+ result = await maintenance.compact_during_rebuild(min_removable_files=6)++ files = deltalake.DeltaTable(path).file_uris()+ assert result == "compacted"+ assert (+ sum(f"{PARTITION_KEY}=a/" in uri for uri in files),+ sum(f"{PARTITION_KEY}=b/" in uri for uri in files),+ ) == (8, 1)++ @parameterized.expand(+ [+ ("transient_object_store_error", TransientObjectStoreError("throttled"), False),+ ("unexpected_error", RuntimeError("boom"), True),+ ]+ )+ @pytest.mark.asyncio+ async def test_a_failed_pass_is_reported_and_never_raises(self, _name: str, error: Exception, expect_capture: bool):+ with (+ tempfile.TemporaryDirectory() as path,+ patch.object(DeltaMaintenance, "_compact", AsyncMock(side_effect=error)),+ patch(f"{_MAINTENANCE_MODULE}.capture_exception") as capture,+ ):+ self._fill(path, partitions=2, files_per_partition=7)++ result = await DeltaMaintenance(make_local_table_ref(path)).compact_during_rebuild(min_removable_files=1)++ assert result == "failed"+ assert capture.called is expect_capture++ @pytest.mark.asyncio+ async def test_deltalite_commits_a_long_compaction_in_chunks(self):+ with (+ tempfile.TemporaryDirectory() as path,+ override_settings(DATA_WAREHOUSE_DELTALITE_COMPACTION=True),+ patch(f"{_MAINTENANCE_MODULE}.COMPACT_MAX_BINS_PER_COMMIT", 1),+ ):+ self._fill(path, partitions=3, files_per_partition=4)+ version_before = deltalake.DeltaTable(path).version()++ result = await DeltaMaintenance(make_local_table_ref(path)).compact_during_rebuild(min_removable_files=9)++ table = deltalake.DeltaTable(path)+ assert result == "compacted"+ assert table.version() - version_before == 3+ assert len(table.file_uris()) == 3++class TestVacuum:@pytest.mark.asyncioasync def test_retries_on_commit_conflict_then_succeeds(self):diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_writer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_writer.pyindex d3be830e0453..c2efa4fa8ffc 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_writer.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_writer.py@@ -201,6 +201,28 @@ class TestHasBatchBeenCommitted:with patch.object(writer._table, "get_delta_table", AsyncMock(return_value=table)):assert await writer.has_batch_been_committed(run_uuid, batch_index) is expected+ @parameterized.expand(+ [+ ("compaction_commits_do_not_use_up_the_window", [("OPTIMIZE", 120)], True),+ ("vacuum_commits_do_not_use_up_the_window", [("VACUUM END", 120)], True),+ ("data_commits_still_close_the_window", [("WRITE", 60)], False),+ ("recent_writes_ahead_of_a_long_compaction", [("WRITE", 49), ("OPTIMIZE", 100)], True),+ ]+ )+ @pytest.mark.asyncio+ async def test_finds_the_batch_behind_later_commits(+ self, _name: str, later: list[tuple[str, int]], expected: bool+ ) -> None:+ commits = [{"operation": operation} for operation, count in later for _ in range(count)] + [+ {"operation": "WRITE", "run_uuid": "run-1", "batch_index": "5"}+ ]+ writer = _make_writer()+ table = MagicMock()+ table.history = MagicMock(side_effect=lambda limit: commits[:limit])+ with patch.object(writer._table, "get_delta_table", AsyncMock(return_value=table)):+ assert await writer.has_batch_been_committed("run-1", 5) is expected+ assert table.history.call_count <= 3+class TestWriteToDeltalakeCommitMetadataPassThrough:"""Covers that commit_metadata is forwarded to deltalake.write_deltalake as CommitProperties."""diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/writer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/writer.pyindex 0601dcaed0a1..47d405a8e24b 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/writer.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/writer.py@@ -161,6 +161,10 @@ def commit_members_tag(members: Sequence[tuple[str, int]]) -> str:return ",".join(f"{run_uuid}:{batch_index}" for run_uuid, batch_index in members)+_MAINTENANCE_OPERATIONS = frozenset({"OPTIMIZE", "VACUUM START", "VACUUM END"})+_MAX_HISTORY_SCAN = 2000++def commit_covers_batch(commit: dict[str, Any], run_uuid: str, batch_index: int) -> bool:"""Whether this commit wrote `batch_index` of `run_uuid`, alone or as one member of a set.@@ -759,10 +763,21 @@ class DeltaWriter:async def has_batch_been_committed(self, run_uuid: str, batch_index: int, *, scan_limit: int = 50) -> bool:"""Check whether a specific (run_uuid, batch_index) has already been committed to delta,- on its own or as a member of a coalesced write (see `commit_covers_batch`)."""+ on its own or as a member of a coalesced write (see `commit_covers_batch`).++ The scan covers the last `scan_limit` commits that wrote data. A chunked compaction can commit+ many times after the batch write, so compaction and vacuum commits do not use up the window.+ """delta_table = await self._table.get_delta_table()if delta_table is None:return False- history = await asyncio.to_thread(delta_table.history, limit=scan_limit)- return any(commit_covers_batch(commit, run_uuid, batch_index) for commit in history)+ limit = scan_limit+ while True:+ history = await asyncio.to_thread(delta_table.history, limit=limit)+ if any(commit_covers_batch(commit, run_uuid, batch_index) for commit in history):+ return True+ data_commits = sum(1 for commit in history if commit.get("operation") not in _MAINTENANCE_OPERATIONS)+ if len(history) < limit or data_commits >= scan_limit or limit >= _MAX_HISTORY_SCAN:+ return False+ limit = min(limit * 2, _MAX_HISTORY_SCAN)diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/partitioning.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/partitioning.pyindex 81a77af20f35..be9ab8112b97 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/partitioning.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/partitioning.py@@ -307,3 +307,23 @@ def append_partition_key_to_table(partition_format=partition_format,partition_keys=normalized_partition_keys,)+++# Delta's fixed marker for a Hive-style partition directory whose column value is null.+HIVE_NULL_PARTITION = "__HIVE_DEFAULT_PARTITION__"+++def partition_key_from_add_path(path: str) -> str | None:+ """Recover the `_ph_partition_key` value from a Hive-style partitioned add-action file path.++ Every value this pipeline actually partitions by — an md5 bucket index, a numerical bucket or the+ `null` sentinel, or a datetime tier like "2024-01" — is plain ASCII with nothing that needs+ percent-encoding, so splitting the path recovers the same value `get_add_actions`'s+ `partition.<key>` column would give.+ """+ prefix = f"{PARTITION_KEY}="+ for segment in path.split("/"):+ if segment.startswith(prefix):+ value = segment[len(prefix) :]+ return None if value == HIVE_NULL_PARTITION else value+ return Nonediff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/repartition.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/repartition.pyindex 5f45859e1a60..e6fa217ebfff 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/repartition.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/repartition.py@@ -52,6 +52,7 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delfrom products.warehouse_sources.backend.temporal.data_imports.pipelines.core.partitioning import (NULL_NUMERICAL_PARTITION,append_partition_key_to_table,+ partition_key_from_add_path,)from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.repartition_stream import (PartitionedFileWriter,@@ -266,26 +267,6 @@ class RepartitionTarget:return cls(**{k: v for k, v in data.items() if k in fields})-# Delta's fixed marker for a Hive-style partition directory whose column value is null.-_HIVE_NULL_PARTITION = "__HIVE_DEFAULT_PARTITION__"---def _partition_key_from_add_path(path: str) -> str | None:- """Recover the `_ph_partition_key` value from a Hive-style partitioned add-action file path.-- Every value this pipeline actually partitions by — an md5 bucket index, a numerical bucket or the- `null` sentinel, or a datetime tier like "2024-01" — is plain ASCII with nothing that needs- percent-encoding, so splitting the path recovers the same value `get_add_actions`'s- `partition.<key>` column would give.- """- prefix = f"{PARTITION_KEY}="- for segment in path.split("/"):- if segment.startswith(prefix):- value = segment[len(prefix) :]- return None if value == _HIVE_NULL_PARTITION else value- return None--def measure_partition_bytes(delta_table: deltalake.DeltaTable) -> dict[str | None, int]:"""At-rest bytes per partition, read from the Delta log (no S3 LIST, no data scan).@@ -301,7 +282,7 @@ def measure_partition_bytes(delta_table: deltalake.DeltaTable) -> dict[str | Nontotals: dict[str | None, int] = defaultdict(int)for path, size in delta_table._table.get_add_file_sizes().items():- key = _partition_key_from_add_path(path) if partitioned else None+ key = partition_key_from_add_path(path) if partitioned else Nonetotals[key] += size or 0return dict(totals)diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/batch_steps.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/batch_steps.pyindex 83df44d4152b..4bee31597887 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/batch_steps.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/batch_steps.py@@ -21,6 +21,7 @@ BatchStep = Literal["write","mark_processed","post_write",+ "compact_during_rebuild",]diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/metrics.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/metrics.pyindex 794cc326c014..4ea7b7c079b3 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/metrics.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/metrics.py@@ -23,6 +23,12 @@ APPEND_ROLLBACK_TOTAL = Counter(labelnames=["outcome"],)+DELTA_COMPACT_DURING_REBUILD_TOTAL = Counter(+ "warehouse_load_delta_compact_during_rebuild_total",+ "Compaction checks between the batches of a rebuild run, by outcome",+ labelnames=["outcome"],+)+PARQUET_READ_DURATION_SECONDS = Histogram("warehouse_load_parquet_read_duration_seconds","Duration of S3 parquet file reads",diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/processor.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/processor.pyindex f410e10c8bad..07dce69f7c86 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/processor.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/processor.py@@ -53,6 +53,7 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.autmaybe_schedule_auto_widen_resync,)from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.consts import PARTITION_KEY+from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.maintenance import DeltaMaintenancefrom products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.memory_governor import get_governorfrom products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.rss_sampler import RssPeakSamplerfrom products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.scd2 import Scd2DeltaWriter@@ -91,6 +92,7 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.pipelinefrom products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.load.metrics import (CDC_DELETE_ENRICHMENT_VIOLATIONS_TOTAL,CDC_SEQ_GUARD_ROWS_DROPPED_TOTAL,+ DELTA_COMPACT_DURING_REBUILD_TOTAL,DELTA_ROWS_WRITTEN_TOTAL,DELTA_WRITE_DURATION_SECONDS,IDEMPOTENCY_HIT_TOTAL,@@ -112,6 +114,7 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline)from products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.table_rebuild import TableRebuildRunfrom products.warehouse_sources.backend.temporal.data_imports.row_tracking import finish_row_tracking+from products.warehouse_sources.backend.temporal.data_imports.schema_flags import is_schema_flag_enabledfrom products.warehouse_sources.backend.temporal.data_imports.sources import SourceRegistryfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.cursor import merge_cursor_payloadsfrom products.warehouse_sources.backend.temporal.data_imports.util import prepare_s3_files_for_querying@@ -125,6 +128,9 @@ logger = structlog.get_logger(__name__)# show a table fragmenting over a multi-hundred-batch load, rare enough that the listing it needs# stays off the per-batch path.FILE_COUNT_LOG_SAMPLE_EVERY = 100+# The check lists the add-file sizes of the open snapshot, O(files in table), so it skips most batches.+REBUILD_COMPACT_CHECK_EVERY = 10+REBUILD_COMPACTION_FLAG = "warehouse-rebuild-run-compaction"def _get_write_type(sync_type: SyncTypeLiteral) -> Literal["incremental", "full_refresh", "append"]:@@ -350,6 +356,27 @@ def _partial_data_loading_applies(export_signal: ExportSignalMessage, schema: Exreturn export_signal.is_first_ever_sync and supports_partial_data_loading(schema)+def _rebuild_compaction_due(+ export_signal: ExportSignalMessage, schema: ExternalDataSchema, members: list[tuple[str, int]]+) -> bool:+ """Whether this write of a rebuild run checks the table for compaction (see `compact_during_rebuild`).++ A rebuild run (first sync, reset, full refresh) loads the table from empty, so a compaction+ rewrites only files of the same run. A run that adds to an existing table would compact files+ from before it, which the rollback of an unfinished append run restores.+ Not for CDC: each CDC run is short and ends with its own post-load pass.+ Not for partial data loading: it publishes the files of each batch, and a compaction replaces them.+ """+ return (+ (export_signal.is_first_ever_sync or export_signal.sync_type == "full_refresh")+ and export_signal.cdc_write_mode is None+ and not _partial_data_loading_applies(export_signal, schema)+ and all(run_uuid == export_signal.run_uuid for run_uuid, _ in members)+ and any(index > 0 and index % REBUILD_COMPACT_CHECK_EVERY == 0 for _, index in members)+ and is_schema_flag_enabled(schema, REBUILD_COMPACTION_FLAG)+ )++async def _handle_partial_data_loading(export_signal: ExportSignalMessage,job: ExternalDataJob,@@ -1078,6 +1105,19 @@ def _read_constituents(constituents: list[ExportSignalMessage]) -> pa.Table:raise CoalescingDeclined(f"batch schemas do not concatenate: {e}") from e+def _compact_during_rebuild(+ delta_table_ref: DeltaTableRef, verify_ownership: Callable[[], None] | None, timer: BatchStepTimer+) -> None:+ if verify_ownership is not None:+ verify_ownership()+ report_phase("compact_during_rebuild")+ with timer.step("compact_during_rebuild"):+ outcome = async_to_sync(DeltaMaintenance(delta_table_ref).compact_during_rebuild)(+ settings.DATA_WAREHOUSE_REBUILD_COMPACT_MIN_REMOVABLE_FILES+ )+ DELTA_COMPACT_DURING_REBUILD_TOTAL.labels(outcome=outcome).inc()++def _process_message_reported(message: Any,export_signal: "ExportSignalMessage",@@ -1466,6 +1506,8 @@ def _process_message_reported(finals = final_run_signals(constituents)else:finals = [export_signal] if export_signal.is_final_batch else []+ if not finals and _rebuild_compaction_due(export_signal, schema, members):+ _compact_during_rebuild(delta_table_ref, verify_ownership, timer)for run_signal in finals:run_job = job if run_signal.job_id == export_signal.job_id else _load_job(run_signal.job_id)run_schema = run_job.schemadiff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/test_processor.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/test_processor.pyindex 6f60600361ed..e2bb8d598a33 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/test_processor.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/test_processor.py@@ -7,6 +7,8 @@ from typing import Anyimport pytestfrom unittest.mock import ANY, AsyncMock, MagicMock, patch+from django.test import override_settings+import pyarrow as pafrom asgiref.sync import async_to_syncfrom deltalake import DeltaTable, write_deltalake@@ -42,6 +44,7 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_mark_job_completed,_promote_staged_cursor,_read_existing_rows_by_first_pk,+ _rebuild_compaction_due,_resolve_cdc_positions,_run_post_load_for_already_processed_batch,_trigger_post_import_workflow,@@ -51,7 +54,9 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.pipelinefrom products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.load.table_handles import (GroupTableHandles,)+from products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.messages import ExportSignalMessagefrom products.warehouse_sources.backend.temporal.data_imports.pipelines.test_mocks import mock_delta_table+from products.warehouse_sources.backend.types import ExternalDataSourceTypefrom products.warehouse_sources_queue.backend.core.batch_consumer import CoalescingDeclined@@ -1990,3 +1995,82 @@ class TestTableHandleReuse:rows = DeltaTable(tmp).to_pyarrow_table()assert rows.sort_by("id").column("id").to_pylist() == [0, 1, 4, 5, 6, 6, 7, 8, 100]assert rows.schema.names == ["id", "name", "added"]+++class TestCompactionDuringRebuild:+ @staticmethod+ def _schema(source_type: str = "Postgres") -> MagicMock:+ schema = MagicMock()+ schema.source.source_type = source_type+ return schema++ @parameterized.expand(+ [+ ("full_refresh", {"sync_type": "full_refresh"}, None, None, True),+ ("first_sync_incremental", {"is_first_ever_sync": True}, None, None, True),+ ("first_sync_append", {"sync_type": "append", "is_first_ever_sync": True}, None, None, True),+ ("incremental_on_an_existing_table", {}, None, None, False),+ ("append_on_an_existing_table", {"sync_type": "append"}, None, None, False),+ ("cdc", {"is_first_ever_sync": True, "cdc_write_mode": "upsert"}, None, None, False),+ ("partial_data_loading", {"is_first_ever_sync": True}, ExternalDataSourceType.STRIPE, None, False),+ ("set_spanning_two_runs", {"sync_type": "full_refresh"}, None, [("run-0", 9), ("run-1", 10)], False),+ (+ "set_crossing_a_check",+ {"sync_type": "full_refresh", "batch_index": 9},+ None,+ [("run-1", 9), ("run-1", 10)],+ True,+ ),+ ("between_checks", {"sync_type": "full_refresh", "batch_index": 11}, None, None, False),+ ("first_batch", {"sync_type": "full_refresh", "batch_index": 0}, None, None, False),+ ]+ )+ def test_which_writes_check_for_compaction(+ self,+ _name: str,+ overrides: dict[str, Any],+ source_type: str | None,+ members: list[tuple[str, int]] | None,+ expected: bool,+ ) -> None:+ signal = ExportSignalMessage.from_dict(_message(**{"batch_index": 10, **overrides}))+ members = members or [(signal.run_uuid, signal.batch_index)]+ with patch(f"{_PROCESSOR}.is_schema_flag_enabled", return_value=True):+ assert _rebuild_compaction_due(signal, self._schema(source_type or "Postgres"), members) is expected++ def test_the_flag_turns_it_off(self) -> None:+ signal = ExportSignalMessage.from_dict(_message(batch_index=10, sync_type="full_refresh"))+ with patch(f"{_PROCESSOR}.is_schema_flag_enabled", return_value=False):+ assert _rebuild_compaction_due(signal, self._schema(), [(signal.run_uuid, 10)]) is False++ def _load(self, table_path: str, sync_type: str, first_index: int, *, enabled: bool) -> None:+ write_deltalake(table_path, TestTableHandleReuse._SEED)+ with (+ override_settings(DATA_WAREHOUSE_REBUILD_COMPACT_MIN_REMOVABLE_FILES=3),+ patch(f"{_PROCESSOR}.REBUILD_COMPACT_CHECK_EVERY", 5),+ TestTableHandleReuse()._loader(table_path, []),+ patch(f"{_PROCESSOR}.is_schema_flag_enabled", return_value=enabled),+ ):+ for index in range(first_index, first_index + 12):+ process_message(TestTableHandleReuse._batch(index, sync_type=sync_type))++ @parameterized.expand(+ [+ ("rebuild_run_compacts_as_it_loads", "full_refresh", 0, True),+ ("append_on_an_existing_table_waits_for_post_load", "append", 1, False),+ ]+ )+ def test_compaction_between_batches_keeps_the_rows(+ self, _name: str, sync_type: str, first_index: int, compacts: bool+ ) -> None:+ with tempfile.TemporaryDirectory() as tmp:+ on_path, off_path = f"{tmp}/on", f"{tmp}/off"+ self._load(on_path, sync_type, first_index, enabled=True)+ self._load(off_path, sync_type, first_index, enabled=False)++ on_table, off_table = DeltaTable(on_path), DeltaTable(off_path)+ assert (+ on_table.to_pyarrow_table().sort_by([("id", "ascending"), ("name", "ascending")]).to_pydict()+ == off_table.to_pyarrow_table().sort_by([("id", "ascending"), ("name", "ascending")]).to_pydict()+ )+ assert (len(on_table.file_uris()) < len(off_table.file_uris())) is compacts