Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-warehouse): compact rebuild runs between batches (#119723)

Daniel RCc248ffcaf94f
diff --git a/posthog/settings/data_warehouse.py b/posthog/settings/data_warehouse.py
index 9e7388a1c89d..5fae06142dbe 100644
--- a/posthog/settings/data_warehouse.py
+++ b/posthog/settings/data_warehouse.py
@@ -152,6 +152,11 @@ DATA_WAREHOUSE_FULL_VACUUM_INTERVAL_HOURS = get_from_env(
# `optimize.compact`. deltalite streams each bin under the load slot's memory budget. A deltalite build
# without `compact`, a table that deltalite refuses, or a deltalite failure falls back to delta-rs.
DATA_WAREHOUSE_DELTALITE_COMPACTION = get_from_env("DATA_WAREHOUSE_DELTALITE_COMPACTION", False, type_cast=str_to_bool)
+# A rebuild run (first sync, reset, full refresh) compacts its own files once a compaction would remove
+# this many, so a long rebuild does not leave one post-load pass that outlasts the stuck-batch timeout.
+DATA_WAREHOUSE_REBUILD_COMPACT_MIN_REMOVABLE_FILES = get_from_env(
+ "DATA_WAREHOUSE_REBUILD_COMPACT_MIN_REMOVABLE_FILES", 20_000, type_cast=int
+)
# delta-rs merge spill-to-disk. A merge decompresses the target partition into an Arrow working set that
# can exceed the 29 GB pod limit and OOM — killing every co-tenant activity on the pod. When set, delta-rs
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/maintenance.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/maintenance.py
index 6fe69b21db21..dfff4a1f0756 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/maintenance.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/maintenance.py
@@ -6,7 +6,7 @@ import datetime as dt
import resource
from collections import defaultdict
from collections.abc import Callable, Iterable, Mapping
-from typing import TYPE_CHECKING, Any, cast
+from typing import TYPE_CHECKING, Any, Literal, cast
from django.conf import settings
@@ -23,6 +23,7 @@ from posthog.sync import database_sync_to_async_pool
from posthog.utils import get_machine_id
from products.warehouse_sources.backend.models.external_data_schema import update_sync_type_config_keys
+from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.consts import PARTITION_KEY
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.deltalite_handles import (
get_handle_cache,
)
@@ -41,6 +42,9 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.del
is_object_store_permission_denied,
)
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.writer import _delta_table_identity
+from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.partitioning import (
+ partition_key_from_add_path,
+)
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.post_load_phases import (
note_post_load_phase,
recorded_phase,
@@ -104,6 +108,11 @@ COMPACT_MIN_TARGET_SIZE_BYTES = 8 * 1024 * 1024
_COMPACT_TASK_MEMORY_FACTOR = 2
# Footers of the largest files only: compacted files are the largest, and they set the worst case.
_COMPACT_RATIO_SAMPLE_FILES = 2
+# deltalite commits after this many bins instead of once at the end. A pass that a deploy or the load
+# worker's stuck-batch timeout stops then keeps the bins it finished, and the next pass plans only the rest.
+COMPACT_MAX_BINS_PER_COMMIT = 16
+
+RebuildCompactOutcome = Literal["compacted", "skipped", "failed"]
@frozen
@@ -212,6 +221,8 @@ def removable_file_count(file_sizes: Iterable[int], target_size: int = DEFAULT_C
@frozen
class _PartitionFileStats:
+ # The `_ph_partition_key` value, or None for an unpartitioned table and the null partition.
+ key: str | None
size_bytes: int
removable_files: int
@@ -226,13 +237,31 @@ def _partition_file_stats(
sizes[path.rpartition("/")[0]].append(size or 0)
return [
_PartitionFileStats(
+ key=partition_key_from_add_path(directory),
size_bytes=sum(partition_sizes),
removable_files=removable_file_count(partition_sizes, target_size),
)
- for partition_sizes in sizes.values()
+ for directory, partition_sizes in sizes.items()
]
+def _partitions_up_to(partitions: list[_PartitionFileStats], removable_files: int) -> list[str] | None:
+ """The partitions with the most removable files, until together they reach `removable_files`.
+
+ None means the whole table: it has no partition column, or every partition is needed anyway.
+ """
+ selected: list[str] = []
+ reached = 0
+ for partition in sorted(partitions, key=lambda p: p.removable_files, reverse=True):
+ if reached >= removable_files:
+ break
+ if partition.key is None:
+ return None
+ selected.append(partition.key)
+ reached += partition.removable_files
+ return selected if len(selected) < len(partitions) else None
+
+
@frozen
class _SmallFileFragmentation:
fragmented: bool
@@ -442,7 +471,9 @@ class DeltaMaintenance:
return plan_compaction(ratio, get_governor().slot_budget_mb())
@recorded_phase("compact")
- async def _compact(self, table: deltalake.DeltaTable, plan: CompactionPlan | None = None) -> bool:
+ async def _compact(
+ self, table: deltalake.DeltaTable, plan: CompactionPlan | None = None, *, partitions: list[str] | None = None
+ ) -> bool:
plan = plan or await self._plan_compaction(table)
target_size = plan.target_size
if target_size is None:
@@ -469,6 +500,12 @@ class DeltaMaintenance:
while True:
def _compact_op(size: int = target_size, tasks: int | None = max_concurrent_tasks) -> dict[str, Any]:
+ if partitions:
+ return table.optimize.compact(
+ partition_filters=[(PARTITION_KEY, "in", partitions)],
+ target_size=size,
+ max_concurrent_tasks=tasks,
+ )
return table.optimize.compact(target_size=size, max_concurrent_tasks=tasks)
try:
@@ -507,7 +544,9 @@ class DeltaMaintenance:
await self._logger.adebug(json.dumps(compact_stats))
return True
- async def _compact_with_deltalite(self, table: deltalake.DeltaTable) -> bool | None:
+ async def _compact_with_deltalite(
+ self, table: deltalake.DeltaTable, *, partitions: list[str] | None = None
+ ) -> bool | None:
"""Compact through deltalite's native compaction. None means delta-rs must compact instead.
deltalite streams each bin under byte budgets and caps every output file by its decoded size,
@@ -536,6 +575,8 @@ class DeltaMaintenance:
# `has_batch_been_committed` would then read a compaction as a committed batch.
"commit_metadata": {"compact_engine": "deltalite", "job_id": str(self._table.job.id)},
"min_partition_removable_files": 1,
+ "max_bins_per_commit": COMPACT_MAX_BINS_PER_COMMIT,
+ "partitions": partitions,
}
def _run() -> dict[str, Any]:
@@ -564,11 +605,14 @@ class DeltaMaintenance:
await self._log_compact_fallback("unsupported_table")
return None
except deltalite.DeltaLiteCommitConflictError:
- # The retries ran out, or a concurrent writer changed the schema or the partitioning.
- # Nothing was committed, and the next maintenance pass compacts the table again.
+ # The retries ran out, or a concurrent writer changed the schema or the partitioning. The
+ # chunks committed before the conflict stay, and the next maintenance pass compacts the rest.
+ self._table.note_deltalite_commit(None)
await self._logger.awarning("compact: deltalite lost to a concurrent commit, not reporting")
return False
except Exception as e:
+ # A pass that commits in chunks can fail after some of them, so the handle is behind the log.
+ self._table.note_deltalite_commit(None)
# The object-store classifiers match the error text, but accept only OS and delta-rs errors.
as_os_error = OSError(str(e))
if is_object_store_permission_denied(as_os_error):
@@ -577,8 +621,8 @@ class DeltaMaintenance:
if is_transient_object_store_error(as_os_error):
await self._logger.awarning("compact: transient object-store error, not reporting")
raise TransientObjectStoreError(OBJECT_STORE_TRANSIENT_MESSAGE) from e
- # A failed deltalite compaction commits nothing and deletes its output files, so delta-rs
- # can compact the same table safely.
+ # deltalite deletes the output files that no commit names, so delta-rs can compact the table
+ # once the caller rereads the log.
capture_exception(e)
await self._log_compact_fallback("deltalite_error")
return None
@@ -749,29 +793,96 @@ class DeltaMaintenance:
await self._logger.adebug(f"compact_if_fragmented: skipping ({stats})")
return False
+ return await self._compact_table(
+ table,
+ lambda planned: triggered(_small_file_fragmentation(planned, table_wide_small_files)),
+ pass_name="compact_if_fragmented",
+ stats=stats,
+ )
+
+ async def _compact_table(
+ self,
+ table: deltalake.DeltaTable,
+ triggered: Callable[[list[_PartitionFileStats]], bool],
+ *,
+ pass_name: str,
+ stats: str,
+ select_partitions: Callable[[list[_PartitionFileStats]], list[str] | None] | None = None,
+ ) -> bool:
+ """Compact with deltalite when it is on, else with delta-rs.
+
+ `select_partitions` limits the pass, from the partition stats at the bin size the engine uses.
+ """
+ partitions = None
+ if select_partitions is not None:
+ partitions = select_partitions(
+ await asyncio.to_thread(_partition_file_stats, table, DEFAULT_COMPACT_TARGET_SIZE_BYTES)
+ )
if settings.DATA_WAREHOUSE_DELTALITE_COMPACTION:
- await self._logger.ainfo(f"compact_if_fragmented: triggering compact ({stats})")
- compacted = await self._compact_with_deltalite(table)
+ await self._logger.ainfo(f"{pass_name}: triggering compact ({stats})")
+ compacted = await self._compact_with_deltalite(table, partitions=partitions)
if compacted is not None:
return compacted
+ reread = await self._table.get_delta_table()
+ if reread is None:
+ return False
+ table = reread
plan = await self._plan_compaction(table)
if plan.target_size is None:
- await self._compact(table, plan)
+ await self._compact(table, plan, partitions=partitions)
return False
if plan.target_size != DEFAULT_COMPACT_TARGET_SIZE_BYTES:
# A smaller bin can merge fewer of the files, so the trigger must hold at the planned size too.
- partitions = await asyncio.to_thread(_partition_file_stats, table, plan.target_size)
- if not triggered(_small_file_fragmentation(partitions, table_wide_small_files)):
+ planned = await asyncio.to_thread(_partition_file_stats, table, plan.target_size)
+ if not triggered(planned):
await self._logger.adebug(
- f"compact_if_fragmented: skipping; no files are removable at planned target_size="
- f"{plan.target_size} ({stats})"
+ f"{pass_name}: skipping; no files are removable at planned target_size={plan.target_size} ({stats})"
)
return False
+ if select_partitions is not None:
+ partitions = select_partitions(planned)
- await self._logger.ainfo(f"compact_if_fragmented: triggering compact ({stats})")
- return await self._compact(table, plan)
+ await self._logger.ainfo(f"{pass_name}: triggering compact ({stats})")
+ return await self._compact(table, plan, partitions=partitions)
+
+ async def compact_during_rebuild(self, min_removable_files: int) -> RebuildCompactOutcome:
+ """Compact a table that a rebuild run is still loading, once a compaction removes enough files.
+
+ A rebuild run loads the table from empty, so every file here is one that the same run wrote.
+ One pass compacts only the partitions with the most removable files, up to
+ `min_removable_files`, so a backlog takes several passes instead of one long batch. Never
+ raises an `Exception`: the batch is committed, and the next check or the post-load pass
+ compacts the same files.
+ """
+ try:
+ table = await self._table.get_delta_table()
+ if table is None:
+ return "skipped"
+
+ partitions = await asyncio.to_thread(_partition_file_stats, table, DEFAULT_COMPACT_TARGET_SIZE_BYTES)
+ removable_files = _small_file_fragmentation(partitions, table_wide_small_files=False).total_removable
+ if removable_files < min_removable_files:
+ return "skipped"
+
+ compacted = await self._compact_table(
+ table,
+ lambda planned: (
+ _small_file_fragmentation(planned, table_wide_small_files=False).total_removable
+ >= min_removable_files
+ ),
+ pass_name="compact_during_rebuild",
+ stats=(
+ f"removable_files={removable_files}, partitions={len(partitions)}, "
+ f"min_removable_files={min_removable_files}"
+ ),
+ select_partitions=lambda stats: _partitions_up_to(stats, min_removable_files),
+ )
+ return "compacted" if compacted else "skipped"
+ except Exception as e:
+ await self._log_maintenance_failure(e, step="compact_during_rebuild")
+ return "failed"
def _vacuumed_during_job(self, watermarks: VacuumWatermarks) -> bool:
# The pre-write pass vacuums before it writes, and it never checks small files. The post-load
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_maintenance.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_maintenance.py
index af441ee92617..05809fb43526 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_maintenance.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_maintenance.py
@@ -18,6 +18,7 @@ import deltalite
import pyarrow.parquet as pq
from parameterized import parameterized
+from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.consts import PARTITION_KEY
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.deltalite_handles import (
DeltaLiteHandleCache,
)
@@ -26,6 +27,7 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.del
)
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.maintenance import (
_COMPACT_RATIO_SAMPLE_FILES,
+ COMPACT_MAX_BINS_PER_COMMIT,
COMPACT_OFFSET_OVERFLOW_RETRIES,
DEFAULT_COMPACT_TARGET_SIZE_BYTES,
DELTA_DELETED_FILE_RETENTION,
@@ -41,7 +43,10 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.del
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.ops import (
ObjectStorePermissionDeniedError,
)
-from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.test.helpers import make_logger
+from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.test.helpers import (
+ make_local_table_ref,
+ make_logger,
+)
_MAINTENANCE_MODULE = "products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.maintenance"
_MB = 1024 * 1024
@@ -640,6 +645,8 @@ class TestDeltaliteCompaction:
slot_budget_bytes=int(self._SLOT_MB * _MB),
commit_metadata={"compact_engine": "deltalite", "job_id": "job-1"},
min_partition_removable_files=1,
+ max_bins_per_commit=COMPACT_MAX_BINS_PER_COMMIT,
+ partitions=None,
)
assert [call.args[0] for call in table_ref.note_deltalite_commit.call_args_list] == expected_noted
[done] = self._info_calls(table_ref, "compact: done")
@@ -692,11 +699,15 @@ class TestDeltaliteCompaction:
) -> None:
compact = MagicMock(side_effect=error) if has_compact else None
maintenance, table_ref, fake_class = self._setup(compact)
+ first = table_ref.get_delta_table.return_value
+ refreshed = _mock_table({f"f{i}.parquet": _MB for i in range(300)})
+ table_ref.get_delta_table = AsyncMock(side_effect=[first, refreshed])
ran, delta_rs_compact, _, capture = await self._run(maintenance, fake_class, enabled=enabled)
assert ran is True
- delta_rs_compact.assert_awaited_once_with(table_ref.get_delta_table.return_value, _DEFAULT_PLAN)
+ delta_rs_compact.assert_awaited_once_with(refreshed if enabled else first, _DEFAULT_PLAN, partitions=None)
+ assert table_ref.note_deltalite_commit.called is (expected_reason == "deltalite_error")
fallbacks = self._info_calls(table_ref, "compact: falling back to delta-rs")
assert [call["compact_fallback_reason"] for call in fallbacks] == ([expected_reason] if expected_reason else [])
assert capture.called is expect_capture
@@ -713,7 +724,7 @@ class TestDeltaliteCompaction:
assert ran is False
delta_rs_compact.assert_not_awaited()
capture.assert_not_called()
- table_ref.note_deltalite_commit.assert_not_called()
+ table_ref.note_deltalite_commit.assert_called_once_with(None)
@parameterized.expand(
[
@@ -740,6 +751,122 @@ class TestDeltaliteCompaction:
assert isinstance(raised.value.__cause__, deltalite.DeltaLiteError)
+class TestCompactDuringRebuild:
+ @staticmethod
+ def _fill(path: str, partitions: int, files_per_partition: int) -> None:
+ for part in range(partitions):
+ for i in range(files_per_partition):
+ deltalake.write_deltalake(
+ path,
+ pa.table({PARTITION_KEY: [str(part)], "id": [part * 1000 + i]}),
+ mode="append",
+ partition_by=[PARTITION_KEY],
+ )
+
+ @parameterized.expand(
+ [
+ # Seven tiny files per partition: a compaction removes six files from each.
+ ("below_the_budget_delta_rs", False, 2, 13, "skipped", 14),
+ ("at_the_budget_delta_rs", False, 2, 12, "compacted", 2),
+ ("below_the_budget_deltalite", True, 2, 13, "skipped", 14),
+ ("at_the_budget_deltalite", True, 2, 12, "compacted", 2),
+ # A backlog over the budget compacts only the partitions that reach it.
+ ("backlog_capped_delta_rs", False, 4, 12, "compacted", 2 + 2 * 7),
+ ("backlog_capped_deltalite", True, 4, 12, "compacted", 2 + 2 * 7),
+ ]
+ )
+ @pytest.mark.asyncio
+ async def test_compacts_up_to_the_budget(
+ self, _name: str, deltalite_engine: bool, partitions: int, budget: int, outcome: str, files_after: int
+ ):
+ with (
+ tempfile.TemporaryDirectory() as path,
+ override_settings(DATA_WAREHOUSE_DELTALITE_COMPACTION=deltalite_engine),
+ ):
+ self._fill(path, partitions=partitions, files_per_partition=7)
+ rows_before = deltalake.DeltaTable(path).to_pyarrow_table().sort_by("id").to_pydict()
+
+ result = await DeltaMaintenance(make_local_table_ref(path)).compact_during_rebuild(
+ min_removable_files=budget
+ )
+
+ table = deltalake.DeltaTable(path)
+ assert result == outcome
+ assert len(table.file_uris()) == files_after
+ assert table.to_pyarrow_table().sort_by("id").to_pydict() == rows_before
+
+ @pytest.mark.asyncio
+ async def test_a_smaller_planned_bin_reselects_the_partitions(self):
+ # At 100 MiB every file is small. At the planned 300 KB, partition "a" holds only files of about
+ # 160 KB, which a compaction cannot merge, so the pass must move to partition "b".
+ with tempfile.TemporaryDirectory() as path:
+ for i in range(8):
+ deltalake.write_deltalake(
+ path,
+ pa.table({PARTITION_KEY: ["a"], "id": [i], "blob": [os.urandom(80_000).hex()]}),
+ mode="append",
+ partition_by=[PARTITION_KEY],
+ )
+ for i in range(7):
+ deltalake.write_deltalake(
+ path,
+ pa.table({PARTITION_KEY: ["b"], "id": [100 + i], "blob": ["x"]}),
+ mode="append",
+ partition_by=[PARTITION_KEY],
+ )
+ maintenance = DeltaMaintenance(make_local_table_ref(path))
+ plan = CompactionPlan(
+ target_size=300_000, max_concurrent_tasks=1, compression_ratio=1.0, slot_budget_mb=1356.8
+ )
+
+ with patch.object(maintenance, "_plan_compaction", AsyncMock(return_value=plan)):
+ result = await maintenance.compact_during_rebuild(min_removable_files=6)
+
+ files = deltalake.DeltaTable(path).file_uris()
+ assert result == "compacted"
+ assert (
+ sum(f"{PARTITION_KEY}=a/" in uri for uri in files),
+ sum(f"{PARTITION_KEY}=b/" in uri for uri in files),
+ ) == (8, 1)
+
+ @parameterized.expand(
+ [
+ ("transient_object_store_error", TransientObjectStoreError("throttled"), False),
+ ("unexpected_error", RuntimeError("boom"), True),
+ ]
+ )
+ @pytest.mark.asyncio
+ async def test_a_failed_pass_is_reported_and_never_raises(self, _name: str, error: Exception, expect_capture: bool):
+ with (
+ tempfile.TemporaryDirectory() as path,
+ patch.object(DeltaMaintenance, "_compact", AsyncMock(side_effect=error)),
+ patch(f"{_MAINTENANCE_MODULE}.capture_exception") as capture,
+ ):
+ self._fill(path, partitions=2, files_per_partition=7)
+
+ result = await DeltaMaintenance(make_local_table_ref(path)).compact_during_rebuild(min_removable_files=1)
+
+ assert result == "failed"
+ assert capture.called is expect_capture
+
+ @pytest.mark.asyncio
+ async def test_deltalite_commits_a_long_compaction_in_chunks(self):
+ with (
+ tempfile.TemporaryDirectory() as path,
+ override_settings(DATA_WAREHOUSE_DELTALITE_COMPACTION=True),
+ patch(f"{_MAINTENANCE_MODULE}.COMPACT_MAX_BINS_PER_COMMIT", 1),
+ ):
+ self._fill(path, partitions=3, files_per_partition=4)
+ version_before = deltalake.DeltaTable(path).version()
+
+ result = await DeltaMaintenance(make_local_table_ref(path)).compact_during_rebuild(min_removable_files=9)
+
+ table = deltalake.DeltaTable(path)
+ assert result == "compacted"
+ assert table.version() - version_before == 3
+ assert len(table.file_uris()) == 3
+
+
class TestVacuum:
@pytest.mark.asyncio
async def test_retries_on_commit_conflict_then_succeeds(self):
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_writer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_writer.py
index d3be830e0453..c2efa4fa8ffc 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_writer.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_writer.py
@@ -201,6 +201,28 @@ class TestHasBatchBeenCommitted:
with patch.object(writer._table, "get_delta_table", AsyncMock(return_value=table)):
assert await writer.has_batch_been_committed(run_uuid, batch_index) is expected
+ @parameterized.expand(
+ [
+ ("compaction_commits_do_not_use_up_the_window", [("OPTIMIZE", 120)], True),
+ ("vacuum_commits_do_not_use_up_the_window", [("VACUUM END", 120)], True),
+ ("data_commits_still_close_the_window", [("WRITE", 60)], False),
+ ("recent_writes_ahead_of_a_long_compaction", [("WRITE", 49), ("OPTIMIZE", 100)], True),
+ ]
+ )
+ @pytest.mark.asyncio
+ async def test_finds_the_batch_behind_later_commits(
+ self, _name: str, later: list[tuple[str, int]], expected: bool
+ ) -> None:
+ commits = [{"operation": operation} for operation, count in later for _ in range(count)] + [
+ {"operation": "WRITE", "run_uuid": "run-1", "batch_index": "5"}
+ ]
+ writer = _make_writer()
+ table = MagicMock()
+ table.history = MagicMock(side_effect=lambda limit: commits[:limit])
+ with patch.object(writer._table, "get_delta_table", AsyncMock(return_value=table)):
+ assert await writer.has_batch_been_committed("run-1", 5) is expected
+ assert table.history.call_count <= 3
+
class TestWriteToDeltalakeCommitMetadataPassThrough:
"""Covers that commit_metadata is forwarded to deltalake.write_deltalake as CommitProperties."""
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/writer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/writer.py
index 0601dcaed0a1..47d405a8e24b 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/writer.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/writer.py
@@ -161,6 +161,10 @@ def commit_members_tag(members: Sequence[tuple[str, int]]) -> str:
return ",".join(f"{run_uuid}:{batch_index}" for run_uuid, batch_index in members)
+_MAINTENANCE_OPERATIONS = frozenset({"OPTIMIZE", "VACUUM START", "VACUUM END"})
+_MAX_HISTORY_SCAN = 2000
+
+
def commit_covers_batch(commit: dict[str, Any], run_uuid: str, batch_index: int) -> bool:
"""Whether this commit wrote `batch_index` of `run_uuid`, alone or as one member of a set.
@@ -759,10 +763,21 @@ class DeltaWriter:
async def has_batch_been_committed(self, run_uuid: str, batch_index: int, *, scan_limit: int = 50) -> bool:
"""Check whether a specific (run_uuid, batch_index) has already been committed to delta,
- on its own or as a member of a coalesced write (see `commit_covers_batch`)."""
+ on its own or as a member of a coalesced write (see `commit_covers_batch`).
+
+ The scan covers the last `scan_limit` commits that wrote data. A chunked compaction can commit
+ many times after the batch write, so compaction and vacuum commits do not use up the window.
+ """
delta_table = await self._table.get_delta_table()
if delta_table is None:
return False
- history = await asyncio.to_thread(delta_table.history, limit=scan_limit)
- return any(commit_covers_batch(commit, run_uuid, batch_index) for commit in history)
+ limit = scan_limit
+ while True:
+ history = await asyncio.to_thread(delta_table.history, limit=limit)
+ if any(commit_covers_batch(commit, run_uuid, batch_index) for commit in history):
+ return True
+ data_commits = sum(1 for commit in history if commit.get("operation") not in _MAINTENANCE_OPERATIONS)
+ if len(history) < limit or data_commits >= scan_limit or limit >= _MAX_HISTORY_SCAN:
+ return False
+ limit = min(limit * 2, _MAX_HISTORY_SCAN)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/partitioning.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/partitioning.py
index 81a77af20f35..be9ab8112b97 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/partitioning.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/partitioning.py
@@ -307,3 +307,23 @@ def append_partition_key_to_table(
partition_format=partition_format,
partition_keys=normalized_partition_keys,
)
+
+
+# Delta's fixed marker for a Hive-style partition directory whose column value is null.
+HIVE_NULL_PARTITION = "__HIVE_DEFAULT_PARTITION__"
+
+
+def partition_key_from_add_path(path: str) -> str | None:
+ """Recover the `_ph_partition_key` value from a Hive-style partitioned add-action file path.
+
+ Every value this pipeline actually partitions by — an md5 bucket index, a numerical bucket or the
+ `null` sentinel, or a datetime tier like "2024-01" — is plain ASCII with nothing that needs
+ percent-encoding, so splitting the path recovers the same value `get_add_actions`'s
+ `partition.<key>` column would give.
+ """
+ prefix = f"{PARTITION_KEY}="
+ for segment in path.split("/"):
+ if segment.startswith(prefix):
+ value = segment[len(prefix) :]
+ return None if value == HIVE_NULL_PARTITION else value
+ return None
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/repartition.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/repartition.py
index 5f45859e1a60..e6fa217ebfff 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/repartition.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/repartition.py
@@ -52,6 +52,7 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.del
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.partitioning import (
NULL_NUMERICAL_PARTITION,
append_partition_key_to_table,
+ partition_key_from_add_path,
)
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.repartition_stream import (
PartitionedFileWriter,
@@ -266,26 +267,6 @@ class RepartitionTarget:
return cls(**{k: v for k, v in data.items() if k in fields})
-# Delta's fixed marker for a Hive-style partition directory whose column value is null.
-_HIVE_NULL_PARTITION = "__HIVE_DEFAULT_PARTITION__"
-
-
-def _partition_key_from_add_path(path: str) -> str | None:
- """Recover the `_ph_partition_key` value from a Hive-style partitioned add-action file path.
-
- Every value this pipeline actually partitions by — an md5 bucket index, a numerical bucket or the
- `null` sentinel, or a datetime tier like "2024-01" — is plain ASCII with nothing that needs
- percent-encoding, so splitting the path recovers the same value `get_add_actions`'s
- `partition.<key>` column would give.
- """
- prefix = f"{PARTITION_KEY}="
- for segment in path.split("/"):
- if segment.startswith(prefix):
- value = segment[len(prefix) :]
- return None if value == _HIVE_NULL_PARTITION else value
- return None
-
-
def measure_partition_bytes(delta_table: deltalake.DeltaTable) -> dict[str | None, int]:
"""At-rest bytes per partition, read from the Delta log (no S3 LIST, no data scan).
@@ -301,7 +282,7 @@ def measure_partition_bytes(delta_table: deltalake.DeltaTable) -> dict[str | Non
totals: dict[str | None, int] = defaultdict(int)
for path, size in delta_table._table.get_add_file_sizes().items():
- key = _partition_key_from_add_path(path) if partitioned else None
+ key = partition_key_from_add_path(path) if partitioned else None
totals[key] += size or 0
return dict(totals)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/batch_steps.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/batch_steps.py
index 83df44d4152b..4bee31597887 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/batch_steps.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/batch_steps.py
@@ -21,6 +21,7 @@ BatchStep = Literal[
"write",
"mark_processed",
"post_write",
+ "compact_during_rebuild",
]
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/metrics.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/metrics.py
index 794cc326c014..4ea7b7c079b3 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/metrics.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/metrics.py
@@ -23,6 +23,12 @@ APPEND_ROLLBACK_TOTAL = Counter(
labelnames=["outcome"],
)
+DELTA_COMPACT_DURING_REBUILD_TOTAL = Counter(
+ "warehouse_load_delta_compact_during_rebuild_total",
+ "Compaction checks between the batches of a rebuild run, by outcome",
+ labelnames=["outcome"],
+)
+
PARQUET_READ_DURATION_SECONDS = Histogram(
"warehouse_load_parquet_read_duration_seconds",
"Duration of S3 parquet file reads",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/processor.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/processor.py
index f410e10c8bad..07dce69f7c86 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/processor.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/processor.py
@@ -53,6 +53,7 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.aut
maybe_schedule_auto_widen_resync,
)
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.consts import PARTITION_KEY
+from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.maintenance import DeltaMaintenance
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.memory_governor import get_governor
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.rss_sampler import RssPeakSampler
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.scd2 import Scd2DeltaWriter
@@ -91,6 +92,7 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline
from products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.load.metrics import (
CDC_DELETE_ENRICHMENT_VIOLATIONS_TOTAL,
CDC_SEQ_GUARD_ROWS_DROPPED_TOTAL,
+ DELTA_COMPACT_DURING_REBUILD_TOTAL,
DELTA_ROWS_WRITTEN_TOTAL,
DELTA_WRITE_DURATION_SECONDS,
IDEMPOTENCY_HIT_TOTAL,
@@ -112,6 +114,7 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline
)
from products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.table_rebuild import TableRebuildRun
from products.warehouse_sources.backend.temporal.data_imports.row_tracking import finish_row_tracking
+from products.warehouse_sources.backend.temporal.data_imports.schema_flags import is_schema_flag_enabled
from products.warehouse_sources.backend.temporal.data_imports.sources import SourceRegistry
from products.warehouse_sources.backend.temporal.data_imports.sources.common.cursor import merge_cursor_payloads
from products.warehouse_sources.backend.temporal.data_imports.util import prepare_s3_files_for_querying
@@ -125,6 +128,9 @@ logger = structlog.get_logger(__name__)
# show a table fragmenting over a multi-hundred-batch load, rare enough that the listing it needs
# stays off the per-batch path.
FILE_COUNT_LOG_SAMPLE_EVERY = 100
+# The check lists the add-file sizes of the open snapshot, O(files in table), so it skips most batches.
+REBUILD_COMPACT_CHECK_EVERY = 10
+REBUILD_COMPACTION_FLAG = "warehouse-rebuild-run-compaction"
def _get_write_type(sync_type: SyncTypeLiteral) -> Literal["incremental", "full_refresh", "append"]:
@@ -350,6 +356,27 @@ def _partial_data_loading_applies(export_signal: ExportSignalMessage, schema: Ex
return export_signal.is_first_ever_sync and supports_partial_data_loading(schema)
+def _rebuild_compaction_due(
+ export_signal: ExportSignalMessage, schema: ExternalDataSchema, members: list[tuple[str, int]]
+) -> bool:
+ """Whether this write of a rebuild run checks the table for compaction (see `compact_during_rebuild`).
+
+ A rebuild run (first sync, reset, full refresh) loads the table from empty, so a compaction
+ rewrites only files of the same run. A run that adds to an existing table would compact files
+ from before it, which the rollback of an unfinished append run restores.
+ Not for CDC: each CDC run is short and ends with its own post-load pass.
+ Not for partial data loading: it publishes the files of each batch, and a compaction replaces them.
+ """
+ return (
+ (export_signal.is_first_ever_sync or export_signal.sync_type == "full_refresh")
+ and export_signal.cdc_write_mode is None
+ and not _partial_data_loading_applies(export_signal, schema)
+ and all(run_uuid == export_signal.run_uuid for run_uuid, _ in members)
+ and any(index > 0 and index % REBUILD_COMPACT_CHECK_EVERY == 0 for _, index in members)
+ and is_schema_flag_enabled(schema, REBUILD_COMPACTION_FLAG)
+ )
+
+
async def _handle_partial_data_loading(
export_signal: ExportSignalMessage,
job: ExternalDataJob,
@@ -1078,6 +1105,19 @@ def _read_constituents(constituents: list[ExportSignalMessage]) -> pa.Table:
raise CoalescingDeclined(f"batch schemas do not concatenate: {e}") from e
+def _compact_during_rebuild(
+ delta_table_ref: DeltaTableRef, verify_ownership: Callable[[], None] | None, timer: BatchStepTimer
+) -> None:
+ if verify_ownership is not None:
+ verify_ownership()
+ report_phase("compact_during_rebuild")
+ with timer.step("compact_during_rebuild"):
+ outcome = async_to_sync(DeltaMaintenance(delta_table_ref).compact_during_rebuild)(
+ settings.DATA_WAREHOUSE_REBUILD_COMPACT_MIN_REMOVABLE_FILES
+ )
+ DELTA_COMPACT_DURING_REBUILD_TOTAL.labels(outcome=outcome).inc()
+
+
def _process_message_reported(
message: Any,
export_signal: "ExportSignalMessage",
@@ -1466,6 +1506,8 @@ def _process_message_reported(
finals = final_run_signals(constituents)
else:
finals = [export_signal] if export_signal.is_final_batch else []
+ if not finals and _rebuild_compaction_due(export_signal, schema, members):
+ _compact_during_rebuild(delta_table_ref, verify_ownership, timer)
for run_signal in finals:
run_job = job if run_signal.job_id == export_signal.job_id else _load_job(run_signal.job_id)
run_schema = run_job.schema
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/test_processor.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/test_processor.py
index 6f60600361ed..e2bb8d598a33 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/test_processor.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/test_processor.py
@@ -7,6 +7,8 @@ from typing import Any
import pytest
from unittest.mock import ANY, AsyncMock, MagicMock, patch
+from django.test import override_settings
+
import pyarrow as pa
from asgiref.sync import async_to_sync
from deltalake import DeltaTable, write_deltalake
@@ -42,6 +44,7 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline
_mark_job_completed,
_promote_staged_cursor,
_read_existing_rows_by_first_pk,
+ _rebuild_compaction_due,
_resolve_cdc_positions,
_run_post_load_for_already_processed_batch,
_trigger_post_import_workflow,
@@ -51,7 +54,9 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline
from products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.load.table_handles import (
GroupTableHandles,
)
+from products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.messages import ExportSignalMessage
from products.warehouse_sources.backend.temporal.data_imports.pipelines.test_mocks import mock_delta_table
+from products.warehouse_sources.backend.types import ExternalDataSourceType
from products.warehouse_sources_queue.backend.core.batch_consumer import CoalescingDeclined
@@ -1990,3 +1995,82 @@ class TestTableHandleReuse:
rows = DeltaTable(tmp).to_pyarrow_table()
assert rows.sort_by("id").column("id").to_pylist() == [0, 1, 4, 5, 6, 6, 7, 8, 100]
assert rows.schema.names == ["id", "name", "added"]
+
+
+class TestCompactionDuringRebuild:
+ @staticmethod
+ def _schema(source_type: str = "Postgres") -> MagicMock:
+ schema = MagicMock()
+ schema.source.source_type = source_type
+ return schema
+
+ @parameterized.expand(
+ [
+ ("full_refresh", {"sync_type": "full_refresh"}, None, None, True),
+ ("first_sync_incremental", {"is_first_ever_sync": True}, None, None, True),
+ ("first_sync_append", {"sync_type": "append", "is_first_ever_sync": True}, None, None, True),
+ ("incremental_on_an_existing_table", {}, None, None, False),
+ ("append_on_an_existing_table", {"sync_type": "append"}, None, None, False),
+ ("cdc", {"is_first_ever_sync": True, "cdc_write_mode": "upsert"}, None, None, False),
+ ("partial_data_loading", {"is_first_ever_sync": True}, ExternalDataSourceType.STRIPE, None, False),
+ ("set_spanning_two_runs", {"sync_type": "full_refresh"}, None, [("run-0", 9), ("run-1", 10)], False),
+ (
+ "set_crossing_a_check",
+ {"sync_type": "full_refresh", "batch_index": 9},
+ None,
+ [("run-1", 9), ("run-1", 10)],
+ True,
+ ),
+ ("between_checks", {"sync_type": "full_refresh", "batch_index": 11}, None, None, False),
+ ("first_batch", {"sync_type": "full_refresh", "batch_index": 0}, None, None, False),
+ ]
+ )
+ def test_which_writes_check_for_compaction(
+ self,
+ _name: str,
+ overrides: dict[str, Any],
+ source_type: str | None,
+ members: list[tuple[str, int]] | None,
+ expected: bool,
+ ) -> None:
+ signal = ExportSignalMessage.from_dict(_message(**{"batch_index": 10, **overrides}))
+ members = members or [(signal.run_uuid, signal.batch_index)]
+ with patch(f"{_PROCESSOR}.is_schema_flag_enabled", return_value=True):
+ assert _rebuild_compaction_due(signal, self._schema(source_type or "Postgres"), members) is expected
+
+ def test_the_flag_turns_it_off(self) -> None:
+ signal = ExportSignalMessage.from_dict(_message(batch_index=10, sync_type="full_refresh"))
+ with patch(f"{_PROCESSOR}.is_schema_flag_enabled", return_value=False):
+ assert _rebuild_compaction_due(signal, self._schema(), [(signal.run_uuid, 10)]) is False
+
+ def _load(self, table_path: str, sync_type: str, first_index: int, *, enabled: bool) -> None:
+ write_deltalake(table_path, TestTableHandleReuse._SEED)
+ with (
+ override_settings(DATA_WAREHOUSE_REBUILD_COMPACT_MIN_REMOVABLE_FILES=3),
+ patch(f"{_PROCESSOR}.REBUILD_COMPACT_CHECK_EVERY", 5),
+ TestTableHandleReuse()._loader(table_path, []),
+ patch(f"{_PROCESSOR}.is_schema_flag_enabled", return_value=enabled),
+ ):
+ for index in range(first_index, first_index + 12):
+ process_message(TestTableHandleReuse._batch(index, sync_type=sync_type))
+
+ @parameterized.expand(
+ [
+ ("rebuild_run_compacts_as_it_loads", "full_refresh", 0, True),
+ ("append_on_an_existing_table_waits_for_post_load", "append", 1, False),
+ ]
+ )
+ def test_compaction_between_batches_keeps_the_rows(
+ self, _name: str, sync_type: str, first_index: int, compacts: bool
+ ) -> None:
+ with tempfile.TemporaryDirectory() as tmp:
+ on_path, off_path = f"{tmp}/on", f"{tmp}/off"
+ self._load(on_path, sync_type, first_index, enabled=True)
+ self._load(off_path, sync_type, first_index, enabled=False)
+
+ on_table, off_table = DeltaTable(on_path), DeltaTable(off_path)
+ assert (
+ on_table.to_pyarrow_table().sort_by([("id", "ascending"), ("name", "ascending")]).to_pydict()
+ == off_table.to_pyarrow_table().sort_by([("id", "ascending"), ("name", "ascending")]).to_pydict()
+ )
+ assert (len(on_table.file_uris()) < len(off_table.file_uris())) is compacts