Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-warehouse): use smaller files for wide-row merge tables (#119682)

Tom Owers57800f8ebb3b
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/maintenance.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/maintenance.py
index dfff4a1f0756..cca8c1c6c0dd 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/maintenance.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/maintenance.py
@@ -35,11 +35,13 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.del
)
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.memory_governor import get_governor
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.ops import (
+ DEFAULT_TARGET_FILE_SIZE_BYTES,
OBJECT_STORE_PERMISSION_DENIED_MESSAGE,
OBJECT_STORE_TRANSIENT_MESSAGE,
ObjectStorePermissionDeniedError,
execute_with_conflict_retry,
is_object_store_permission_denied,
+ table_target_file_size,
)
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.writer import _delta_table_identity
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.partitioning import (
@@ -89,9 +91,11 @@ DEFAULT_COMPACT_REMOVABLE_FILES_PER_PARTITION_THRESHOLD = 8
DEFAULT_COMPACT_REMOVABLE_FILES_THRESHOLD = 100
# delta-rs's own default target size for a compaction rewrite bin when none is given (see
-# `delta.targetFileSize`, or 100MB absent that table property). Starting the retry ladder here
-# instead of passing None keeps every attempt's size explicit and halvable.
-DEFAULT_COMPACT_TARGET_SIZE_BYTES = 100 * 1024 * 1024
+# `delta.targetFileSize`, or 100MB absent that table property). A table that sets the property
+# compacts to its own value instead (see `table_target_file_size`), so that compaction keeps the
+# file size its writes use. Starting the retry ladder at an explicit size instead of passing None
+# keeps every attempt's size explicit and halvable.
+DEFAULT_COMPACT_TARGET_SIZE_BYTES = DEFAULT_TARGET_FILE_SIZE_BYTES
# Halving the bin size on a byte-array-offset-overflow panic (see is_offset_overflow_compaction_error)
# shrinks how many files get concatenated into one Arrow batch. Bounded so a table whose overflow
# can't be avoided at any reasonable bin size still surfaces to error tracking instead of looping.
@@ -123,8 +127,12 @@ class CompactionPlan:
slot_budget_mb: float | None
-def plan_compaction(compression_ratio: float | None, slot_budget_mb: float | None) -> CompactionPlan:
- """Bin size and parallelism that keep one compaction inside its memory slot."""
+def plan_compaction(
+ compression_ratio: float | None,
+ slot_budget_mb: float | None,
+ table_target_size: int = DEFAULT_COMPACT_TARGET_SIZE_BYTES,
+) -> CompactionPlan:
+ """Bin size and parallelism that keep one compaction inside its memory slot, at most `table_target_size`."""
rounded_ratio = round(compression_ratio, 2) if compression_ratio is not None else None
rounded_slot = round(slot_budget_mb, 1) if slot_budget_mb is not None else None
if compression_ratio is None or (slot_budget_mb is not None and slot_budget_mb <= 0):
@@ -149,7 +157,7 @@ def plan_compaction(compression_ratio: float | None, slot_budget_mb: float | Non
slot_budget_mb=rounded_slot,
)
- target_size = int(min(DEFAULT_COMPACT_TARGET_SIZE_BYTES, safe_target_size))
+ target_size = int(min(table_target_size, safe_target_size))
max_concurrent_tasks = None
if slot_budget_bytes is not None:
per_task_bytes = target_size * ratio * _COMPACT_TASK_MEMORY_FACTOR
@@ -468,7 +476,7 @@ class DeltaMaintenance:
except Exception as e:
await self._logger.awarning(f"compact: could not sample the compression ratio: {e}")
ratio = None
- return plan_compaction(ratio, get_governor().slot_budget_mb())
+ return plan_compaction(ratio, get_governor().slot_budget_mb(), table_target_file_size(table))
@recorded_phase("compact")
async def _compact(
@@ -566,8 +574,9 @@ class DeltaMaintenance:
storage_options = self._table.get_storage_options()
identity = _delta_table_identity(table, self._table)
slot_budget_mb = get_governor().slot_budget_mb()
+ target_size = table_target_file_size(table)
compact_kwargs: dict[str, Any] = {
- "target_file_size": DEFAULT_COMPACT_TARGET_SIZE_BYTES,
+ "target_file_size": target_size,
# deltalite lowers this until the bins in flight fit the slot budget.
"max_parallel_bins": os.cpu_count() or 1,
"slot_budget_bytes": int(slot_budget_mb * 1024 * 1024) if slot_budget_mb else None,
@@ -593,7 +602,7 @@ class DeltaMaintenance:
await self._logger.ainfo(
"Compacting table...",
compact_engine="deltalite",
- compact_target_size=DEFAULT_COMPACT_TARGET_SIZE_BYTES,
+ compact_target_size=target_size,
compact_max_parallel_bins=compact_kwargs["max_parallel_bins"],
compact_slot_budget_mb=round(slot_budget_mb, 1) if slot_budget_mb is not None else None,
pod_memory_mb=pod_mb_before,
@@ -778,7 +787,8 @@ class DeltaMaintenance:
compact_small_files and small_file_stats.fragmented
)
- partitions = await asyncio.to_thread(_partition_file_stats, table, DEFAULT_COMPACT_TARGET_SIZE_BYTES)
+ table_target_size = table_target_file_size(table)
+ partitions = await asyncio.to_thread(_partition_file_stats, table, table_target_size)
# Repartition detection runs after this pass and compares partition bytes to its budget.
# Small files take more bytes at rest than the same rows after compaction, so an
# over-budget partition that compaction can shrink must be compacted before it is measured.
@@ -813,11 +823,10 @@ class DeltaMaintenance:
`select_partitions` limits the pass, from the partition stats at the bin size the engine uses.
"""
+ table_target_size = table_target_file_size(table)
partitions = None
if select_partitions is not None:
- partitions = select_partitions(
- await asyncio.to_thread(_partition_file_stats, table, DEFAULT_COMPACT_TARGET_SIZE_BYTES)
- )
+ partitions = select_partitions(await asyncio.to_thread(_partition_file_stats, table, table_target_size))
if settings.DATA_WAREHOUSE_DELTALITE_COMPACTION:
await self._logger.ainfo(f"{pass_name}: triggering compact ({stats})")
compacted = await self._compact_with_deltalite(table, partitions=partitions)
@@ -833,7 +842,7 @@ class DeltaMaintenance:
await self._compact(table, plan, partitions=partitions)
return False
- if plan.target_size != DEFAULT_COMPACT_TARGET_SIZE_BYTES:
+ if plan.target_size != table_target_size:
# A smaller bin can merge fewer of the files, so the trigger must hold at the planned size too.
planned = await asyncio.to_thread(_partition_file_stats, table, plan.target_size)
if not triggered(planned):
@@ -861,7 +870,7 @@ class DeltaMaintenance:
if table is None:
return "skipped"
- partitions = await asyncio.to_thread(_partition_file_stats, table, DEFAULT_COMPACT_TARGET_SIZE_BYTES)
+ partitions = await asyncio.to_thread(_partition_file_stats, table, table_target_file_size(table))
removable_files = _small_file_fragmentation(partitions, table_wide_small_files=False).total_removable
if removable_files < min_removable_files:
return "skipped"
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/memory_governor.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/memory_governor.py
index ec1a5aeb3e71..ec529e18250c 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/memory_governor.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/memory_governor.py
@@ -65,6 +65,10 @@ import pyarrow.compute as pc
from posthog.dataclasses import frozen
+from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.ops import (
+ DEFAULT_TARGET_FILE_SIZE_BYTES,
+ table_target_file_size,
+)
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.rss_sampler import (
RssPeakSampler,
RssWindow,
@@ -132,8 +136,7 @@ _SOURCE_INUSE_PER_MB = 0.73
#: deltalite keeps one add action per table file in its snapshot. Measured at ~2.2 KB per file on
#: the 30-column harness tables (6,000 files hold 12 MB more than 600 files).
_SNAPSHOT_KB_PER_FILE = 2.2
-#: deltalite and delta-rs both default to 100 MiB when the table does not set ``delta.targetFileSize``.
-_DEFAULT_TARGET_FILE_SIZE = 100 * MB
+_DEFAULT_TARGET_FILE_SIZE = DEFAULT_TARGET_FILE_SIZE_BYTES
#: RSS per byte of malloc in-use memory. glibc keeps freed memory in its arenas, and with its
#: default mmap threshold the large Arrow buffers stay there too. RSS ran 1.25–2.1× in-use in most
@@ -360,21 +363,13 @@ def rewrite_profile(
)
-def _table_target_file_size(table: deltalake.DeltaTable) -> int:
- raw = table.metadata().configuration.get("delta.targetFileSize")
- try:
- return int(raw) if raw and int(raw) > 0 else _DEFAULT_TARGET_FILE_SIZE
- except ValueError:
- return _DEFAULT_TARGET_FILE_SIZE
-
-
def estimate_rewrite_profile(
table: deltalake.DeltaTable, source: pa.Table, partition_col: str | None, primary_keys: Sequence[str]
) -> RewriteProfile | None:
"""``rewrite_profile`` for a live table, or None when its add actions cannot be read."""
try:
add_actions = pa.table(table.get_add_actions(flatten=True))
- return rewrite_profile(add_actions, source, partition_col, primary_keys, _table_target_file_size(table))
+ return rewrite_profile(add_actions, source, partition_col, primary_keys, table_target_file_size(table))
except Exception as e: # noqa: BLE001 - an estimate must never fail a write; None sizes conservatively
logger.debug("deltalite governor: could not estimate the rewrite size: %s", e)
return None
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/ops.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/ops.py
index 276152a19a66..ef48093ef05d 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/ops.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/ops.py
@@ -4,7 +4,9 @@ from typing import TypedDict, TypeVar
from django.conf import settings
+import pyarrow as pa
import deltalake
+import pyarrow.compute as pc
import botocore.exceptions
import deltalake.exceptions
from structlog.types import FilteringBoundLogger
@@ -189,14 +191,84 @@ async def execute_with_conflict_retry(
# property when it commits, so both writers checkpoint on the same cadence.
DELTA_TABLE_PROPERTIES: dict[str, str] = {"delta.checkpointInterval": "25"}
+MiB = 1024 * 1024
+TARGET_FILE_SIZE_PROPERTY = "delta.targetFileSize"
+#: deltalite and delta-rs both write files up to this size when a table does not set the property.
+DEFAULT_TARGET_FILE_SIZE_BYTES = 100 * MiB
-async def ensure_table_properties(table: deltalake.DeltaTable, logger: FilteringBoundLogger) -> bool:
+# A merge rewrites every file that holds a matched row, and deltalite writes each file as a single
+# row group. When rows are wide, a default-size file holds few rows, so a small update set touches
+# almost every file and copies many unchanged rows. A large row group also leaves the memory
+# governor room for fewer readers and partition workers in one upsert's slice, often only one of
+# each. Smaller files for these tables lower both costs.
+#: Stored bytes per row at or above which a merge table counts as wide. A default-size file of such
+#: rows holds at most 12,800 rows.
+WIDE_ROW_MIN_STORED_BYTES_PER_ROW = 8 * 1024
+#: Rows that one file of a wide-row table holds, before the size band below applies.
+WIDE_ROW_TARGET_ROWS_PER_FILE = 1024
+#: Smaller files cost more footer probes and log entries than they save on a rewrite.
+WIDE_ROW_MIN_TARGET_FILE_SIZE_BYTES = 16 * MiB
+#: Small enough that the governor can plan several readers and partition workers per upsert.
+WIDE_ROW_MAX_TARGET_FILE_SIZE_BYTES = 32 * MiB
+#: A smaller table fits in one file at either target, and its per-row estimate still includes the
+#: Parquet footer overhead, so the decision waits until the table holds this much data.
+WIDE_ROW_MIN_TABLE_BYTES = WIDE_ROW_MAX_TARGET_FILE_SIZE_BYTES
+
+
+def table_target_file_size(table: deltalake.DeltaTable) -> int:
+ """The table's `delta.targetFileSize` in bytes, or the writers' default when it is absent or invalid."""
+ raw = (table.metadata().configuration or {}).get(TARGET_FILE_SIZE_PROPERTY)
+ if not isinstance(raw, str) or not raw.strip().isdigit() or int(raw) <= 0:
+ return DEFAULT_TARGET_FILE_SIZE_BYTES
+ return int(raw)
+
+
+def wide_row_target_file_size(add_actions: pa.Table) -> int | None:
+ """A smaller `delta.targetFileSize` for a table of wide rows, or None to keep the default.
+
+ Bytes per row come from the add actions: the stored size and the row count of each file that
+ records its row count. The result is a whole number of MiB, so the same table always gets the same value.
+ """
+ if add_actions.num_rows == 0 or not {"size_bytes", "num_records"} <= set(add_actions.column_names):
+ return None
+ counted = add_actions.filter(pc.greater(add_actions["num_records"], pa.scalar(0)))
+ stored_bytes = pc.sum(counted["size_bytes"]).as_py() or 0
+ rows = pc.sum(counted["num_records"]).as_py() or 0
+ if rows == 0 or stored_bytes < WIDE_ROW_MIN_TABLE_BYTES:
+ return None
+ bytes_per_row = stored_bytes / rows
+ if bytes_per_row < WIDE_ROW_MIN_STORED_BYTES_PER_ROW:
+ return None
+ target_mib = round(WIDE_ROW_TARGET_ROWS_PER_FILE * bytes_per_row / MiB)
+ return max(WIDE_ROW_MIN_TARGET_FILE_SIZE_BYTES, min(WIDE_ROW_MAX_TARGET_FILE_SIZE_BYTES, target_mib * MiB))
+
+
+def _may_hold_wide_rows(batch: pa.Table) -> bool:
+ # Arrow holds a row in at least as many bytes as Parquet stores it in, so a narrow batch rules
+ # out a wide table without the add-action read, which costs O(files) on every write.
+ return batch.num_rows > 0 and batch.nbytes / batch.num_rows >= WIDE_ROW_MIN_STORED_BYTES_PER_ROW
+
+
+def _wide_row_table_properties(table: deltalake.DeltaTable) -> dict[str, str]:
+ add_actions = pa.table(table.get_add_actions(flatten=False)).select(["size_bytes", "num_records"])
+ target = wide_row_target_file_size(add_actions)
+ return {} if target is None else {TARGET_FILE_SIZE_PROPERTY: str(target)}
+
+
+async def ensure_table_properties(
+ table: deltalake.DeltaTable, logger: FilteringBoundLogger, *, merge_batch: pa.Table | None = None
+) -> bool:
"""Apply DELTA_TABLE_PROPERTIES to a table that was created without them.
A metadata-only commit, made once per table: the check reads the handle's own snapshot, and
`set_table_properties` refreshes that snapshot, so a table that already carries the values costs
nothing here. Never raises, because the data write has already committed and a property that
fails to land only waits for the next write. Returns True when it committed.
+
+ `merge_batch` is the batch just merged into a table that loads by merging on a primary key, and
+ None for every other write. Such a table of wide rows also gets a smaller `delta.targetFileSize`
+ (see `wide_row_target_file_size`). The value is set only when the table has none, so it never
+ changes after that and never replaces a value set by hand.
"""
try:
# The metadata read is in the same best-effort boundary as the write below: it touches the
@@ -204,6 +276,8 @@ async def ensure_table_properties(table: deltalake.DeltaTable, logger: Filtering
# the data commit has already landed either way, so a failure here must not propagate either.
current = table.metadata().configuration or {}
missing = {key: value for key, value in DELTA_TABLE_PROPERTIES.items() if current.get(key) != value}
+ if merge_batch is not None and TARGET_FILE_SIZE_PROPERTY not in current and _may_hold_wide_rows(merge_batch):
+ missing.update(await asyncio.to_thread(_wide_row_table_properties, table))
if not missing:
return False
await execute_with_conflict_retry(
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_maintenance.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_maintenance.py
index 05809fb43526..8e47bb00f072 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_maintenance.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_maintenance.py
@@ -52,6 +52,7 @@ _MAINTENANCE_MODULE = "products.warehouse_sources.backend.temporal.data_imports.
_MB = 1024 * 1024
# Small enough that a local table can hold files above the compaction target.
_SMALL_TARGET = 100_000
+_DEFAULT_TABLE_TARGET = 100 * 1024 * 1024
_NOW = dt.datetime(2026, 6, 1, 12, 0, tzinfo=dt.UTC)
_DEFAULT_PLAN = CompactionPlan(
target_size=DEFAULT_COMPACT_TARGET_SIZE_BYTES, max_concurrent_tasks=1, compression_ratio=1.0, slot_budget_mb=1356.8
@@ -281,15 +282,15 @@ class TestCompactIfFragmented:
)
@pytest.mark.asyncio
async def test_small_file_trigger_on_a_real_table(self, _name: str, interleave_oversized: bool, expected_ran: bool):
- with (
- tempfile.TemporaryDirectory() as path,
- patch(f"{_MAINTENANCE_MODULE}.DEFAULT_COMPACT_TARGET_SIZE_BYTES", _SMALL_TARGET),
- ):
+ with tempfile.TemporaryDirectory() as path:
for i in range(12):
_append_file(path, i, payload_bytes=500)
if interleave_oversized:
_append_file(path, 100 + i, payload_bytes=2 * _SMALL_TARGET)
table = deltalake.DeltaTable(path)
+ # The oversized files are small next to the default target, so the skip case also fails
+ # when compaction ignores the table's own target.
+ table.alter.set_table_properties({"delta.targetFileSize": str(_SMALL_TARGET)})
files_before = len(table.file_uris())
rows_before = table.to_pyarrow_table().num_rows
@@ -415,17 +416,23 @@ class TestCompactionMemoryBounds:
@parameterized.expand(
[
# Incompressible data keeps delta-rs's own bin size; the slot fits a few bins at once.
- ("incompressible", 1.0, 1356.8, DEFAULT_COMPACT_TARGET_SIZE_BYTES, 6),
- ("modest_ratio", 2.0, 1356.8, DEFAULT_COMPACT_TARGET_SIZE_BYTES, 3),
+ ("incompressible", 1.0, 1356.8, _DEFAULT_TABLE_TARGET, DEFAULT_COMPACT_TARGET_SIZE_BYTES, 6),
+ ("modest_ratio", 2.0, 1356.8, _DEFAULT_TABLE_TARGET, DEFAULT_COMPACT_TARGET_SIZE_BYTES, 3),
# Wide JSON documents: 100 MB on disk decodes past the 2 GiB offset limit, so the bin shrinks
# until it decodes to half the slot and only one bin runs at a time. delta-rs leaves files
# larger than the bin alone, so the bin also caps the largest file that a compaction decodes.
- ("wide_json", 25.0, 1356.8, int(1356.8 * _MB / 2 / 25), 1),
- ("extreme_ratio_skips", 500.0, 1356.8, None, None),
- ("zero_budget_skips", 1.0, 0.0, None, None),
+ ("wide_json", 25.0, 1356.8, _DEFAULT_TABLE_TARGET, int(1356.8 * _MB / 2 / 25), 1),
+ ("extreme_ratio_skips", 500.0, 1356.8, _DEFAULT_TABLE_TARGET, None, None),
+ ("zero_budget_skips", 1.0, 0.0, _DEFAULT_TABLE_TARGET, None, None),
# No readable limit (local dev): only the decoded bin is capped, delta-rs picks parallelism.
- ("no_limit", 25.0, None, int(1024 * _MB / 25), None),
- ("no_limit_no_ratio_skips", None, None, None, None),
+ ("no_limit", 25.0, None, _DEFAULT_TABLE_TARGET, int(1024 * _MB / 25), None),
+ ("no_limit_no_ratio_skips", None, None, _DEFAULT_TABLE_TARGET, None, None),
+ # A table with its own smaller target keeps it, so compaction does not rebuild larger files,
+ # and the smaller bins run in parallel.
+ ("table_target_caps_the_bin", 4.0, 1356.8, 32 * _MB, 32 * _MB, 5),
+ ("table_target_no_limit", 1.0, None, 32 * _MB, 32 * _MB, None),
+ # The memory cap still wins when it is smaller than the table's target.
+ ("memory_cap_below_table_target", 25.0, 1356.8, 32 * _MB, int(1356.8 * _MB / 2 / 25), 1),
]
)
def test_plan_keeps_one_compaction_inside_its_slot(
@@ -433,11 +440,12 @@ class TestCompactionMemoryBounds:
_name: str,
ratio: float | None,
slot_mb: float | None,
+ table_target: int,
expected_target: int | None,
expected_tasks: int | None,
) -> None:
with patch(f"{_MAINTENANCE_MODULE}.os.cpu_count", return_value=7):
- plan = plan_compaction(ratio, slot_mb)
+ plan = plan_compaction(ratio, slot_mb, table_target)
assert plan.target_size == expected_target
assert plan.max_concurrent_tasks == expected_tasks
@@ -571,10 +579,13 @@ class TestDeltaliteCompaction:
_URI = "s3://bucket/table"
_SLOT_MB = 1356.8
- def _setup(self, compact: MagicMock | None, *, table_id: object = "table-id") -> tuple[DeltaMaintenance, Any, Any]:
+ def _setup(
+ self, compact: MagicMock | None, *, table_id: object = "table-id", configuration: dict[str, str] | None = None
+ ) -> tuple[DeltaMaintenance, Any, Any]:
# 300 small files in one partition trip the per-partition count trigger.
delta_table = _mock_table({f"f{i}.parquet": _MB for i in range(300)})
delta_table.metadata.return_value.id = table_id
+ delta_table.metadata.return_value.configuration = configuration or {}
table_ref = MagicMock()
table_ref.logger = make_logger()
table_ref.get_delta_table = AsyncMock(return_value=delta_table)
@@ -618,18 +629,26 @@ class TestDeltaliteCompaction:
@parameterized.expand(
[
- ("committed", 1, 152, [152]),
- ("nothing_to_rewrite", 0, 150, []),
+ ("committed", 1, 152, [152], {}, DEFAULT_COMPACT_TARGET_SIZE_BYTES),
+ ("nothing_to_rewrite", 0, 150, [], {}, DEFAULT_COMPACT_TARGET_SIZE_BYTES),
+ ("table_target", 1, 152, [152], {"delta.targetFileSize": str(32 * _MB)}, 32 * _MB),
+ ("invalid_table_target", 1, 152, [152], {"delta.targetFileSize": "-1"}, DEFAULT_COMPACT_TARGET_SIZE_BYTES),
]
)
@pytest.mark.asyncio
async def test_compacts_through_a_leased_deltalite_handle(
- self, _name: str, commits: int, version: int, expected_noted: list[int]
+ self,
+ _name: str,
+ commits: int,
+ version: int,
+ expected_noted: list[int],
+ configuration: dict[str, str],
+ expected_target: int,
) -> None:
compact = MagicMock(
return_value={"commits": commits, "version": version, "numFilesAdded": 3, "numFilesRemoved": 300}
)
- maintenance, table_ref, fake_class = self._setup(compact)
+ maintenance, table_ref, fake_class = self._setup(compact, configuration=configuration)
with patch(f"{_MAINTENANCE_MODULE}._sample_compression_ratio", side_effect=AssertionError("sampled")):
ran, delta_rs_compact, plan, capture = await self._run(maintenance, fake_class)
@@ -640,7 +659,7 @@ class TestDeltaliteCompaction:
capture.assert_not_called()
fake_class.open.assert_called_once_with(self._URI, {"AWS_REGION": "us-east-1"})
compact.assert_called_once_with(
- target_file_size=DEFAULT_COMPACT_TARGET_SIZE_BYTES,
+ target_file_size=expected_target,
max_parallel_bins=7,
slot_budget_bytes=int(self._SLOT_MB * _MB),
commit_metadata={"compact_engine": "deltalite", "job_id": "job-1"},
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_memory_governor.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_memory_governor.py
index 618f70e0d5a0..9154a016b39a 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_memory_governor.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_memory_governor.py
@@ -310,6 +310,28 @@ class TestSizeUpsert:
# Unknown shapes assume target-sized files of 64 columns in every worker.
("unknown_shape_is_conservative", None, 4, 1.4, _SLOT, 3, 5, True),
("unknown_shape_does_not_fit_a_small_slot", None, 4, 1.4, _SLOT_14GI, 1, 1, False),
+ # The same partitions of wide rows in default-size files and in 32 MB files: the smaller row
+ # groups and write buffers leave room for more workers and readers in the same slice.
+ (
+ "wide_rows_in_default_files",
+ _profile(*([[100.0] * 2] * 4), columns=30),
+ None,
+ 1.4,
+ _SLOT_14GI,
+ 1,
+ 1,
+ True,
+ ),
+ (
+ "wide_rows_in_smaller_files",
+ _profile(*([[32.0] * 6] * 4), columns=30, target_mb=32.0),
+ None,
+ 1.4,
+ _SLOT_14GI,
+ 2,
+ 8,
+ True,
+ ),
]
)
def test_sizing(self, _name, profile, n_partitions, retention, slot, exp_mpp, exp_files, exp_fits):
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_ops.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_ops.py
index e5a350a1a3dd..c05ff2489c64 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_ops.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_ops.py
@@ -3,6 +3,7 @@ from unittest.mock import MagicMock
from django.test import override_settings
+import pyarrow as pa
import deltalake
import botocore.exceptions
from parameterized import parameterized
@@ -15,6 +16,7 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.del
ObjectStorePermissionDeniedError,
delta_merge_spill_kwargs,
execute_with_conflict_retry,
+ wide_row_target_file_size,
)
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.test.helpers import make_logger
@@ -31,6 +33,49 @@ def _permission_error_with_client_error_cause() -> PermissionError:
return permission_error
+_KiB = 1024
+_MiB = 1024 * 1024
+
+
+def _add_actions(files: list[tuple[int, int | None]]) -> pa.Table:
+ return pa.table(
+ {
+ "size_bytes": pa.array([size for size, _ in files], pa.int64()),
+ "num_records": pa.array([rows for _, rows in files], pa.int64()),
+ }
+ )
+
+
+class TestWideRowTargetFileSize:
+ @parameterized.expand(
+ [
+ ("empty_table", [], None),
+ ("narrow_rows", [(64 * _MiB, 64 * 1024)], None),
+ ("just_under_the_wide_threshold", [(64 * _MiB, 64 * _MiB // (8 * _KiB - 1))], None),
+ # 1,024 rows of 8 KiB is 8 MiB, which the lower bound raises to 16 MiB.
+ ("wide_threshold_clamps_to_the_lower_bound", [(64 * _MiB, 64 * _MiB // (8 * _KiB))], 16 * _MiB),
+ ("inside_the_band", [(100 * _MiB, 100 * _MiB // (20 * _KiB))], 20 * _MiB),
+ ("very_wide_rows_clamp_to_the_upper_bound", [(100 * _MiB, 100 * _MiB // (100 * _KiB))], 32 * _MiB),
+ # Bytes per row come from the whole table, not from the largest file.
+ ("weighted_over_files", [(80 * _MiB, 80 * _MiB // (40 * _KiB)), (20 * _MiB, 20 * 1024)], None),
+ ("small_wide_table_waits", [(31 * _MiB, 31 * _MiB // (100 * _KiB))], None),
+ # A file without a row count says nothing about bytes per row, so it does not count.
+ (
+ "files_without_row_counts_are_skipped",
+ [(100 * _MiB, 100 * _MiB // (20 * _KiB)), (500 * _MiB, None)],
+ 20 * _MiB,
+ ),
+ ("no_file_records_its_rows", [(100 * _MiB, None)], None),
+ ("rounds_to_whole_mib", [(100 * _MiB, round(100 * _MiB / (20.3 * _KiB)))], 20 * _MiB),
+ ]
+ )
+ def test_target(self, _name: str, files: list[tuple[int, int | None]], expected: int | None) -> None:
+ assert wide_row_target_file_size(_add_actions(files)) == expected
+
+ def test_add_actions_without_row_counts(self) -> None:
+ assert wide_row_target_file_size(pa.table({"size_bytes": pa.array([100 * _MiB], pa.int64())})) is None
+
+
class TestDeltaMergeSpillKwargs:
# Guards the wiring from settings → delta-rs merge kwargs. A silent break here (renamed setting,
# dropped forwarding, or emitting a None kwarg) stops merges spilling to disk and OOMs return.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_writer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_writer.py
index 126784a5def4..2ca0d9467acd 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_writer.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_writer.py
@@ -1,3 +1,4 @@
+import os
import json
from decimal import Decimal
from pathlib import Path
@@ -56,6 +57,7 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.del
_REPARTITION_MODULE = "products.warehouse_sources.backend.temporal.data_imports.pipelines.core.repartition"
_CONTROLLER_MODULE = "products.warehouse_sources.backend.temporal.data_imports.pipelines.core.repartition_controller"
_WRITER_MODULE = "products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.writer"
+_OPS_MODULE = "products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.ops"
_COMMIT_LAYOUT_CASES: list[tuple[str, list[dict], dict, bool]] = [
@@ -886,6 +888,66 @@ class TestCheckpointIntervalProperty:
assert result is False
+def _rows(ids: list[int], payload_chars: int) -> pa.Table:
+ # Random hex does not compress much, so the stored bytes per row stay near the payload size.
+ return pa.table({"id": ids, "blob": [os.urandom(payload_chars).hex()[:payload_chars] for _ in ids]})
+
+
+class TestWideRowTargetFileSize:
+ @pytest.mark.parametrize(
+ "write_type,primary_keys,payload_chars,existing_target,expected_target",
+ [
+ ("incremental", ["id"], 4000, None, str(16 * 1024 * 1024)),
+ ("incremental", ["id"], 1, None, None),
+ ("append", None, 4000, None, None),
+ ("full_refresh", None, 4000, None, None),
+ ("incremental", ["id"], 4000, str(64 * 1024 * 1024), str(64 * 1024 * 1024)),
+ ],
+ ids=[
+ "wide_merge_table_gets_a_smaller_target",
+ "narrow_merge_table_keeps_the_default",
+ "wide_append_table_keeps_the_default",
+ "wide_full_refresh_table_keeps_the_default",
+ "hand_set_target_is_kept",
+ ],
+ )
+ @pytest.mark.asyncio
+ async def test_target_after_writes(
+ self,
+ write_type: Literal["incremental", "append", "full_refresh"],
+ primary_keys: list[str] | None,
+ payload_chars: int,
+ existing_target: str | None,
+ expected_target: str | None,
+ tmp_path: Path,
+ ) -> None:
+ delta_path = str(tmp_path / "table")
+ deltalake.write_deltalake(
+ delta_path,
+ _rows(list(range(20)), payload_chars),
+ configuration={"delta.targetFileSize": existing_target} if existing_target else None,
+ )
+ table_ref = make_local_table_ref(delta_path)
+
+ # Local tables are far below the production size gates, so only those gates are lowered.
+ with (
+ patch(f"{_OPS_MODULE}.WIDE_ROW_MIN_TABLE_BYTES", 0),
+ patch(f"{_OPS_MODULE}.WIDE_ROW_MIN_STORED_BYTES_PER_ROW", 1024),
+ ):
+ for ids in ([1, 2], [3, 4]):
+ await DeltaWriter(table_ref).write(
+ data=_rows(ids, payload_chars),
+ write_type=write_type,
+ should_overwrite_table=False,
+ primary_keys=primary_keys,
+ )
+
+ configuration = deltalake.DeltaTable(delta_path).metadata().configuration
+ assert configuration.get("delta.targetFileSize") == expected_target
+ # The checkpoint property and the target land in one commit, and the second write adds none.
+ assert _commit_operations(delta_path).count("SET TBLPROPERTIES") == 1
+
+
class TestCreateRaceWithExistingTable:
"""DeltaTable.create() defaults to mode="error", raising "table already exists at that
location" whenever the destination is non-empty. get_delta_table() can report "no table
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/writer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/writer.py
index cc64bd8d8169..2f3e42b4d94b 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/writer.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/writer.py
@@ -758,7 +758,8 @@ class DeltaWriter:
delta_table = await self._table.get_delta_table(allow_stale=True)
assert delta_table is not None
- await ensure_table_properties(delta_table, self._logger)
+ merges = write_type == "incremental" and bool(primary_keys)
+ await ensure_table_properties(delta_table, self._logger, merge_batch=data if merges else None)
return delta_table