:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(data-warehouse): use smaller files for wide-row merge tables (#119682)
Tom Owers57800f8ebb3b
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/maintenance.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/maintenance.pyindex dfff4a1f0756..cca8c1c6c0dd 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/maintenance.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/maintenance.py@@ -35,11 +35,13 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.del)from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.memory_governor import get_governorfrom products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.ops import (+ DEFAULT_TARGET_FILE_SIZE_BYTES,OBJECT_STORE_PERMISSION_DENIED_MESSAGE,OBJECT_STORE_TRANSIENT_MESSAGE,ObjectStorePermissionDeniedError,execute_with_conflict_retry,is_object_store_permission_denied,+ table_target_file_size,)from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.writer import _delta_table_identityfrom products.warehouse_sources.backend.temporal.data_imports.pipelines.core.partitioning import (@@ -89,9 +91,11 @@ DEFAULT_COMPACT_REMOVABLE_FILES_PER_PARTITION_THRESHOLD = 8DEFAULT_COMPACT_REMOVABLE_FILES_THRESHOLD = 100# delta-rs's own default target size for a compaction rewrite bin when none is given (see-# `delta.targetFileSize`, or 100MB absent that table property). Starting the retry ladder here-# instead of passing None keeps every attempt's size explicit and halvable.-DEFAULT_COMPACT_TARGET_SIZE_BYTES = 100 * 1024 * 1024+# `delta.targetFileSize`, or 100MB absent that table property). A table that sets the property+# compacts to its own value instead (see `table_target_file_size`), so that compaction keeps the+# file size its writes use. Starting the retry ladder at an explicit size instead of passing None+# keeps every attempt's size explicit and halvable.+DEFAULT_COMPACT_TARGET_SIZE_BYTES = DEFAULT_TARGET_FILE_SIZE_BYTES# Halving the bin size on a byte-array-offset-overflow panic (see is_offset_overflow_compaction_error)# shrinks how many files get concatenated into one Arrow batch. Bounded so a table whose overflow# can't be avoided at any reasonable bin size still surfaces to error tracking instead of looping.@@ -123,8 +127,12 @@ class CompactionPlan:slot_budget_mb: float | None-def plan_compaction(compression_ratio: float | None, slot_budget_mb: float | None) -> CompactionPlan:- """Bin size and parallelism that keep one compaction inside its memory slot."""+def plan_compaction(+ compression_ratio: float | None,+ slot_budget_mb: float | None,+ table_target_size: int = DEFAULT_COMPACT_TARGET_SIZE_BYTES,+) -> CompactionPlan:+ """Bin size and parallelism that keep one compaction inside its memory slot, at most `table_target_size`."""rounded_ratio = round(compression_ratio, 2) if compression_ratio is not None else Nonerounded_slot = round(slot_budget_mb, 1) if slot_budget_mb is not None else Noneif compression_ratio is None or (slot_budget_mb is not None and slot_budget_mb <= 0):@@ -149,7 +157,7 @@ def plan_compaction(compression_ratio: float | None, slot_budget_mb: float | Nonslot_budget_mb=rounded_slot,)- target_size = int(min(DEFAULT_COMPACT_TARGET_SIZE_BYTES, safe_target_size))+ target_size = int(min(table_target_size, safe_target_size))max_concurrent_tasks = Noneif slot_budget_bytes is not None:per_task_bytes = target_size * ratio * _COMPACT_TASK_MEMORY_FACTOR@@ -468,7 +476,7 @@ class DeltaMaintenance:except Exception as e:await self._logger.awarning(f"compact: could not sample the compression ratio: {e}")ratio = None- return plan_compaction(ratio, get_governor().slot_budget_mb())+ return plan_compaction(ratio, get_governor().slot_budget_mb(), table_target_file_size(table))@recorded_phase("compact")async def _compact(@@ -566,8 +574,9 @@ class DeltaMaintenance:storage_options = self._table.get_storage_options()identity = _delta_table_identity(table, self._table)slot_budget_mb = get_governor().slot_budget_mb()+ target_size = table_target_file_size(table)compact_kwargs: dict[str, Any] = {- "target_file_size": DEFAULT_COMPACT_TARGET_SIZE_BYTES,+ "target_file_size": target_size,# deltalite lowers this until the bins in flight fit the slot budget."max_parallel_bins": os.cpu_count() or 1,"slot_budget_bytes": int(slot_budget_mb * 1024 * 1024) if slot_budget_mb else None,@@ -593,7 +602,7 @@ class DeltaMaintenance:await self._logger.ainfo("Compacting table...",compact_engine="deltalite",- compact_target_size=DEFAULT_COMPACT_TARGET_SIZE_BYTES,+ compact_target_size=target_size,compact_max_parallel_bins=compact_kwargs["max_parallel_bins"],compact_slot_budget_mb=round(slot_budget_mb, 1) if slot_budget_mb is not None else None,pod_memory_mb=pod_mb_before,@@ -778,7 +787,8 @@ class DeltaMaintenance:compact_small_files and small_file_stats.fragmented)- partitions = await asyncio.to_thread(_partition_file_stats, table, DEFAULT_COMPACT_TARGET_SIZE_BYTES)+ table_target_size = table_target_file_size(table)+ partitions = await asyncio.to_thread(_partition_file_stats, table, table_target_size)# Repartition detection runs after this pass and compares partition bytes to its budget.# Small files take more bytes at rest than the same rows after compaction, so an# over-budget partition that compaction can shrink must be compacted before it is measured.@@ -813,11 +823,10 @@ class DeltaMaintenance:`select_partitions` limits the pass, from the partition stats at the bin size the engine uses."""+ table_target_size = table_target_file_size(table)partitions = Noneif select_partitions is not None:- partitions = select_partitions(- await asyncio.to_thread(_partition_file_stats, table, DEFAULT_COMPACT_TARGET_SIZE_BYTES)- )+ partitions = select_partitions(await asyncio.to_thread(_partition_file_stats, table, table_target_size))if settings.DATA_WAREHOUSE_DELTALITE_COMPACTION:await self._logger.ainfo(f"{pass_name}: triggering compact ({stats})")compacted = await self._compact_with_deltalite(table, partitions=partitions)@@ -833,7 +842,7 @@ class DeltaMaintenance:await self._compact(table, plan, partitions=partitions)return False- if plan.target_size != DEFAULT_COMPACT_TARGET_SIZE_BYTES:+ if plan.target_size != table_target_size:# A smaller bin can merge fewer of the files, so the trigger must hold at the planned size too.planned = await asyncio.to_thread(_partition_file_stats, table, plan.target_size)if not triggered(planned):@@ -861,7 +870,7 @@ class DeltaMaintenance:if table is None:return "skipped"- partitions = await asyncio.to_thread(_partition_file_stats, table, DEFAULT_COMPACT_TARGET_SIZE_BYTES)+ partitions = await asyncio.to_thread(_partition_file_stats, table, table_target_file_size(table))removable_files = _small_file_fragmentation(partitions, table_wide_small_files=False).total_removableif removable_files < min_removable_files:return "skipped"diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/memory_governor.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/memory_governor.pyindex ec1a5aeb3e71..ec529e18250c 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/memory_governor.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/memory_governor.py@@ -65,6 +65,10 @@ import pyarrow.compute as pcfrom posthog.dataclasses import frozen+from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.ops import (+ DEFAULT_TARGET_FILE_SIZE_BYTES,+ table_target_file_size,+)from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.rss_sampler import (RssPeakSampler,RssWindow,@@ -132,8 +136,7 @@ _SOURCE_INUSE_PER_MB = 0.73#: deltalite keeps one add action per table file in its snapshot. Measured at ~2.2 KB per file on#: the 30-column harness tables (6,000 files hold 12 MB more than 600 files)._SNAPSHOT_KB_PER_FILE = 2.2-#: deltalite and delta-rs both default to 100 MiB when the table does not set ``delta.targetFileSize``.-_DEFAULT_TARGET_FILE_SIZE = 100 * MB+_DEFAULT_TARGET_FILE_SIZE = DEFAULT_TARGET_FILE_SIZE_BYTES#: RSS per byte of malloc in-use memory. glibc keeps freed memory in its arenas, and with its#: default mmap threshold the large Arrow buffers stay there too. RSS ran 1.25–2.1× in-use in most@@ -360,21 +363,13 @@ def rewrite_profile()-def _table_target_file_size(table: deltalake.DeltaTable) -> int:- raw = table.metadata().configuration.get("delta.targetFileSize")- try:- return int(raw) if raw and int(raw) > 0 else _DEFAULT_TARGET_FILE_SIZE- except ValueError:- return _DEFAULT_TARGET_FILE_SIZE--def estimate_rewrite_profile(table: deltalake.DeltaTable, source: pa.Table, partition_col: str | None, primary_keys: Sequence[str]) -> RewriteProfile | None:"""``rewrite_profile`` for a live table, or None when its add actions cannot be read."""try:add_actions = pa.table(table.get_add_actions(flatten=True))- return rewrite_profile(add_actions, source, partition_col, primary_keys, _table_target_file_size(table))+ return rewrite_profile(add_actions, source, partition_col, primary_keys, table_target_file_size(table))except Exception as e: # noqa: BLE001 - an estimate must never fail a write; None sizes conservativelylogger.debug("deltalite governor: could not estimate the rewrite size: %s", e)return Nonediff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/ops.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/ops.pyindex 276152a19a66..ef48093ef05d 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/ops.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/ops.py@@ -4,7 +4,9 @@ from typing import TypedDict, TypeVarfrom django.conf import settings+import pyarrow as paimport deltalake+import pyarrow.compute as pcimport botocore.exceptionsimport deltalake.exceptionsfrom structlog.types import FilteringBoundLogger@@ -189,14 +191,84 @@ async def execute_with_conflict_retry(# property when it commits, so both writers checkpoint on the same cadence.DELTA_TABLE_PROPERTIES: dict[str, str] = {"delta.checkpointInterval": "25"}+MiB = 1024 * 1024+TARGET_FILE_SIZE_PROPERTY = "delta.targetFileSize"+#: deltalite and delta-rs both write files up to this size when a table does not set the property.+DEFAULT_TARGET_FILE_SIZE_BYTES = 100 * MiB-async def ensure_table_properties(table: deltalake.DeltaTable, logger: FilteringBoundLogger) -> bool:+# A merge rewrites every file that holds a matched row, and deltalite writes each file as a single+# row group. When rows are wide, a default-size file holds few rows, so a small update set touches+# almost every file and copies many unchanged rows. A large row group also leaves the memory+# governor room for fewer readers and partition workers in one upsert's slice, often only one of+# each. Smaller files for these tables lower both costs.+#: Stored bytes per row at or above which a merge table counts as wide. A default-size file of such+#: rows holds at most 12,800 rows.+WIDE_ROW_MIN_STORED_BYTES_PER_ROW = 8 * 1024+#: Rows that one file of a wide-row table holds, before the size band below applies.+WIDE_ROW_TARGET_ROWS_PER_FILE = 1024+#: Smaller files cost more footer probes and log entries than they save on a rewrite.+WIDE_ROW_MIN_TARGET_FILE_SIZE_BYTES = 16 * MiB+#: Small enough that the governor can plan several readers and partition workers per upsert.+WIDE_ROW_MAX_TARGET_FILE_SIZE_BYTES = 32 * MiB+#: A smaller table fits in one file at either target, and its per-row estimate still includes the+#: Parquet footer overhead, so the decision waits until the table holds this much data.+WIDE_ROW_MIN_TABLE_BYTES = WIDE_ROW_MAX_TARGET_FILE_SIZE_BYTES+++def table_target_file_size(table: deltalake.DeltaTable) -> int:+ """The table's `delta.targetFileSize` in bytes, or the writers' default when it is absent or invalid."""+ raw = (table.metadata().configuration or {}).get(TARGET_FILE_SIZE_PROPERTY)+ if not isinstance(raw, str) or not raw.strip().isdigit() or int(raw) <= 0:+ return DEFAULT_TARGET_FILE_SIZE_BYTES+ return int(raw)+++def wide_row_target_file_size(add_actions: pa.Table) -> int | None:+ """A smaller `delta.targetFileSize` for a table of wide rows, or None to keep the default.++ Bytes per row come from the add actions: the stored size and the row count of each file that+ records its row count. The result is a whole number of MiB, so the same table always gets the same value.+ """+ if add_actions.num_rows == 0 or not {"size_bytes", "num_records"} <= set(add_actions.column_names):+ return None+ counted = add_actions.filter(pc.greater(add_actions["num_records"], pa.scalar(0)))+ stored_bytes = pc.sum(counted["size_bytes"]).as_py() or 0+ rows = pc.sum(counted["num_records"]).as_py() or 0+ if rows == 0 or stored_bytes < WIDE_ROW_MIN_TABLE_BYTES:+ return None+ bytes_per_row = stored_bytes / rows+ if bytes_per_row < WIDE_ROW_MIN_STORED_BYTES_PER_ROW:+ return None+ target_mib = round(WIDE_ROW_TARGET_ROWS_PER_FILE * bytes_per_row / MiB)+ return max(WIDE_ROW_MIN_TARGET_FILE_SIZE_BYTES, min(WIDE_ROW_MAX_TARGET_FILE_SIZE_BYTES, target_mib * MiB))+++def _may_hold_wide_rows(batch: pa.Table) -> bool:+ # Arrow holds a row in at least as many bytes as Parquet stores it in, so a narrow batch rules+ # out a wide table without the add-action read, which costs O(files) on every write.+ return batch.num_rows > 0 and batch.nbytes / batch.num_rows >= WIDE_ROW_MIN_STORED_BYTES_PER_ROW+++def _wide_row_table_properties(table: deltalake.DeltaTable) -> dict[str, str]:+ add_actions = pa.table(table.get_add_actions(flatten=False)).select(["size_bytes", "num_records"])+ target = wide_row_target_file_size(add_actions)+ return {} if target is None else {TARGET_FILE_SIZE_PROPERTY: str(target)}+++async def ensure_table_properties(+ table: deltalake.DeltaTable, logger: FilteringBoundLogger, *, merge_batch: pa.Table | None = None+) -> bool:"""Apply DELTA_TABLE_PROPERTIES to a table that was created without them.A metadata-only commit, made once per table: the check reads the handle's own snapshot, and`set_table_properties` refreshes that snapshot, so a table that already carries the values costsnothing here. Never raises, because the data write has already committed and a property thatfails to land only waits for the next write. Returns True when it committed.++ `merge_batch` is the batch just merged into a table that loads by merging on a primary key, and+ None for every other write. Such a table of wide rows also gets a smaller `delta.targetFileSize`+ (see `wide_row_target_file_size`). The value is set only when the table has none, so it never+ changes after that and never replaces a value set by hand."""try:# The metadata read is in the same best-effort boundary as the write below: it touches the@@ -204,6 +276,8 @@ async def ensure_table_properties(table: deltalake.DeltaTable, logger: Filtering# the data commit has already landed either way, so a failure here must not propagate either.current = table.metadata().configuration or {}missing = {key: value for key, value in DELTA_TABLE_PROPERTIES.items() if current.get(key) != value}+ if merge_batch is not None and TARGET_FILE_SIZE_PROPERTY not in current and _may_hold_wide_rows(merge_batch):+ missing.update(await asyncio.to_thread(_wide_row_table_properties, table))if not missing:return Falseawait execute_with_conflict_retry(diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_maintenance.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_maintenance.pyindex 05809fb43526..8e47bb00f072 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_maintenance.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_maintenance.py@@ -52,6 +52,7 @@ _MAINTENANCE_MODULE = "products.warehouse_sources.backend.temporal.data_imports._MB = 1024 * 1024# Small enough that a local table can hold files above the compaction target._SMALL_TARGET = 100_000+_DEFAULT_TABLE_TARGET = 100 * 1024 * 1024_NOW = dt.datetime(2026, 6, 1, 12, 0, tzinfo=dt.UTC)_DEFAULT_PLAN = CompactionPlan(target_size=DEFAULT_COMPACT_TARGET_SIZE_BYTES, max_concurrent_tasks=1, compression_ratio=1.0, slot_budget_mb=1356.8@@ -281,15 +282,15 @@ class TestCompactIfFragmented:)@pytest.mark.asyncioasync def test_small_file_trigger_on_a_real_table(self, _name: str, interleave_oversized: bool, expected_ran: bool):- with (- tempfile.TemporaryDirectory() as path,- patch(f"{_MAINTENANCE_MODULE}.DEFAULT_COMPACT_TARGET_SIZE_BYTES", _SMALL_TARGET),- ):+ with tempfile.TemporaryDirectory() as path:for i in range(12):_append_file(path, i, payload_bytes=500)if interleave_oversized:_append_file(path, 100 + i, payload_bytes=2 * _SMALL_TARGET)table = deltalake.DeltaTable(path)+ # The oversized files are small next to the default target, so the skip case also fails+ # when compaction ignores the table's own target.+ table.alter.set_table_properties({"delta.targetFileSize": str(_SMALL_TARGET)})files_before = len(table.file_uris())rows_before = table.to_pyarrow_table().num_rows@@ -415,17 +416,23 @@ class TestCompactionMemoryBounds:@parameterized.expand([# Incompressible data keeps delta-rs's own bin size; the slot fits a few bins at once.- ("incompressible", 1.0, 1356.8, DEFAULT_COMPACT_TARGET_SIZE_BYTES, 6),- ("modest_ratio", 2.0, 1356.8, DEFAULT_COMPACT_TARGET_SIZE_BYTES, 3),+ ("incompressible", 1.0, 1356.8, _DEFAULT_TABLE_TARGET, DEFAULT_COMPACT_TARGET_SIZE_BYTES, 6),+ ("modest_ratio", 2.0, 1356.8, _DEFAULT_TABLE_TARGET, DEFAULT_COMPACT_TARGET_SIZE_BYTES, 3),# Wide JSON documents: 100 MB on disk decodes past the 2 GiB offset limit, so the bin shrinks# until it decodes to half the slot and only one bin runs at a time. delta-rs leaves files# larger than the bin alone, so the bin also caps the largest file that a compaction decodes.- ("wide_json", 25.0, 1356.8, int(1356.8 * _MB / 2 / 25), 1),- ("extreme_ratio_skips", 500.0, 1356.8, None, None),- ("zero_budget_skips", 1.0, 0.0, None, None),+ ("wide_json", 25.0, 1356.8, _DEFAULT_TABLE_TARGET, int(1356.8 * _MB / 2 / 25), 1),+ ("extreme_ratio_skips", 500.0, 1356.8, _DEFAULT_TABLE_TARGET, None, None),+ ("zero_budget_skips", 1.0, 0.0, _DEFAULT_TABLE_TARGET, None, None),# No readable limit (local dev): only the decoded bin is capped, delta-rs picks parallelism.- ("no_limit", 25.0, None, int(1024 * _MB / 25), None),- ("no_limit_no_ratio_skips", None, None, None, None),+ ("no_limit", 25.0, None, _DEFAULT_TABLE_TARGET, int(1024 * _MB / 25), None),+ ("no_limit_no_ratio_skips", None, None, _DEFAULT_TABLE_TARGET, None, None),+ # A table with its own smaller target keeps it, so compaction does not rebuild larger files,+ # and the smaller bins run in parallel.+ ("table_target_caps_the_bin", 4.0, 1356.8, 32 * _MB, 32 * _MB, 5),+ ("table_target_no_limit", 1.0, None, 32 * _MB, 32 * _MB, None),+ # The memory cap still wins when it is smaller than the table's target.+ ("memory_cap_below_table_target", 25.0, 1356.8, 32 * _MB, int(1356.8 * _MB / 2 / 25), 1),])def test_plan_keeps_one_compaction_inside_its_slot(@@ -433,11 +440,12 @@ class TestCompactionMemoryBounds:_name: str,ratio: float | None,slot_mb: float | None,+ table_target: int,expected_target: int | None,expected_tasks: int | None,) -> None:with patch(f"{_MAINTENANCE_MODULE}.os.cpu_count", return_value=7):- plan = plan_compaction(ratio, slot_mb)+ plan = plan_compaction(ratio, slot_mb, table_target)assert plan.target_size == expected_targetassert plan.max_concurrent_tasks == expected_tasks@@ -571,10 +579,13 @@ class TestDeltaliteCompaction:_URI = "s3://bucket/table"_SLOT_MB = 1356.8- def _setup(self, compact: MagicMock | None, *, table_id: object = "table-id") -> tuple[DeltaMaintenance, Any, Any]:+ def _setup(+ self, compact: MagicMock | None, *, table_id: object = "table-id", configuration: dict[str, str] | None = None+ ) -> tuple[DeltaMaintenance, Any, Any]:# 300 small files in one partition trip the per-partition count trigger.delta_table = _mock_table({f"f{i}.parquet": _MB for i in range(300)})delta_table.metadata.return_value.id = table_id+ delta_table.metadata.return_value.configuration = configuration or {}table_ref = MagicMock()table_ref.logger = make_logger()table_ref.get_delta_table = AsyncMock(return_value=delta_table)@@ -618,18 +629,26 @@ class TestDeltaliteCompaction:@parameterized.expand([- ("committed", 1, 152, [152]),- ("nothing_to_rewrite", 0, 150, []),+ ("committed", 1, 152, [152], {}, DEFAULT_COMPACT_TARGET_SIZE_BYTES),+ ("nothing_to_rewrite", 0, 150, [], {}, DEFAULT_COMPACT_TARGET_SIZE_BYTES),+ ("table_target", 1, 152, [152], {"delta.targetFileSize": str(32 * _MB)}, 32 * _MB),+ ("invalid_table_target", 1, 152, [152], {"delta.targetFileSize": "-1"}, DEFAULT_COMPACT_TARGET_SIZE_BYTES),])@pytest.mark.asyncioasync def test_compacts_through_a_leased_deltalite_handle(- self, _name: str, commits: int, version: int, expected_noted: list[int]+ self,+ _name: str,+ commits: int,+ version: int,+ expected_noted: list[int],+ configuration: dict[str, str],+ expected_target: int,) -> None:compact = MagicMock(return_value={"commits": commits, "version": version, "numFilesAdded": 3, "numFilesRemoved": 300})- maintenance, table_ref, fake_class = self._setup(compact)+ maintenance, table_ref, fake_class = self._setup(compact, configuration=configuration)with patch(f"{_MAINTENANCE_MODULE}._sample_compression_ratio", side_effect=AssertionError("sampled")):ran, delta_rs_compact, plan, capture = await self._run(maintenance, fake_class)@@ -640,7 +659,7 @@ class TestDeltaliteCompaction:capture.assert_not_called()fake_class.open.assert_called_once_with(self._URI, {"AWS_REGION": "us-east-1"})compact.assert_called_once_with(- target_file_size=DEFAULT_COMPACT_TARGET_SIZE_BYTES,+ target_file_size=expected_target,max_parallel_bins=7,slot_budget_bytes=int(self._SLOT_MB * _MB),commit_metadata={"compact_engine": "deltalite", "job_id": "job-1"},diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_memory_governor.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_memory_governor.pyindex 618f70e0d5a0..9154a016b39a 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_memory_governor.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_memory_governor.py@@ -310,6 +310,28 @@ class TestSizeUpsert:# Unknown shapes assume target-sized files of 64 columns in every worker.("unknown_shape_is_conservative", None, 4, 1.4, _SLOT, 3, 5, True),("unknown_shape_does_not_fit_a_small_slot", None, 4, 1.4, _SLOT_14GI, 1, 1, False),+ # The same partitions of wide rows in default-size files and in 32 MB files: the smaller row+ # groups and write buffers leave room for more workers and readers in the same slice.+ (+ "wide_rows_in_default_files",+ _profile(*([[100.0] * 2] * 4), columns=30),+ None,+ 1.4,+ _SLOT_14GI,+ 1,+ 1,+ True,+ ),+ (+ "wide_rows_in_smaller_files",+ _profile(*([[32.0] * 6] * 4), columns=30, target_mb=32.0),+ None,+ 1.4,+ _SLOT_14GI,+ 2,+ 8,+ True,+ ),])def test_sizing(self, _name, profile, n_partitions, retention, slot, exp_mpp, exp_files, exp_fits):diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_ops.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_ops.pyindex e5a350a1a3dd..c05ff2489c64 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_ops.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_ops.py@@ -3,6 +3,7 @@ from unittest.mock import MagicMockfrom django.test import override_settings+import pyarrow as paimport deltalakeimport botocore.exceptionsfrom parameterized import parameterized@@ -15,6 +16,7 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delObjectStorePermissionDeniedError,delta_merge_spill_kwargs,execute_with_conflict_retry,+ wide_row_target_file_size,)from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.test.helpers import make_logger@@ -31,6 +33,49 @@ def _permission_error_with_client_error_cause() -> PermissionError:return permission_error+_KiB = 1024+_MiB = 1024 * 1024+++def _add_actions(files: list[tuple[int, int | None]]) -> pa.Table:+ return pa.table(+ {+ "size_bytes": pa.array([size for size, _ in files], pa.int64()),+ "num_records": pa.array([rows for _, rows in files], pa.int64()),+ }+ )+++class TestWideRowTargetFileSize:+ @parameterized.expand(+ [+ ("empty_table", [], None),+ ("narrow_rows", [(64 * _MiB, 64 * 1024)], None),+ ("just_under_the_wide_threshold", [(64 * _MiB, 64 * _MiB // (8 * _KiB - 1))], None),+ # 1,024 rows of 8 KiB is 8 MiB, which the lower bound raises to 16 MiB.+ ("wide_threshold_clamps_to_the_lower_bound", [(64 * _MiB, 64 * _MiB // (8 * _KiB))], 16 * _MiB),+ ("inside_the_band", [(100 * _MiB, 100 * _MiB // (20 * _KiB))], 20 * _MiB),+ ("very_wide_rows_clamp_to_the_upper_bound", [(100 * _MiB, 100 * _MiB // (100 * _KiB))], 32 * _MiB),+ # Bytes per row come from the whole table, not from the largest file.+ ("weighted_over_files", [(80 * _MiB, 80 * _MiB // (40 * _KiB)), (20 * _MiB, 20 * 1024)], None),+ ("small_wide_table_waits", [(31 * _MiB, 31 * _MiB // (100 * _KiB))], None),+ # A file without a row count says nothing about bytes per row, so it does not count.+ (+ "files_without_row_counts_are_skipped",+ [(100 * _MiB, 100 * _MiB // (20 * _KiB)), (500 * _MiB, None)],+ 20 * _MiB,+ ),+ ("no_file_records_its_rows", [(100 * _MiB, None)], None),+ ("rounds_to_whole_mib", [(100 * _MiB, round(100 * _MiB / (20.3 * _KiB)))], 20 * _MiB),+ ]+ )+ def test_target(self, _name: str, files: list[tuple[int, int | None]], expected: int | None) -> None:+ assert wide_row_target_file_size(_add_actions(files)) == expected++ def test_add_actions_without_row_counts(self) -> None:+ assert wide_row_target_file_size(pa.table({"size_bytes": pa.array([100 * _MiB], pa.int64())})) is None++class TestDeltaMergeSpillKwargs:# Guards the wiring from settings → delta-rs merge kwargs. A silent break here (renamed setting,# dropped forwarding, or emitting a None kwarg) stops merges spilling to disk and OOMs return.diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_writer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_writer.pyindex 126784a5def4..2ca0d9467acd 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_writer.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_writer.py@@ -1,3 +1,4 @@+import osimport jsonfrom decimal import Decimalfrom pathlib import Path@@ -56,6 +57,7 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.del_REPARTITION_MODULE = "products.warehouse_sources.backend.temporal.data_imports.pipelines.core.repartition"_CONTROLLER_MODULE = "products.warehouse_sources.backend.temporal.data_imports.pipelines.core.repartition_controller"_WRITER_MODULE = "products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.writer"+_OPS_MODULE = "products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.ops"_COMMIT_LAYOUT_CASES: list[tuple[str, list[dict], dict, bool]] = [@@ -886,6 +888,66 @@ class TestCheckpointIntervalProperty:assert result is False+def _rows(ids: list[int], payload_chars: int) -> pa.Table:+ # Random hex does not compress much, so the stored bytes per row stay near the payload size.+ return pa.table({"id": ids, "blob": [os.urandom(payload_chars).hex()[:payload_chars] for _ in ids]})+++class TestWideRowTargetFileSize:+ @pytest.mark.parametrize(+ "write_type,primary_keys,payload_chars,existing_target,expected_target",+ [+ ("incremental", ["id"], 4000, None, str(16 * 1024 * 1024)),+ ("incremental", ["id"], 1, None, None),+ ("append", None, 4000, None, None),+ ("full_refresh", None, 4000, None, None),+ ("incremental", ["id"], 4000, str(64 * 1024 * 1024), str(64 * 1024 * 1024)),+ ],+ ids=[+ "wide_merge_table_gets_a_smaller_target",+ "narrow_merge_table_keeps_the_default",+ "wide_append_table_keeps_the_default",+ "wide_full_refresh_table_keeps_the_default",+ "hand_set_target_is_kept",+ ],+ )+ @pytest.mark.asyncio+ async def test_target_after_writes(+ self,+ write_type: Literal["incremental", "append", "full_refresh"],+ primary_keys: list[str] | None,+ payload_chars: int,+ existing_target: str | None,+ expected_target: str | None,+ tmp_path: Path,+ ) -> None:+ delta_path = str(tmp_path / "table")+ deltalake.write_deltalake(+ delta_path,+ _rows(list(range(20)), payload_chars),+ configuration={"delta.targetFileSize": existing_target} if existing_target else None,+ )+ table_ref = make_local_table_ref(delta_path)++ # Local tables are far below the production size gates, so only those gates are lowered.+ with (+ patch(f"{_OPS_MODULE}.WIDE_ROW_MIN_TABLE_BYTES", 0),+ patch(f"{_OPS_MODULE}.WIDE_ROW_MIN_STORED_BYTES_PER_ROW", 1024),+ ):+ for ids in ([1, 2], [3, 4]):+ await DeltaWriter(table_ref).write(+ data=_rows(ids, payload_chars),+ write_type=write_type,+ should_overwrite_table=False,+ primary_keys=primary_keys,+ )++ configuration = deltalake.DeltaTable(delta_path).metadata().configuration+ assert configuration.get("delta.targetFileSize") == expected_target+ # The checkpoint property and the target land in one commit, and the second write adds none.+ assert _commit_operations(delta_path).count("SET TBLPROPERTIES") == 1++class TestCreateRaceWithExistingTable:"""DeltaTable.create() defaults to mode="error", raising "table already exists at thatlocation" whenever the destination is non-empty. get_delta_table() can report "no tablediff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/writer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/writer.pyindex cc64bd8d8169..2f3e42b4d94b 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/writer.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/writer.py@@ -758,7 +758,8 @@ class DeltaWriter:delta_table = await self._table.get_delta_table(allow_stale=True)assert delta_table is not None- await ensure_table_properties(delta_table, self._logger)+ merges = write_type == "incremental" and bool(primary_keys)+ await ensure_table_properties(delta_table, self._logger, merge_batch=data if merges else None)return delta_table