:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): size governor readers by their largest row group (#120990)
Tom Owersaa094e826810
diff --git a/posthog/settings/data_warehouse.py b/posthog/settings/data_warehouse.pyindex dbca08805e55..17b2e499ebea 100644--- a/posthog/settings/data_warehouse.py+++ b/posthog/settings/data_warehouse.py@@ -173,6 +173,12 @@ DATA_WAREHOUSE_DELTALITE_COMPACTION = get_from_env("DATA_WAREHOUSE_DELTALITE_COMDATA_WAREHOUSE_DELTALITE_MAX_ROW_GROUP_BYTES = get_from_env("DATA_WAREHOUSE_DELTALITE_MAX_ROW_GROUP_BYTES", 16 * 1024 * 1024, type_cast=int)+# Size each deltalite reader by the largest row group of its file instead of by the whole file. The+# writer reads the footers of the candidate files only when the whole-file sizes keep the governor's plan+# narrow. A footer that cannot be read keeps the whole-file size. False turns the footer reads off.+DATA_WAREHOUSE_DELTALITE_GOVERNOR_ROW_GROUP_SIZING = get_from_env(+ "DATA_WAREHOUSE_DELTALITE_GOVERNOR_ROW_GROUP_SIZING", True, type_cast=str_to_bool+)# A rebuild run (first sync, reset, full refresh) compacts its own files once a compaction would remove# this many, so a long rebuild does not leave one post-load pass that outlasts the stuck-batch timeout.DATA_WAREHOUSE_REBUILD_COMPACT_MIN_REMOVABLE_FILES = get_from_env(diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/memory_governor.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/memory_governor.pyindex ec529e18250c..13e32cce7810 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/memory_governor.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/memory_governor.py@@ -32,6 +32,9 @@ in deltalite 0.1.10 holds three things at its peak:* **Readers.** Each reader fetches one whole compressed row group before it decodes it. A readerreserves its row group from the per-call fetch budget (``max_fetch_bytes``) first, so all readersof one upsert together hold at most the budget, or one row group when a row group is larger.+ Add actions record file sizes, not row groups, so a file's size is its reader bound. When that+ bound keeps the plan narrow, the writer can read the footers (``row_group_sizes``) and use the+ real largest row group of each file instead.* **Writers.** Each partition worker streams its open file into fixed-size chunks, so it holds aboutone ``target_file_size`` file, and less when the partition writes less. Each column also keepspage and dictionary buffers, so wide tables cost more per worker than narrow ones.@@ -78,6 +81,11 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delif TYPE_CHECKING:import deltalake+ from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.row_group_sizes import (+ RowGroupSizeReader,+ RowGroupSizes,+ )+logger = logging.getLogger(__name__)MB = 1024 * 1024@@ -114,6 +122,9 @@ _READER_PAGE_BUFFER_SHARE = 0.4#: deltalite's per-call ``max_fetch_bytes``, passed explicitly. All readers of one upsert hold at#: most this much fetched row-group data, or one row group when a row group is larger._FETCH_BUDGET_BYTES = 128 * MB+#: Files at or below this size are not worth a footer read: their whole size is already a small+#: reader bound. It matches the default row-group cap of the files deltalite writes.+_ROW_GROUP_PROBE_MIN_BYTES = 16 * MB#: Resident bytes per byte of the file one partition worker writes, up to ``target_file_size``._WRITER_BYTES_PER_TARGET_BYTE = 1.8#: Page and dictionary buffers per column of one open output file, once the file is large enough@@ -199,9 +210,9 @@ def _rss_retention(environ: Mapping[str, str] = os.environ) -> float:class PartitionShape:"""The part of one touched partition that sizes its worker."""- #: Stored bytes of the largest files the merge can rewrite, largest first. Only as many as one- #: worker can read at once are kept; each bounds the row group a reader fetches.- largest_file_bytes: tuple[int, ...]+ #: Per candidate file, the largest row group a reader of it fetches, largest first. Only as many+ #: as one worker can read at once are kept. A file whose footer was not read counts its whole size.+ largest_row_group_bytes: tuple[int, ...]#: All candidate files in the partition, and their stored bytes.files: intstored_bytes: int@@ -212,14 +223,20 @@ class PartitionShape:decoded_bytes: int | None = None@staticmethod- def of(file_bytes: Sequence[int], source_bytes: int = 0, decoded_bytes: int | None = None) -> PartitionShape:- ordered = sorted(file_bytes, reverse=True)+ def of(+ file_bytes: Sequence[int],+ source_bytes: int = 0,+ decoded_bytes: int | None = None,+ row_group_bytes: Sequence[int] | None = None,+ ) -> PartitionShape:+ """``row_group_bytes`` holds the largest row group of each file in ``file_bytes``, in the same order."""+ readers = file_bytes if row_group_bytes is None else row_group_bytesreturn PartitionShape(- largest_file_bytes=tuple(ordered[:_MAX_PARALLEL_FILES]),- files=len(ordered),- stored_bytes=sum(ordered),+ largest_row_group_bytes=tuple(sorted(readers, reverse=True)[:_MAX_PARALLEL_FILES]),+ files=len(file_bytes),+ stored_bytes=sum(file_bytes),source_bytes=source_bytes,- decoded_bytes=0 if not ordered else decoded_bytes,+ decoded_bytes=0 if not file_bytes else decoded_bytes,)@@ -235,6 +252,11 @@ class RewriteProfile:target_file_size: int = _DEFAULT_TARGET_FILE_SIZE#: Columns of the rows the merge writes. Each column of an open output file keeps buffers.columns: int = _UNKNOWN_COLUMNS+ #: Candidate files whose reader bound is their largest row group, read from the footer. The+ #: other candidates count their whole size.+ row_group_sized_files: int = 0+ #: Footers fetched from storage to size this merge. Cached footers are not counted.+ footers_read: int = 0@propertydef files(self) -> int:@@ -246,15 +268,23 @@ class RewriteProfile:@propertydef max_row_group_mb(self) -> float:- """Upper bound on the largest row group a reader fetches: the largest candidate file.+ """Upper bound on the largest row group a reader fetches.- Add actions carry each file's size but not its row-group count, and the footer that has it- costs one request per file. A file holds at least one row group, so its size bounds its- largest one. The bound is exact for single-row-group files, which is what deltalite and- delta-rs write while a file stays under the 1M-row row-group limit. For files with several- row groups it over-counts (about 1.5× on 2–3 row-group files), so it errs toward a smaller plan.+ A file holds at least one row group, so its size bounds its largest one. The bound is exact+ for single-row-group files. For a file whose footer was read, the bound is its real largest+ row group, which is much smaller for files written with a row-group cap."""- return max((p.largest_file_bytes[0] for p in self.partitions if p.largest_file_bytes), default=0) / MB+ return max((p.largest_row_group_bytes[0] for p in self.partitions if p.largest_row_group_bytes), default=0) / MB+++@frozen+class CandidateFile:+ """One existing file that a merge can rewrite."""++ partition: str | None+ path: str | None+ size_bytes: int+ num_records: int | Nonedef _files_that_can_match(add_actions: pa.Table, source: pa.Table, primary_keys: Sequence[str]) -> list[bool]:@@ -291,19 +321,63 @@ def _files_that_can_match(add_actions: pa.Table, source: pa.Table, primary_keys:return [d is not True for d in disjoint.to_pylist()]+def _rows_per_partition(source: pa.Table, partition_col: str | None) -> dict[str | None, int]:+ if partition_col is None:+ return {None: source.num_rows}+ counts = pc.value_counts(source[partition_col].cast(pa.string()))+ return {+ value: int(count or 0)+ for value, count in zip(counts.field("values").to_pylist(), counts.field("counts").to_pylist())+ }+++def candidate_files(+ add_actions: pa.Table, source: pa.Table, partition_col: str | None, primary_keys: Sequence[str]+) -> list[CandidateFile]:+ """The existing files a merge of ``source`` can rewrite.++ A file is a candidate when it is in a partition the batch touches and its PK range overlaps the+ batch. Every candidate is counted because duplicate target keys can make one source row match+ several files.+ """+ if source.num_rows == 0 or add_actions.num_rows == 0:+ return []+ touched = _rows_per_partition(source, partition_col)+ file_partitions: list[str | None] = (+ [None] * add_actions.num_rows+ if partition_col is None+ else add_actions[f"partition.{partition_col}"].cast(pa.string()).to_pylist()+ )+ paths = add_actions["path"].to_pylist() if "path" in add_actions.column_names else [None] * add_actions.num_rows+ file_rows = (+ add_actions["num_records"].to_pylist()+ if "num_records" in add_actions.column_names+ else [None] * add_actions.num_rows+ )+ can_match = _files_that_can_match(add_actions, source, primary_keys)+ return [+ CandidateFile(partition=partition, path=path, size_bytes=size or 0, num_records=rows)+ for partition, path, size, rows, candidate in zip(+ file_partitions, paths, add_actions["size_bytes"].to_pylist(), file_rows, can_match+ )+ if candidate and partition in touched+ ]++def rewrite_profile(add_actions: pa.Table,source: pa.Table,partition_col: str | None,primary_keys: Sequence[str],target_file_size: int = _DEFAULT_TARGET_FILE_SIZE,+ row_groups: RowGroupSizes | None = None,+ candidates: Sequence[CandidateFile] | None = None,) -> RewriteProfile:"""Upper bound on the existing files a merge of ``source`` rewrites, from the table's add actions.- A file is a candidate when it is in a partition the batch touches and its PK range overlaps the- batch. Every candidate is counted because duplicate target keys can make one source row match- several files. Every touched partition gets an entry with its share of the source, also when it- has no candidate files.+ Every touched partition gets an entry with its share of the source, also when it has no+ candidate files. ``row_groups`` gives the largest row group of the files whose footer was read.+ Each other file counts its whole size. ``candidates`` reuses a ``candidate_files`` result."""if source.num_rows == 0:return RewriteProfile(@@ -313,35 +387,26 @@ def rewrite_profile(columns=source.num_columns,)- if partition_col is None:- rows_per_partition: dict[str | None, int] = {None: source.num_rows}- file_partitions: list[str | None] = [None] * add_actions.num_rows- else:- counts = pc.value_counts(source[partition_col].cast(pa.string()))- rows_per_partition = {- value: int(count or 0)- for value, count in zip(counts.field("values").to_pylist(), counts.field("counts").to_pylist())- }- file_partitions = (- add_actions[f"partition.{partition_col}"].cast(pa.string()).to_pylist() if add_actions.num_rows else []- )-+ rows_per_partition = _rows_per_partition(source, partition_col)+ known_row_groups: Mapping[str, int] = row_groups.bytes_by_path if row_groups is not None else {}sizes_by_partition: dict[str | None, list[int]] = {partition: [] for partition in rows_per_partition}+ readers_by_partition: dict[str | None, list[int]] = {partition: [] for partition in rows_per_partition}rows_by_partition: dict[str | None, int | None] = dict.fromkeys(rows_per_partition, 0)- if add_actions.num_rows:- can_match = _files_that_can_match(add_actions, source, primary_keys)- file_rows = (- add_actions["num_records"].to_pylist()- if "num_records" in add_actions.column_names- else [None] * add_actions.num_rows+ row_group_sized = 0+ if candidates is None:+ candidates = candidate_files(add_actions, source, partition_col, primary_keys)+ for file in candidates:+ sizes_by_partition[file.partition].append(file.size_bytes)+ largest_row_group = known_row_groups.get(file.path) if file.path is not None else None+ if largest_row_group is None:+ readers_by_partition[file.partition].append(file.size_bytes)+ else:+ row_group_sized += 1+ readers_by_partition[file.partition].append(min(largest_row_group, file.size_bytes))+ known = rows_by_partition[file.partition]+ rows_by_partition[file.partition] = (+ None if known is None or file.num_records is None else known + file.num_records)- for partition, size, rows, candidate in zip(- file_partitions, add_actions["size_bytes"].to_pylist(), file_rows, can_match- ):- if candidate and partition in sizes_by_partition:- sizes_by_partition[partition].append(size or 0)- known = rows_by_partition[partition]- rows_by_partition[partition] = None if known is None or rows is None else known + rows# The source has the table's columns, so its bytes per row also size the decoded candidates.bytes_per_row = source.nbytes / source.num_rows@@ -353,6 +418,7 @@ def rewrite_profile(sizes,source_bytes=round(bytes_per_row * rows_per_partition[partition]),decoded_bytes=None if candidate_rows is None else round(bytes_per_row * candidate_rows),+ row_group_bytes=readers_by_partition[partition],))return RewriteProfile(@@ -360,19 +426,52 @@ def rewrite_profile(table_files=add_actions.num_rows,target_file_size=target_file_size,columns=source.num_columns,+ row_group_sized_files=row_group_sized,+ footers_read=row_groups.footers_read if row_groups is not None else 0,)def estimate_rewrite_profile(- table: deltalake.DeltaTable, source: pa.Table, partition_col: str | None, primary_keys: Sequence[str]+ table: deltalake.DeltaTable,+ source: pa.Table,+ partition_col: str | None,+ primary_keys: Sequence[str],+ row_group_reader: RowGroupSizeReader | None = None,+ wants_row_groups: Callable[[RewriteProfile], bool] | None = None,) -> RewriteProfile | None:- """``rewrite_profile`` for a live table, or None when its add actions cannot be read."""+ """``rewrite_profile`` for a live table, or None when its add actions cannot be read.++ With a ``row_group_reader``, the footers of candidate files above ``_ROW_GROUP_PROBE_MIN_BYTES``+ are read when ``wants_row_groups`` says the whole-file profile limits the plan. Any failure+ there keeps the whole-file profile.+ """try:add_actions = pa.table(table.get_add_actions(flatten=True))- return rewrite_profile(add_actions, source, partition_col, primary_keys, table_target_file_size(table))+ target_file_size = table_target_file_size(table)+ candidates = candidate_files(add_actions, source, partition_col, primary_keys)+ profile = rewrite_profile(+ add_actions, source, partition_col, primary_keys, target_file_size, candidates=candidates+ )except Exception as e: # noqa: BLE001 - an estimate must never fail a write; None sizes conservativelylogger.debug("deltalite governor: could not estimate the rewrite size: %s", e)return None+ if row_group_reader is None or profile.max_row_group_mb * MB <= _ROW_GROUP_PROBE_MIN_BYTES:+ return profile+ try:+ if wants_row_groups is not None and not wants_row_groups(profile):+ return profile+ to_read = {+ file.path: file.size_bytes+ for file in candidates+ if file.path is not None and file.size_bytes > _ROW_GROUP_PROBE_MIN_BYTES+ }+ row_groups = row_group_reader.largest_row_groups(to_read)+ return rewrite_profile(+ add_actions, source, partition_col, primary_keys, target_file_size, row_groups, candidates+ )+ except Exception as e: # noqa: BLE001 - row-group sizing is an optimization; keep the whole-file bound+ logger.debug("deltalite governor: could not read row group sizes: %s", e)+ return profile@frozen@@ -423,7 +522,7 @@ def _writer_mb(written_mb: float, target_mb: float, columns: int) -> float:def _worker_terms(shape: PartitionShape, files_per_worker: int, target_mb: float, columns: int) -> WorkerTerms:- row_groups_mb = [size / MB for size in shape.largest_file_bytes[:files_per_worker]]+ row_groups_mb = [size / MB for size in shape.largest_row_group_bytes[:files_per_worker]]written = (shape.stored_bytes + shape.source_bytes) / MBreturn WorkerTerms(held_mb=sum(row_groups_mb),@@ -552,6 +651,22 @@ def _reader_options(mpp: int) -> list[int]:return sorted({f for f in (widest, _DEFAULT_FILES_PER_WORKER, 2, 1) if f <= widest}, reverse=True)+def _partition_cap(profile: RewriteProfile, n_partitions: int | None) -> int:+ cap = min(_MAX_PARALLEL_PARTITIONS, max(1, len(profile.partitions)))+ if n_partitions is not None and n_partitions >= 1:+ cap = min(cap, n_partitions)+ return cap+++def is_widest_plan(plan: UpsertPlan, profile: RewriteProfile, n_partitions: int | None = None) -> bool:+ """True when ``plan`` fits and already runs the most workers and readers ``size_upsert`` ever picks."""+ partition_cap = _partition_cap(profile, n_partitions)+ return plan.fits and (plan.max_parallel_partitions, plan.max_parallel_files) == (+ partition_cap,+ _reader_options(partition_cap)[0],+ )++def size_upsert(available_mb: float,source_mb: float,@@ -584,10 +699,7 @@ def size_upsert(columns=profile.columns,)- partition_cap = min(_MAX_PARALLEL_PARTITIONS, max(1, len(profile.partitions)))- if n_partitions is not None and n_partitions >= 1:- partition_cap = min(partition_cap, n_partitions)-+ partition_cap = _partition_cap(profile, n_partitions)# One pass over the partitions per reader count, however many plans are tried.costliest: dict[int, WorkerBounds] = {}for mpp in range(partition_cap, 0, -1):@@ -792,8 +904,11 @@ class Admission:capacity_exceeded: bool = False#: The predicted peak by term, in-use and RSS.estimate: MemoryEstimate | None = None- #: Largest candidate file, the bound on the largest row group a reader fetches. None when unknown.+ #: The bound on the largest row group a reader fetches. None when unknown.max_row_group_mb: float | None = None+ #: Candidate files sized by their largest row group, and footers fetched for that. None when unknown.+ row_group_sized_files: int | None = None+ footers_read: int | None = None#: Stored MB of all candidate files the merge can rewrite, over every touched partition.rewrite_total_mb: float | None = Nonerewrite_files: int | None = None@@ -874,6 +989,22 @@ class MemoryGovernor:limit_mb = self.pod.limit_mb()return None if limit_mb is None else self._per_upsert_budget_mb(limit_mb)+ def row_groups_can_widen(+ self, profile: RewriteProfile, *, source_bytes: int, n_partitions: int | None = None+ ) -> bool:+ """True when the plan for ``profile`` is narrower than the widest plan.++ Only then can smaller reader bounds change the plan, so only then are footer reads worth+ their requests. False when this governor sizes no plan (mode off, or no pod limit).+ """+ limit_mb = self.pod.limit_mb()+ if self.config.mode == "off" or limit_mb is None:+ return False+ plan = size_upsert(+ self._per_upsert_budget_mb(limit_mb), source_bytes / MB, n_partitions, profile, self.config.rss_retention+ )+ return not is_widest_plan(plan, profile, n_partitions)+def _has_room(self, reserve_mb: float, usable_mb: float) -> bool:return self._reserved_mb + reserve_mb <= usable_mb + _ROOM_TOLERANCE_MB@@ -974,6 +1105,8 @@ class MemoryGovernor:capacity_exceeded=plan is not None and not plan.fits,estimate=plan.estimate if plan is not None else None,max_row_group_mb=round(rewrite.max_row_group_mb, 1) if rewrite is not None else None,+ row_group_sized_files=rewrite.row_group_sized_files if rewrite is not None else None,+ footers_read=rewrite.footers_read if rewrite is not None else None,rewrite_total_mb=round(rewrite.total_mb, 1) if rewrite is not None else None,rewrite_files=rewrite.files if rewrite is not None else None,columns=rewrite.columns if rewrite is not None else None,diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/row_group_sizes.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/row_group_sizes.pynew file mode 100644index 000000000000..a9c6b7152aaf--- /dev/null+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/row_group_sizes.py@@ -0,0 +1,204 @@+"""The largest row group of each Parquet file a merge can rewrite, read from the file footers.++A deltalite reader reserves fetch budget for the largest compressed row group of a file, and then+holds one row group at a time. Delta add actions record the size of each file but not its row+groups, so the memory governor otherwise sizes each reader by the whole file. This module reads the+footers so that the governor can use the real row-group size.+"""++from __future__ import annotations++import logging+import threading+import urllib.parse+from collections import OrderedDict+from collections.abc import Callable, Mapping+from concurrent.futures import Future, ThreadPoolExecutor, wait++import pyarrow.fs as pafs+import pyarrow.parquet as pq+from deltalake.fs import DeltaStorageHandler++from posthog.dataclasses import frozen++logger = logging.getLogger(__name__)++#: Most footers one merge reads. Files past this keep their whole size as the bound.+_MAX_READS = 1024+#: Footers read at the same time. Each read is one ranged GET of the file tail.+_READ_CONCURRENCY = 32+#: Longest the footer reads of one merge may take. Files not read in time keep their whole size.+_READ_TIMEOUT_S = 10.0+#: Footer reads queued or running across all merges. A merge past this reads no footers.+_MAX_PENDING_READS = 2 * _MAX_READS+#: Files are immutable, so a footer read once stays correct. About 100 bytes per entry.+_CACHE_ENTRIES = 100_000+++def largest_row_group_bytes(metadata: pq.FileMetaData) -> int:+ """Compressed bytes of the largest row group, counted as deltalite counts its fetch reservation."""+ largest = 0+ for i in range(metadata.num_row_groups):+ row_group = metadata.row_group(i)+ largest = max(largest, sum(row_group.column(c).total_compressed_size for c in range(row_group.num_columns)))+ return largest+++class _ReadSlots:+ """Bounds the footer reads queued or running on the shared executor, over all merges."""++ def __init__(self, limit: int) -> None:+ self._limit = limit+ self._used = 0+ self._lock = threading.Lock()++ def acquire(self, wanted: int) -> int:+ with self._lock:+ granted = max(0, min(wanted, self._limit - self._used))+ self._used += granted+ return granted++ def release(self, count: int = 1) -> None:+ with self._lock:+ self._used -= count+++def _read_footer(path: str, filesystem: pafs.FileSystem, expired: threading.Event, slots: _ReadSlots) -> int | None:+ # A read that its merge stopped waiting for still holds a slot until a worker takes it, so+ # stalled storage stops new reads instead of growing the queue.+ try:+ if expired.is_set():+ return None+ return largest_row_group_bytes(pq.read_metadata(path, filesystem=filesystem))+ finally:+ slots.release()+++@frozen+class RowGroupSizes:+ """Largest row group per file path, for the files whose footer could be read."""++ bytes_by_path: Mapping[str, int]+ #: Footers fetched from storage for this call. Cache hits are not counted.+ footers_read: int = 0+++class _FooterCache:+ """Process-wide LRU of the largest row group per (table, file)."""++ def __init__(self, max_entries: int) -> None:+ self._max_entries = max_entries+ self._entries: OrderedDict[tuple[str, str], int] = OrderedDict()+ self._lock = threading.Lock()++ def get(self, key: tuple[str, str]) -> int | None:+ with self._lock:+ value = self._entries.get(key)+ if value is not None:+ self._entries.move_to_end(key)+ return value++ def put(self, key: tuple[str, str], value: int) -> None:+ with self._lock:+ self._entries[key] = value+ self._entries.move_to_end(key)+ while len(self._entries) > self._max_entries:+ self._entries.popitem(last=False)++ def clear(self) -> None:+ with self._lock:+ self._entries.clear()+++_CACHE = _FooterCache(_CACHE_ENTRIES)++#: Shared by all merges, so reads that outlive their deadline cannot add threads past this bound.+_EXECUTOR = ThreadPoolExecutor(max_workers=_READ_CONCURRENCY, thread_name_prefix="row-group-footers")+_SLOTS = _ReadSlots(_MAX_PENDING_READS)++#: Opens a filesystem for a table that knows the sizes of the given files, so no read needs a HEAD.+OpenFilesystem = Callable[[Mapping[str, int]], pafs.FileSystem]+++class RowGroupSizeReader:+ """Reads the largest row group of a table's files from their footers.++ A file whose footer cannot be read in time, or at all, is left out of the result, and the+ caller keeps its whole size as the bound. So a failure can only make the plan smaller.+ """++ def __init__(+ self,+ table_uri: str,+ storage_options: Mapping[str, str] | None = None,+ *,+ open_filesystem: OpenFilesystem | None = None,+ max_reads: int = _MAX_READS,+ executor: ThreadPoolExecutor = _EXECUTOR,+ slots: _ReadSlots = _SLOTS,+ timeout_s: float = _READ_TIMEOUT_S,+ cache: _FooterCache = _CACHE,+ ) -> None:+ self._table_uri = table_uri+ self._storage_options = dict(storage_options or {})+ self._open_filesystem = open_filesystem or self._delta_filesystem+ self._max_reads = max_reads+ self._executor = executor+ self._slots = slots+ self._timeout_s = timeout_s+ self._cache = cache++ def _delta_filesystem(self, known_sizes: Mapping[str, int]) -> pafs.FileSystem:+ handler = DeltaStorageHandler(self._table_uri, self._storage_options, dict(known_sizes))+ return pafs.PyFileSystem(handler)++ def largest_row_groups(self, sizes_by_path: Mapping[str, int]) -> RowGroupSizes:+ """The largest row group of each file in ``sizes_by_path``, largest files first."""+ found: dict[str, int] = {}+ to_read: dict[str, int] = {}+ for path, size in sorted(sizes_by_path.items(), key=lambda item: item[1], reverse=True):+ cached = self._cache.get((self._table_uri, path))+ if cached is not None:+ found[path] = cached+ elif len(to_read) < self._max_reads:+ to_read[path] = size+ granted = self._slots.acquire(len(to_read))+ to_read = dict(list(to_read.items())[:granted])+ if not to_read:+ return RowGroupSizes(bytes_by_path=found)++ # Add action paths are URI-encoded; the storage handler takes decoded paths.+ storage_paths = {path: urllib.parse.unquote(path) for path in to_read}+ expired = threading.Event()+ futures: dict[Future[int | None], str] = {}+ try:+ filesystem = self._open_filesystem({storage_paths[path]: size for path, size in to_read.items()})+ for path in to_read:+ future = self._executor.submit(_read_footer, storage_paths[path], filesystem, expired, self._slots)+ futures[future] = path+ except BaseException:+ # Each submitted read frees its own slot; the rest were never handed to a worker.+ self._slots.release(len(to_read) - len(futures))+ raise+ done, not_done = wait(futures, timeout=self._timeout_s)+ expired.set()++ failed = 0+ for future in done:+ path = futures[future]+ if future.exception() is not None:+ failed += 1+ continue+ result = future.result()+ if result is None:+ continue+ largest = min(result, to_read[path])+ self._cache.put((self._table_uri, path), largest)+ found[path] = largest+ if failed or not_done:+ logger.debug(+ "row group sizes: %d footer reads failed and %d timed out; those files keep their whole size",+ failed,+ len(not_done),+ )+ return RowGroupSizes(bytes_by_path=found, footers_read=len(done) - failed)diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_memory_governor.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_memory_governor.pyindex 9154a016b39a..6d29068ad0c5 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_memory_governor.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_memory_governor.py@@ -8,6 +8,8 @@ from django.test import override_settingsimport pyarrow as paimport deltalake+import pyarrow.fs as pafs+import pyarrow.parquet as pqfrom parameterized import parameterizedfrom products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta import memory_governor as _mg@@ -25,6 +27,11 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delrewrite_profile,size_upsert,)+from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.row_group_sizes import (+ RowGroupSizeReader,+ RowGroupSizes,+ _FooterCache,+)from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.rss_sampler import RssPeakSampler@@ -78,14 +85,18 @@ def _profile(columns: int = 64,decoded_mb: float | None = None,source_mb_per_partition: float = 0.0,+ row_group_mb: float | None = None,) -> RewriteProfile:- """A profile from per-partition candidate file sizes in MB."""+ """A profile from per-partition candidate file sizes in MB, each file of row groups up to ``row_group_mb``."""return RewriteProfile(partitions=tuple(PartitionShape.of([round(size * MB) for size in sizes],source_bytes=round(source_mb_per_partition * MB),decoded_bytes=None if decoded_mb is None else round(decoded_mb * MB),+ row_group_bytes=None+ if row_group_mb is None+ else [round(min(size, row_group_mb) * MB) for size in sizes],)for sizes in partitions),@@ -332,6 +343,31 @@ class TestSizeUpsert:8,True,),+ # The same default-size files written as 16 MB row groups: each reader holds 16 MB, not 100.+ (+ "wide_rows_in_16mb_row_groups",+ _profile(*([[100.0] * 2] * 4), columns=30, row_group_mb=16.0),+ None,+ 1.4,+ _SLOT_14GI,+ 1,+ 8,+ True,+ ),+ ("single_row_group_files_take_one_worker", _profile(*([[130.0] * 2] * 4)), None, 2.0, _SLOT, 1, 8, True),+ (+ "small_row_groups_take_two_workers",+ _profile(*([[130.0] * 2] * 4), row_group_mb=16.0),+ None,+ 2.0,+ _SLOT,+ 2,+ 8,+ True,+ ),+ # Three 130 MB single-row-group files overflow a 900 MB slice; in 16 MB row groups they fit 1 x 8.+ ("single_row_group_files_overflow", _profile([130.0] * 3), None, 2.0, 900.0, 1, 1, False),+ ("small_row_groups_fit", _profile([130.0] * 3, row_group_mb=16.0), None, 2.0, 900.0, 1, 8, True),])def test_sizing(self, _name, profile, n_partitions, retention, slot, exp_mpp, exp_files, exp_fits):@@ -514,6 +550,19 @@ class TestGovernorSizing:passassert (adm.max_row_group_mb, adm.rewrite_files, adm.rewrite_total_mb, adm.columns) == (57.0, 24, 1368.0, 64)+ @parameterized.expand(+ [+ ("narrow_plan_can_widen", "enforce", 15_000.0, _profile(*([[130.0] * 2] * 4)), True),+ ("advisory_sizes_a_plan_too", "advisory", 15_000.0, _profile(*([[130.0] * 2] * 4)), True),+ ("widest_plan_cannot_widen", "enforce", 30_000.0, _SMALL, False),+ ("off_sizes_no_plan", "off", 15_000.0, _profile(*([[130.0] * 2] * 4)), False),+ ("no_pod_limit_sizes_no_plan", "enforce", None, _profile(*([[130.0] * 2] * 4)), False),+ ]+ )+ def test_row_groups_can_widen_only_a_narrow_plan(self, _name, mode, limit_mb, profile, expected):+ gov = _governor(mode, limit_mb=limit_mb)+ assert gov.row_groups_can_widen(profile, source_bytes=10 * MB) is expected+async def test_reservation_released_on_exception(self):sampler = _fake_sampler(1_000.0, 1_000.0)gov = _governor("enforce", rss_sampler=sampler)@@ -680,7 +729,7 @@ class TestRewriteProfile:)def test_profile(self, _name, source, partition_col, expected_files, expected_count):profile = rewrite_profile(_add_actions(), source, partition_col, ["id"])- assert tuple(p.largest_file_bytes for p in profile.partitions) == expected_files+ assert tuple(p.largest_row_group_bytes for p in profile.partitions) == expected_filesassert profile.files == expected_countassert profile.table_files == len(_FILES)# Each partition carries its share of the source, for the bytes its worker writes.@@ -698,7 +747,7 @@ class TestRewriteProfile:source = _source([("a", 5000), ("a", 5001), ("a", 5002)])source = source.append_column("other", pa.array([1, 2, 3], pa.int64()))profile = rewrite_profile(add_actions, source, _PART, primary_keys)- assert [(p.largest_file_bytes, p.stored_bytes) for p in profile.partitions] == [((100, 50, 30), 180)]+ assert [(p.largest_row_group_bytes, p.stored_bytes) for p in profile.partitions] == [((100, 50, 30), 180)]@parameterized.expand([@@ -720,7 +769,7 @@ class TestRewriteProfile:def test_one_source_row_can_match_duplicate_keys_in_multiple_files(self):files = [("a", 100, 100, 200), ("a", 50, 150, 250)]profile = rewrite_profile(_add_actions(files), _source([("a", 175)]), _PART, ["id"])- assert (profile.partitions[0].largest_file_bytes, profile.files) == ((100, 50), 2)+ assert (profile.partitions[0].largest_row_group_bytes, profile.files) == ((100, 50), 2)def test_string_keys_are_not_pruned(self):# Delta truncates long string stats, so a string max proves nothing.@@ -736,7 +785,7 @@ class TestRewriteProfile:files = [("a", size, 0, 99) for size in range(1, 21)]source = _source([("a", i) for i in range(20)])shape = rewrite_profile(_add_actions(files), source, _PART, ["id"]).partitions[0]- assert shape.largest_file_bytes == (20, 19, 18, 17, 16, 15, 14, 13)+ assert shape.largest_row_group_bytes == (20, 19, 18, 17, 16, 15, 14, 13)assert (shape.files, shape.stored_bytes) == (20, sum(range(1, 21)))@parameterized.expand(@@ -747,7 +796,7 @@ class TestRewriteProfile:)def test_nothing_to_rewrite(self, _name, files, source, expected_files, table_files):profile = rewrite_profile(_add_actions(files), source, _PART, ["id"])- assert tuple(p.largest_file_bytes for p in profile.partitions) == expected_files+ assert tuple(p.largest_row_group_bytes for p in profile.partitions) == expected_filesassert (profile.files, profile.total_mb, profile.max_row_group_mb, profile.table_files) == (0,0.0,@@ -781,6 +830,87 @@ class TestRewriteProfile:assert (profile.files, profile.table_files, profile.target_file_size) == (2, 3, expected_target)assert profile.max_row_group_mb == max(sizes[f"{_PART}=a"], sizes[f"{_PART}=b"]) / MB+ @parameterized.expand(+ [+ # a/f0 holds row groups of 10 bytes at most; the other files keep their size.+ ("footer_bounds_the_reader", {f"{_PART}=a/f0.parquet": 10}, (50, 30, 10), 1),+ # A footer cannot report more than the file holds; the file size still bounds it.+ ("never_above_the_file_size", {f"{_PART}=a/f0.parquet": 500}, (100, 50, 30), 1),+ ("unread_files_keep_their_size", {"elsewhere.parquet": 1}, (100, 50, 30), 0),+ ]+ )+ def test_row_groups_bound_the_readers(self, _name, bytes_by_path, expected, sized):+ row_groups = RowGroupSizes(bytes_by_path=bytes_by_path, footers_read=2)+ source = _source([("a", 5), ("a", 150), ("a", 250)])+ profile = rewrite_profile(_add_actions(), source, _PART, ["id"], row_groups=row_groups)+ shape = profile.partitions[0]+ assert shape.largest_row_group_bytes == expected+ assert (shape.files, shape.stored_bytes) == (3, 180)+ assert (profile.row_group_sized_files, profile.footers_read) == (sized, 2)++ @pytest.mark.parametrize(+ "reader,wants,sized",+ [+ ("local", None, True),+ # The setting off passes no reader: today's whole-file sizing.+ (None, None, False),+ # A plan that is already the widest gains nothing from footers, so none are read.+ ("local", False, False),+ ("broken", None, False),+ ],+ ids=["reads_the_footers", "no_reader", "widest_plan_reads_nothing", "unreadable_footers"],+ )+ def test_estimate_sizes_readers_by_their_row_groups(self, reader, wants, sized, tmp_path: Path):+ existing = pa.table(+ {+ _PART: ["a"] * 2000 + ["b"] * 2000,+ "id": pa.array(range(4000), pa.int64()),+ "v": [f"value-{i:06d}" * 8 for i in range(4000)],+ }+ )+ deltalake.write_deltalake(+ str(tmp_path),+ existing,+ partition_by=_PART,+ writer_properties=deltalake.WriterProperties(max_row_group_size=250, compression="UNCOMPRESSED"),+ )+ table = deltalake.DeltaTable(str(tmp_path))+ sizes = table._table.get_add_file_sizes()+ footers = [pq.read_metadata(tmp_path / path) for path in sizes]+ largest_row_group = max(+ sum(m.row_group(i).column(c).total_compressed_size for c in range(m.num_columns))+ for m in footers+ for i in range(m.num_row_groups)+ )++ def _broken(_sizes) -> pafs.FileSystem:+ raise OSError("unreachable")++ readers = {+ "local": RowGroupSizeReader(str(tmp_path), cache=_FooterCache(10)),+ "broken": RowGroupSizeReader(str(tmp_path), open_filesystem=_broken, cache=_FooterCache(10)),+ None: None,+ }+ with patch.object(_mg, "_ROW_GROUP_PROBE_MIN_BYTES", 0):+ profile = estimate_rewrite_profile(+ table,+ _source([("a", 5), ("b", 2500)]),+ _PART,+ ["id"],+ readers[reader],+ None if wants is None else lambda _profile: wants,+ )++ assert profile is not None+ assert (profile.files, profile.total_mb) == (2, sum(sizes.values()) / MB)+ if sized:+ assert largest_row_group < max(sizes.values()) / 4+ assert profile.max_row_group_mb == largest_row_group / MB+ assert (profile.row_group_sized_files, profile.footers_read) == (2, 2)+ else:+ assert profile.max_row_group_mb == max(sizes.values()) / MB+ assert (profile.row_group_sized_files, profile.footers_read) == (0, 0)+def test_estimate_unreadable_table_is_unknown(self):broken = MagicMock()broken.get_add_actions.side_effect = RuntimeError("no snapshot")diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_row_group_sizes.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_row_group_sizes.pynew file mode 100644index 000000000000..088b940b91a6--- /dev/null+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_row_group_sizes.py@@ -0,0 +1,149 @@+import threading+from pathlib import Path++import pytest+from unittest.mock import patch++import pyarrow as pa+import pyarrow.fs as pafs+import pyarrow.parquet as pq++from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta import row_group_sizes+from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.row_group_sizes import (+ RowGroupSizeReader,+ _FooterCache,+ _ReadSlots,+)+++def _write(path: Path, rows: int, row_group_rows: int) -> int:+ table = pa.table({"id": pa.array(range(rows), pa.int64()), "v": [f"value-{i:06d}" * 4 for i in range(rows)]})+ pq.write_table(table, path, row_group_size=row_group_rows, compression="none")+ return path.stat().st_size+++def _expected(path: str) -> int:+ metadata = pq.read_metadata(path)+ return max(+ sum(metadata.row_group(i).column(c).total_compressed_size for c in range(metadata.num_columns))+ for i in range(metadata.num_row_groups)+ )+++_READ_METADATA = pq.read_metadata+++class _Footers:+ """Counts footer reads, and fails or stalls the reads of chosen files."""++ def __init__(self, broken: frozenset[str] = frozenset(), stalled: frozenset[str] = frozenset()) -> None:+ self.read: list[str] = []+ self._broken = broken+ self._stalled = stalled+ self.release = threading.Event()+ self._lock = threading.Lock()++ def __call__(self, path: str, filesystem: pafs.FileSystem) -> pq.FileMetaData:+ with self._lock:+ self.read.append(path)+ if path in self._broken:+ raise OSError("unreachable")+ if path in self._stalled:+ self.release.wait(5)+ return _READ_METADATA(path, filesystem=filesystem)+++def files(tmp_path: Path) -> dict[str, int]:+ # Many small row groups, one row group, and a small file.+ return {+ str(tmp_path / name): _write(tmp_path / name, rows, row_group_rows)+ for name, rows, row_group_rows in (("many", 4000, 500), ("one", 3000, 3000), ("small", 100, 100))+ }+++def _read(sizes: dict[str, int], footers: _Footers, cache: _FooterCache | None = None, **kwargs):+ reader = RowGroupSizeReader(+ "table", open_filesystem=lambda _sizes: pafs.LocalFileSystem(), cache=cache or _FooterCache(100), **kwargs+ )+ with patch.object(row_group_sizes.pq, "read_metadata", side_effect=footers):+ return reader.largest_row_groups(sizes)+++class TestRowGroupSizeReader:+ def test_reads_the_largest_row_group_of_each_file(self, files: dict[str, int]) -> None:+ result = _read(files, _Footers())++ assert result.bytes_by_path == {path: _expected(path) for path in files}+ assert result.footers_read == 3+ many, one = (next(p for p in files if p.endswith(name)) for name in ("many", "one"))+ assert result.bytes_by_path[many] < files[many] / 4+ assert result.bytes_by_path[one] > files[one] / 2++ def test_a_cached_footer_is_not_read_again(self, files: dict[str, int]) -> None:+ cache = _FooterCache(100)+ footers = _Footers()+ first = _read(files, footers, cache)+ second = _read(files, footers, cache)++ assert second.bytes_by_path == first.bytes_by_path+ assert (second.footers_read, len(footers.read)) == (0, 3)++ def test_the_read_cap_keeps_the_largest_files(self, files: dict[str, int]) -> None:+ footers = _Footers()+ result = _read(files, footers, max_reads=2)++ largest_two = sorted(sorted(files, key=files.__getitem__, reverse=True)[:2])+ assert sorted(result.bytes_by_path) == largest_two+ assert sorted(footers.read) == largest_two++ @pytest.mark.parametrize("failure", ["broken", "stalled"])+ def test_a_file_that_cannot_be_read_is_left_out(self, failure: str, files: dict[str, int]) -> None:+ bad = next(p for p in files if p.endswith("many"))+ footers = _Footers(**{failure: frozenset({bad})})+ try:+ result = _read(files, footers, timeout_s=0.5)+ finally:+ footers.release.set()++ assert set(result.bytes_by_path) == set(files) - {bad}+ assert result.footers_read == 2++ def test_no_files_reads_nothing(self) -> None:+ footers = _Footers()+ assert _read({}, footers).bytes_by_path == {}+ assert footers.read == []++ def test_reads_a_uri_encoded_path_by_its_decoded_name(self, tmp_path: Path) -> None:+ (tmp_path / "p=a b").mkdir()+ size = _write(tmp_path / "p=a b" / "file.parquet", 2000, 500)+ encoded = str(tmp_path / "p=a%20b" / "file.parquet")++ result = _read({encoded: size}, _Footers())++ assert result.bytes_by_path == {encoded: _expected(str(tmp_path / "p=a b" / "file.parquet"))}++ def test_stalled_reads_hold_their_slots_so_later_calls_read_nothing(self, files: dict[str, int]) -> None:+ many = next(p for p in files if p.endswith("many"))+ slots = _ReadSlots(1)+ footers = _Footers(stalled=frozenset({many}))+ try:+ stalled = _read(files, footers, slots=slots, timeout_s=0.2)+ later = _read(files, footers, slots=slots)+ finally:+ footers.release.set()++ assert (stalled.bytes_by_path, later.bytes_by_path) == ({}, {})+ assert footers.read == [many]++ def test_a_filesystem_that_cannot_open_frees_its_slots(self, files: dict[str, int]) -> None:+ slots = _ReadSlots(len(files))++ def broken(_sizes: object) -> pafs.FileSystem:+ raise OSError("no credentials")++ reader = RowGroupSizeReader("table", open_filesystem=broken, cache=_FooterCache(100), slots=slots)+ with pytest.raises(OSError):+ reader.largest_row_groups(files)++ assert slots.acquire(len(files)) == len(files)diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/writer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/writer.pyindex 38fa23eba368..d484bd0092c3 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/writer.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/writer.py@@ -1,6 +1,7 @@import jsonimport timeimport asyncio+import functoolsimport contextlibfrom collections.abc import Callable, Sequencefrom typing import TYPE_CHECKING, Any, Literal@@ -303,6 +304,9 @@ class DeltaWriter:estimate_rewrite_profile,get_governor,)+ from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.row_group_sizes import (+ RowGroupSizeReader,+ )from products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.load.metrics import (DELTALITE_WRITE_DURATION_SECONDS,DELTALITE_WRITE_TOTAL,@@ -325,12 +329,25 @@ class DeltaWriter:# always writes — the governor never falls back to the delta-rs MERGE for capacity, because# the MERGE is the *more* memory-hungry path. A source too big for its slice just runs with# the smallest plan (governor logs a capacity_exceeded ops signal). The shape of the files- # the merge can rewrite comes from the handle's add actions, so this costs no object-store- # request.+ # the merge can rewrite comes from the handle's add actions. Footers of large candidate+ # files are read only when their whole sizes keep the plan narrow.governor = get_governor()+ row_group_reader = (+ RowGroupSizeReader(uri, storage_options)+ if settings.DATA_WAREHOUSE_DELTALITE_GOVERNOR_ROW_GROUP_SIZING+ else None+ )rewrite = (await asyncio.to_thread(- estimate_rewrite_profile, existing_delta_table, data, partition_key, normalized_primary_keys+ estimate_rewrite_profile,+ existing_delta_table,+ data,+ partition_key,+ normalized_primary_keys,+ row_group_reader,+ functools.partial(+ governor.row_groups_can_widen, source_bytes=data.nbytes, n_partitions=n_partitions+ ),)if governor.config.mode != "off"else None@@ -415,6 +432,8 @@ class DeltaWriter:governor_mpp=adm.planned_mpp,governor_mpf=adm.planned_mpf,governor_max_row_group_mb=adm.max_row_group_mb,+ governor_row_group_sized_files=adm.row_group_sized_files,+ governor_footers_read=adm.footers_read,governor_rewrite_total_mb=adm.rewrite_total_mb,governor_rewrite_files=adm.rewrite_files,governor_columns=adm.columns,