Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): size governor readers by their largest row group (#120990)

Tom Owersaa094e826810
diff --git a/posthog/settings/data_warehouse.py b/posthog/settings/data_warehouse.py
index dbca08805e55..17b2e499ebea 100644
--- a/posthog/settings/data_warehouse.py
+++ b/posthog/settings/data_warehouse.py
@@ -173,6 +173,12 @@ DATA_WAREHOUSE_DELTALITE_COMPACTION = get_from_env("DATA_WAREHOUSE_DELTALITE_COM
DATA_WAREHOUSE_DELTALITE_MAX_ROW_GROUP_BYTES = get_from_env(
"DATA_WAREHOUSE_DELTALITE_MAX_ROW_GROUP_BYTES", 16 * 1024 * 1024, type_cast=int
)
+# Size each deltalite reader by the largest row group of its file instead of by the whole file. The
+# writer reads the footers of the candidate files only when the whole-file sizes keep the governor's plan
+# narrow. A footer that cannot be read keeps the whole-file size. False turns the footer reads off.
+DATA_WAREHOUSE_DELTALITE_GOVERNOR_ROW_GROUP_SIZING = get_from_env(
+ "DATA_WAREHOUSE_DELTALITE_GOVERNOR_ROW_GROUP_SIZING", True, type_cast=str_to_bool
+)
# A rebuild run (first sync, reset, full refresh) compacts its own files once a compaction would remove
# this many, so a long rebuild does not leave one post-load pass that outlasts the stuck-batch timeout.
DATA_WAREHOUSE_REBUILD_COMPACT_MIN_REMOVABLE_FILES = get_from_env(
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/memory_governor.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/memory_governor.py
index ec529e18250c..13e32cce7810 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/memory_governor.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/memory_governor.py
@@ -32,6 +32,9 @@ in deltalite 0.1.10 holds three things at its peak:
* **Readers.** Each reader fetches one whole compressed row group before it decodes it. A reader
reserves its row group from the per-call fetch budget (``max_fetch_bytes``) first, so all readers
of one upsert together hold at most the budget, or one row group when a row group is larger.
+ Add actions record file sizes, not row groups, so a file's size is its reader bound. When that
+ bound keeps the plan narrow, the writer can read the footers (``row_group_sizes``) and use the
+ real largest row group of each file instead.
* **Writers.** Each partition worker streams its open file into fixed-size chunks, so it holds about
one ``target_file_size`` file, and less when the partition writes less. Each column also keeps
page and dictionary buffers, so wide tables cost more per worker than narrow ones.
@@ -78,6 +81,11 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.del
if TYPE_CHECKING:
import deltalake
+ from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.row_group_sizes import (
+ RowGroupSizeReader,
+ RowGroupSizes,
+ )
+
logger = logging.getLogger(__name__)
MB = 1024 * 1024
@@ -114,6 +122,9 @@ _READER_PAGE_BUFFER_SHARE = 0.4
#: deltalite's per-call ``max_fetch_bytes``, passed explicitly. All readers of one upsert hold at
#: most this much fetched row-group data, or one row group when a row group is larger.
_FETCH_BUDGET_BYTES = 128 * MB
+#: Files at or below this size are not worth a footer read: their whole size is already a small
+#: reader bound. It matches the default row-group cap of the files deltalite writes.
+_ROW_GROUP_PROBE_MIN_BYTES = 16 * MB
#: Resident bytes per byte of the file one partition worker writes, up to ``target_file_size``.
_WRITER_BYTES_PER_TARGET_BYTE = 1.8
#: Page and dictionary buffers per column of one open output file, once the file is large enough
@@ -199,9 +210,9 @@ def _rss_retention(environ: Mapping[str, str] = os.environ) -> float:
class PartitionShape:
"""The part of one touched partition that sizes its worker."""
- #: Stored bytes of the largest files the merge can rewrite, largest first. Only as many as one
- #: worker can read at once are kept; each bounds the row group a reader fetches.
- largest_file_bytes: tuple[int, ...]
+ #: Per candidate file, the largest row group a reader of it fetches, largest first. Only as many
+ #: as one worker can read at once are kept. A file whose footer was not read counts its whole size.
+ largest_row_group_bytes: tuple[int, ...]
#: All candidate files in the partition, and their stored bytes.
files: int
stored_bytes: int
@@ -212,14 +223,20 @@ class PartitionShape:
decoded_bytes: int | None = None
@staticmethod
- def of(file_bytes: Sequence[int], source_bytes: int = 0, decoded_bytes: int | None = None) -> PartitionShape:
- ordered = sorted(file_bytes, reverse=True)
+ def of(
+ file_bytes: Sequence[int],
+ source_bytes: int = 0,
+ decoded_bytes: int | None = None,
+ row_group_bytes: Sequence[int] | None = None,
+ ) -> PartitionShape:
+ """``row_group_bytes`` holds the largest row group of each file in ``file_bytes``, in the same order."""
+ readers = file_bytes if row_group_bytes is None else row_group_bytes
return PartitionShape(
- largest_file_bytes=tuple(ordered[:_MAX_PARALLEL_FILES]),
- files=len(ordered),
- stored_bytes=sum(ordered),
+ largest_row_group_bytes=tuple(sorted(readers, reverse=True)[:_MAX_PARALLEL_FILES]),
+ files=len(file_bytes),
+ stored_bytes=sum(file_bytes),
source_bytes=source_bytes,
- decoded_bytes=0 if not ordered else decoded_bytes,
+ decoded_bytes=0 if not file_bytes else decoded_bytes,
)
@@ -235,6 +252,11 @@ class RewriteProfile:
target_file_size: int = _DEFAULT_TARGET_FILE_SIZE
#: Columns of the rows the merge writes. Each column of an open output file keeps buffers.
columns: int = _UNKNOWN_COLUMNS
+ #: Candidate files whose reader bound is their largest row group, read from the footer. The
+ #: other candidates count their whole size.
+ row_group_sized_files: int = 0
+ #: Footers fetched from storage to size this merge. Cached footers are not counted.
+ footers_read: int = 0
@property
def files(self) -> int:
@@ -246,15 +268,23 @@ class RewriteProfile:
@property
def max_row_group_mb(self) -> float:
- """Upper bound on the largest row group a reader fetches: the largest candidate file.
+ """Upper bound on the largest row group a reader fetches.
- Add actions carry each file's size but not its row-group count, and the footer that has it
- costs one request per file. A file holds at least one row group, so its size bounds its
- largest one. The bound is exact for single-row-group files, which is what deltalite and
- delta-rs write while a file stays under the 1M-row row-group limit. For files with several
- row groups it over-counts (about 1.5× on 2–3 row-group files), so it errs toward a smaller plan.
+ A file holds at least one row group, so its size bounds its largest one. The bound is exact
+ for single-row-group files. For a file whose footer was read, the bound is its real largest
+ row group, which is much smaller for files written with a row-group cap.
"""
- return max((p.largest_file_bytes[0] for p in self.partitions if p.largest_file_bytes), default=0) / MB
+ return max((p.largest_row_group_bytes[0] for p in self.partitions if p.largest_row_group_bytes), default=0) / MB
+
+
+@frozen
+class CandidateFile:
+ """One existing file that a merge can rewrite."""
+
+ partition: str | None
+ path: str | None
+ size_bytes: int
+ num_records: int | None
def _files_that_can_match(add_actions: pa.Table, source: pa.Table, primary_keys: Sequence[str]) -> list[bool]:
@@ -291,19 +321,63 @@ def _files_that_can_match(add_actions: pa.Table, source: pa.Table, primary_keys:
return [d is not True for d in disjoint.to_pylist()]
+def _rows_per_partition(source: pa.Table, partition_col: str | None) -> dict[str | None, int]:
+ if partition_col is None:
+ return {None: source.num_rows}
+ counts = pc.value_counts(source[partition_col].cast(pa.string()))
+ return {
+ value: int(count or 0)
+ for value, count in zip(counts.field("values").to_pylist(), counts.field("counts").to_pylist())
+ }
+
+
+def candidate_files(
+ add_actions: pa.Table, source: pa.Table, partition_col: str | None, primary_keys: Sequence[str]
+) -> list[CandidateFile]:
+ """The existing files a merge of ``source`` can rewrite.
+
+ A file is a candidate when it is in a partition the batch touches and its PK range overlaps the
+ batch. Every candidate is counted because duplicate target keys can make one source row match
+ several files.
+ """
+ if source.num_rows == 0 or add_actions.num_rows == 0:
+ return []
+ touched = _rows_per_partition(source, partition_col)
+ file_partitions: list[str | None] = (
+ [None] * add_actions.num_rows
+ if partition_col is None
+ else add_actions[f"partition.{partition_col}"].cast(pa.string()).to_pylist()
+ )
+ paths = add_actions["path"].to_pylist() if "path" in add_actions.column_names else [None] * add_actions.num_rows
+ file_rows = (
+ add_actions["num_records"].to_pylist()
+ if "num_records" in add_actions.column_names
+ else [None] * add_actions.num_rows
+ )
+ can_match = _files_that_can_match(add_actions, source, primary_keys)
+ return [
+ CandidateFile(partition=partition, path=path, size_bytes=size or 0, num_records=rows)
+ for partition, path, size, rows, candidate in zip(
+ file_partitions, paths, add_actions["size_bytes"].to_pylist(), file_rows, can_match
+ )
+ if candidate and partition in touched
+ ]
+
+
def rewrite_profile(
add_actions: pa.Table,
source: pa.Table,
partition_col: str | None,
primary_keys: Sequence[str],
target_file_size: int = _DEFAULT_TARGET_FILE_SIZE,
+ row_groups: RowGroupSizes | None = None,
+ candidates: Sequence[CandidateFile] | None = None,
) -> RewriteProfile:
"""Upper bound on the existing files a merge of ``source`` rewrites, from the table's add actions.
- A file is a candidate when it is in a partition the batch touches and its PK range overlaps the
- batch. Every candidate is counted because duplicate target keys can make one source row match
- several files. Every touched partition gets an entry with its share of the source, also when it
- has no candidate files.
+ Every touched partition gets an entry with its share of the source, also when it has no
+ candidate files. ``row_groups`` gives the largest row group of the files whose footer was read.
+ Each other file counts its whole size. ``candidates`` reuses a ``candidate_files`` result.
"""
if source.num_rows == 0:
return RewriteProfile(
@@ -313,35 +387,26 @@ def rewrite_profile(
columns=source.num_columns,
)
- if partition_col is None:
- rows_per_partition: dict[str | None, int] = {None: source.num_rows}
- file_partitions: list[str | None] = [None] * add_actions.num_rows
- else:
- counts = pc.value_counts(source[partition_col].cast(pa.string()))
- rows_per_partition = {
- value: int(count or 0)
- for value, count in zip(counts.field("values").to_pylist(), counts.field("counts").to_pylist())
- }
- file_partitions = (
- add_actions[f"partition.{partition_col}"].cast(pa.string()).to_pylist() if add_actions.num_rows else []
- )
-
+ rows_per_partition = _rows_per_partition(source, partition_col)
+ known_row_groups: Mapping[str, int] = row_groups.bytes_by_path if row_groups is not None else {}
sizes_by_partition: dict[str | None, list[int]] = {partition: [] for partition in rows_per_partition}
+ readers_by_partition: dict[str | None, list[int]] = {partition: [] for partition in rows_per_partition}
rows_by_partition: dict[str | None, int | None] = dict.fromkeys(rows_per_partition, 0)
- if add_actions.num_rows:
- can_match = _files_that_can_match(add_actions, source, primary_keys)
- file_rows = (
- add_actions["num_records"].to_pylist()
- if "num_records" in add_actions.column_names
- else [None] * add_actions.num_rows
+ row_group_sized = 0
+ if candidates is None:
+ candidates = candidate_files(add_actions, source, partition_col, primary_keys)
+ for file in candidates:
+ sizes_by_partition[file.partition].append(file.size_bytes)
+ largest_row_group = known_row_groups.get(file.path) if file.path is not None else None
+ if largest_row_group is None:
+ readers_by_partition[file.partition].append(file.size_bytes)
+ else:
+ row_group_sized += 1
+ readers_by_partition[file.partition].append(min(largest_row_group, file.size_bytes))
+ known = rows_by_partition[file.partition]
+ rows_by_partition[file.partition] = (
+ None if known is None or file.num_records is None else known + file.num_records
)
- for partition, size, rows, candidate in zip(
- file_partitions, add_actions["size_bytes"].to_pylist(), file_rows, can_match
- ):
- if candidate and partition in sizes_by_partition:
- sizes_by_partition[partition].append(size or 0)
- known = rows_by_partition[partition]
- rows_by_partition[partition] = None if known is None or rows is None else known + rows
# The source has the table's columns, so its bytes per row also size the decoded candidates.
bytes_per_row = source.nbytes / source.num_rows
@@ -353,6 +418,7 @@ def rewrite_profile(
sizes,
source_bytes=round(bytes_per_row * rows_per_partition[partition]),
decoded_bytes=None if candidate_rows is None else round(bytes_per_row * candidate_rows),
+ row_group_bytes=readers_by_partition[partition],
)
)
return RewriteProfile(
@@ -360,19 +426,52 @@ def rewrite_profile(
table_files=add_actions.num_rows,
target_file_size=target_file_size,
columns=source.num_columns,
+ row_group_sized_files=row_group_sized,
+ footers_read=row_groups.footers_read if row_groups is not None else 0,
)
def estimate_rewrite_profile(
- table: deltalake.DeltaTable, source: pa.Table, partition_col: str | None, primary_keys: Sequence[str]
+ table: deltalake.DeltaTable,
+ source: pa.Table,
+ partition_col: str | None,
+ primary_keys: Sequence[str],
+ row_group_reader: RowGroupSizeReader | None = None,
+ wants_row_groups: Callable[[RewriteProfile], bool] | None = None,
) -> RewriteProfile | None:
- """``rewrite_profile`` for a live table, or None when its add actions cannot be read."""
+ """``rewrite_profile`` for a live table, or None when its add actions cannot be read.
+
+ With a ``row_group_reader``, the footers of candidate files above ``_ROW_GROUP_PROBE_MIN_BYTES``
+ are read when ``wants_row_groups`` says the whole-file profile limits the plan. Any failure
+ there keeps the whole-file profile.
+ """
try:
add_actions = pa.table(table.get_add_actions(flatten=True))
- return rewrite_profile(add_actions, source, partition_col, primary_keys, table_target_file_size(table))
+ target_file_size = table_target_file_size(table)
+ candidates = candidate_files(add_actions, source, partition_col, primary_keys)
+ profile = rewrite_profile(
+ add_actions, source, partition_col, primary_keys, target_file_size, candidates=candidates
+ )
except Exception as e: # noqa: BLE001 - an estimate must never fail a write; None sizes conservatively
logger.debug("deltalite governor: could not estimate the rewrite size: %s", e)
return None
+ if row_group_reader is None or profile.max_row_group_mb * MB <= _ROW_GROUP_PROBE_MIN_BYTES:
+ return profile
+ try:
+ if wants_row_groups is not None and not wants_row_groups(profile):
+ return profile
+ to_read = {
+ file.path: file.size_bytes
+ for file in candidates
+ if file.path is not None and file.size_bytes > _ROW_GROUP_PROBE_MIN_BYTES
+ }
+ row_groups = row_group_reader.largest_row_groups(to_read)
+ return rewrite_profile(
+ add_actions, source, partition_col, primary_keys, target_file_size, row_groups, candidates
+ )
+ except Exception as e: # noqa: BLE001 - row-group sizing is an optimization; keep the whole-file bound
+ logger.debug("deltalite governor: could not read row group sizes: %s", e)
+ return profile
@frozen
@@ -423,7 +522,7 @@ def _writer_mb(written_mb: float, target_mb: float, columns: int) -> float:
def _worker_terms(shape: PartitionShape, files_per_worker: int, target_mb: float, columns: int) -> WorkerTerms:
- row_groups_mb = [size / MB for size in shape.largest_file_bytes[:files_per_worker]]
+ row_groups_mb = [size / MB for size in shape.largest_row_group_bytes[:files_per_worker]]
written = (shape.stored_bytes + shape.source_bytes) / MB
return WorkerTerms(
held_mb=sum(row_groups_mb),
@@ -552,6 +651,22 @@ def _reader_options(mpp: int) -> list[int]:
return sorted({f for f in (widest, _DEFAULT_FILES_PER_WORKER, 2, 1) if f <= widest}, reverse=True)
+def _partition_cap(profile: RewriteProfile, n_partitions: int | None) -> int:
+ cap = min(_MAX_PARALLEL_PARTITIONS, max(1, len(profile.partitions)))
+ if n_partitions is not None and n_partitions >= 1:
+ cap = min(cap, n_partitions)
+ return cap
+
+
+def is_widest_plan(plan: UpsertPlan, profile: RewriteProfile, n_partitions: int | None = None) -> bool:
+ """True when ``plan`` fits and already runs the most workers and readers ``size_upsert`` ever picks."""
+ partition_cap = _partition_cap(profile, n_partitions)
+ return plan.fits and (plan.max_parallel_partitions, plan.max_parallel_files) == (
+ partition_cap,
+ _reader_options(partition_cap)[0],
+ )
+
+
def size_upsert(
available_mb: float,
source_mb: float,
@@ -584,10 +699,7 @@ def size_upsert(
columns=profile.columns,
)
- partition_cap = min(_MAX_PARALLEL_PARTITIONS, max(1, len(profile.partitions)))
- if n_partitions is not None and n_partitions >= 1:
- partition_cap = min(partition_cap, n_partitions)
-
+ partition_cap = _partition_cap(profile, n_partitions)
# One pass over the partitions per reader count, however many plans are tried.
costliest: dict[int, WorkerBounds] = {}
for mpp in range(partition_cap, 0, -1):
@@ -792,8 +904,11 @@ class Admission:
capacity_exceeded: bool = False
#: The predicted peak by term, in-use and RSS.
estimate: MemoryEstimate | None = None
- #: Largest candidate file, the bound on the largest row group a reader fetches. None when unknown.
+ #: The bound on the largest row group a reader fetches. None when unknown.
max_row_group_mb: float | None = None
+ #: Candidate files sized by their largest row group, and footers fetched for that. None when unknown.
+ row_group_sized_files: int | None = None
+ footers_read: int | None = None
#: Stored MB of all candidate files the merge can rewrite, over every touched partition.
rewrite_total_mb: float | None = None
rewrite_files: int | None = None
@@ -874,6 +989,22 @@ class MemoryGovernor:
limit_mb = self.pod.limit_mb()
return None if limit_mb is None else self._per_upsert_budget_mb(limit_mb)
+ def row_groups_can_widen(
+ self, profile: RewriteProfile, *, source_bytes: int, n_partitions: int | None = None
+ ) -> bool:
+ """True when the plan for ``profile`` is narrower than the widest plan.
+
+ Only then can smaller reader bounds change the plan, so only then are footer reads worth
+ their requests. False when this governor sizes no plan (mode off, or no pod limit).
+ """
+ limit_mb = self.pod.limit_mb()
+ if self.config.mode == "off" or limit_mb is None:
+ return False
+ plan = size_upsert(
+ self._per_upsert_budget_mb(limit_mb), source_bytes / MB, n_partitions, profile, self.config.rss_retention
+ )
+ return not is_widest_plan(plan, profile, n_partitions)
+
def _has_room(self, reserve_mb: float, usable_mb: float) -> bool:
return self._reserved_mb + reserve_mb <= usable_mb + _ROOM_TOLERANCE_MB
@@ -974,6 +1105,8 @@ class MemoryGovernor:
capacity_exceeded=plan is not None and not plan.fits,
estimate=plan.estimate if plan is not None else None,
max_row_group_mb=round(rewrite.max_row_group_mb, 1) if rewrite is not None else None,
+ row_group_sized_files=rewrite.row_group_sized_files if rewrite is not None else None,
+ footers_read=rewrite.footers_read if rewrite is not None else None,
rewrite_total_mb=round(rewrite.total_mb, 1) if rewrite is not None else None,
rewrite_files=rewrite.files if rewrite is not None else None,
columns=rewrite.columns if rewrite is not None else None,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/row_group_sizes.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/row_group_sizes.py
new file mode 100644
index 000000000000..a9c6b7152aaf
--- /dev/null
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/row_group_sizes.py
@@ -0,0 +1,204 @@
+"""The largest row group of each Parquet file a merge can rewrite, read from the file footers.
+
+A deltalite reader reserves fetch budget for the largest compressed row group of a file, and then
+holds one row group at a time. Delta add actions record the size of each file but not its row
+groups, so the memory governor otherwise sizes each reader by the whole file. This module reads the
+footers so that the governor can use the real row-group size.
+"""
+
+from __future__ import annotations
+
+import logging
+import threading
+import urllib.parse
+from collections import OrderedDict
+from collections.abc import Callable, Mapping
+from concurrent.futures import Future, ThreadPoolExecutor, wait
+
+import pyarrow.fs as pafs
+import pyarrow.parquet as pq
+from deltalake.fs import DeltaStorageHandler
+
+from posthog.dataclasses import frozen
+
+logger = logging.getLogger(__name__)
+
+#: Most footers one merge reads. Files past this keep their whole size as the bound.
+_MAX_READS = 1024
+#: Footers read at the same time. Each read is one ranged GET of the file tail.
+_READ_CONCURRENCY = 32
+#: Longest the footer reads of one merge may take. Files not read in time keep their whole size.
+_READ_TIMEOUT_S = 10.0
+#: Footer reads queued or running across all merges. A merge past this reads no footers.
+_MAX_PENDING_READS = 2 * _MAX_READS
+#: Files are immutable, so a footer read once stays correct. About 100 bytes per entry.
+_CACHE_ENTRIES = 100_000
+
+
+def largest_row_group_bytes(metadata: pq.FileMetaData) -> int:
+ """Compressed bytes of the largest row group, counted as deltalite counts its fetch reservation."""
+ largest = 0
+ for i in range(metadata.num_row_groups):
+ row_group = metadata.row_group(i)
+ largest = max(largest, sum(row_group.column(c).total_compressed_size for c in range(row_group.num_columns)))
+ return largest
+
+
+class _ReadSlots:
+ """Bounds the footer reads queued or running on the shared executor, over all merges."""
+
+ def __init__(self, limit: int) -> None:
+ self._limit = limit
+ self._used = 0
+ self._lock = threading.Lock()
+
+ def acquire(self, wanted: int) -> int:
+ with self._lock:
+ granted = max(0, min(wanted, self._limit - self._used))
+ self._used += granted
+ return granted
+
+ def release(self, count: int = 1) -> None:
+ with self._lock:
+ self._used -= count
+
+
+def _read_footer(path: str, filesystem: pafs.FileSystem, expired: threading.Event, slots: _ReadSlots) -> int | None:
+ # A read that its merge stopped waiting for still holds a slot until a worker takes it, so
+ # stalled storage stops new reads instead of growing the queue.
+ try:
+ if expired.is_set():
+ return None
+ return largest_row_group_bytes(pq.read_metadata(path, filesystem=filesystem))
+ finally:
+ slots.release()
+
+
+@frozen
+class RowGroupSizes:
+ """Largest row group per file path, for the files whose footer could be read."""
+
+ bytes_by_path: Mapping[str, int]
+ #: Footers fetched from storage for this call. Cache hits are not counted.
+ footers_read: int = 0
+
+
+class _FooterCache:
+ """Process-wide LRU of the largest row group per (table, file)."""
+
+ def __init__(self, max_entries: int) -> None:
+ self._max_entries = max_entries
+ self._entries: OrderedDict[tuple[str, str], int] = OrderedDict()
+ self._lock = threading.Lock()
+
+ def get(self, key: tuple[str, str]) -> int | None:
+ with self._lock:
+ value = self._entries.get(key)
+ if value is not None:
+ self._entries.move_to_end(key)
+ return value
+
+ def put(self, key: tuple[str, str], value: int) -> None:
+ with self._lock:
+ self._entries[key] = value
+ self._entries.move_to_end(key)
+ while len(self._entries) > self._max_entries:
+ self._entries.popitem(last=False)
+
+ def clear(self) -> None:
+ with self._lock:
+ self._entries.clear()
+
+
+_CACHE = _FooterCache(_CACHE_ENTRIES)
+
+#: Shared by all merges, so reads that outlive their deadline cannot add threads past this bound.
+_EXECUTOR = ThreadPoolExecutor(max_workers=_READ_CONCURRENCY, thread_name_prefix="row-group-footers")
+_SLOTS = _ReadSlots(_MAX_PENDING_READS)
+
+#: Opens a filesystem for a table that knows the sizes of the given files, so no read needs a HEAD.
+OpenFilesystem = Callable[[Mapping[str, int]], pafs.FileSystem]
+
+
+class RowGroupSizeReader:
+ """Reads the largest row group of a table's files from their footers.
+
+ A file whose footer cannot be read in time, or at all, is left out of the result, and the
+ caller keeps its whole size as the bound. So a failure can only make the plan smaller.
+ """
+
+ def __init__(
+ self,
+ table_uri: str,
+ storage_options: Mapping[str, str] | None = None,
+ *,
+ open_filesystem: OpenFilesystem | None = None,
+ max_reads: int = _MAX_READS,
+ executor: ThreadPoolExecutor = _EXECUTOR,
+ slots: _ReadSlots = _SLOTS,
+ timeout_s: float = _READ_TIMEOUT_S,
+ cache: _FooterCache = _CACHE,
+ ) -> None:
+ self._table_uri = table_uri
+ self._storage_options = dict(storage_options or {})
+ self._open_filesystem = open_filesystem or self._delta_filesystem
+ self._max_reads = max_reads
+ self._executor = executor
+ self._slots = slots
+ self._timeout_s = timeout_s
+ self._cache = cache
+
+ def _delta_filesystem(self, known_sizes: Mapping[str, int]) -> pafs.FileSystem:
+ handler = DeltaStorageHandler(self._table_uri, self._storage_options, dict(known_sizes))
+ return pafs.PyFileSystem(handler)
+
+ def largest_row_groups(self, sizes_by_path: Mapping[str, int]) -> RowGroupSizes:
+ """The largest row group of each file in ``sizes_by_path``, largest files first."""
+ found: dict[str, int] = {}
+ to_read: dict[str, int] = {}
+ for path, size in sorted(sizes_by_path.items(), key=lambda item: item[1], reverse=True):
+ cached = self._cache.get((self._table_uri, path))
+ if cached is not None:
+ found[path] = cached
+ elif len(to_read) < self._max_reads:
+ to_read[path] = size
+ granted = self._slots.acquire(len(to_read))
+ to_read = dict(list(to_read.items())[:granted])
+ if not to_read:
+ return RowGroupSizes(bytes_by_path=found)
+
+ # Add action paths are URI-encoded; the storage handler takes decoded paths.
+ storage_paths = {path: urllib.parse.unquote(path) for path in to_read}
+ expired = threading.Event()
+ futures: dict[Future[int | None], str] = {}
+ try:
+ filesystem = self._open_filesystem({storage_paths[path]: size for path, size in to_read.items()})
+ for path in to_read:
+ future = self._executor.submit(_read_footer, storage_paths[path], filesystem, expired, self._slots)
+ futures[future] = path
+ except BaseException:
+ # Each submitted read frees its own slot; the rest were never handed to a worker.
+ self._slots.release(len(to_read) - len(futures))
+ raise
+ done, not_done = wait(futures, timeout=self._timeout_s)
+ expired.set()
+
+ failed = 0
+ for future in done:
+ path = futures[future]
+ if future.exception() is not None:
+ failed += 1
+ continue
+ result = future.result()
+ if result is None:
+ continue
+ largest = min(result, to_read[path])
+ self._cache.put((self._table_uri, path), largest)
+ found[path] = largest
+ if failed or not_done:
+ logger.debug(
+ "row group sizes: %d footer reads failed and %d timed out; those files keep their whole size",
+ failed,
+ len(not_done),
+ )
+ return RowGroupSizes(bytes_by_path=found, footers_read=len(done) - failed)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_memory_governor.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_memory_governor.py
index 9154a016b39a..6d29068ad0c5 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_memory_governor.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_memory_governor.py
@@ -8,6 +8,8 @@ from django.test import override_settings
import pyarrow as pa
import deltalake
+import pyarrow.fs as pafs
+import pyarrow.parquet as pq
from parameterized import parameterized
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta import memory_governor as _mg
@@ -25,6 +27,11 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.del
rewrite_profile,
size_upsert,
)
+from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.row_group_sizes import (
+ RowGroupSizeReader,
+ RowGroupSizes,
+ _FooterCache,
+)
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.rss_sampler import RssPeakSampler
@@ -78,14 +85,18 @@ def _profile(
columns: int = 64,
decoded_mb: float | None = None,
source_mb_per_partition: float = 0.0,
+ row_group_mb: float | None = None,
) -> RewriteProfile:
- """A profile from per-partition candidate file sizes in MB."""
+ """A profile from per-partition candidate file sizes in MB, each file of row groups up to ``row_group_mb``."""
return RewriteProfile(
partitions=tuple(
PartitionShape.of(
[round(size * MB) for size in sizes],
source_bytes=round(source_mb_per_partition * MB),
decoded_bytes=None if decoded_mb is None else round(decoded_mb * MB),
+ row_group_bytes=None
+ if row_group_mb is None
+ else [round(min(size, row_group_mb) * MB) for size in sizes],
)
for sizes in partitions
),
@@ -332,6 +343,31 @@ class TestSizeUpsert:
8,
True,
),
+ # The same default-size files written as 16 MB row groups: each reader holds 16 MB, not 100.
+ (
+ "wide_rows_in_16mb_row_groups",
+ _profile(*([[100.0] * 2] * 4), columns=30, row_group_mb=16.0),
+ None,
+ 1.4,
+ _SLOT_14GI,
+ 1,
+ 8,
+ True,
+ ),
+ ("single_row_group_files_take_one_worker", _profile(*([[130.0] * 2] * 4)), None, 2.0, _SLOT, 1, 8, True),
+ (
+ "small_row_groups_take_two_workers",
+ _profile(*([[130.0] * 2] * 4), row_group_mb=16.0),
+ None,
+ 2.0,
+ _SLOT,
+ 2,
+ 8,
+ True,
+ ),
+ # Three 130 MB single-row-group files overflow a 900 MB slice; in 16 MB row groups they fit 1 x 8.
+ ("single_row_group_files_overflow", _profile([130.0] * 3), None, 2.0, 900.0, 1, 1, False),
+ ("small_row_groups_fit", _profile([130.0] * 3, row_group_mb=16.0), None, 2.0, 900.0, 1, 8, True),
]
)
def test_sizing(self, _name, profile, n_partitions, retention, slot, exp_mpp, exp_files, exp_fits):
@@ -514,6 +550,19 @@ class TestGovernorSizing:
pass
assert (adm.max_row_group_mb, adm.rewrite_files, adm.rewrite_total_mb, adm.columns) == (57.0, 24, 1368.0, 64)
+ @parameterized.expand(
+ [
+ ("narrow_plan_can_widen", "enforce", 15_000.0, _profile(*([[130.0] * 2] * 4)), True),
+ ("advisory_sizes_a_plan_too", "advisory", 15_000.0, _profile(*([[130.0] * 2] * 4)), True),
+ ("widest_plan_cannot_widen", "enforce", 30_000.0, _SMALL, False),
+ ("off_sizes_no_plan", "off", 15_000.0, _profile(*([[130.0] * 2] * 4)), False),
+ ("no_pod_limit_sizes_no_plan", "enforce", None, _profile(*([[130.0] * 2] * 4)), False),
+ ]
+ )
+ def test_row_groups_can_widen_only_a_narrow_plan(self, _name, mode, limit_mb, profile, expected):
+ gov = _governor(mode, limit_mb=limit_mb)
+ assert gov.row_groups_can_widen(profile, source_bytes=10 * MB) is expected
+
async def test_reservation_released_on_exception(self):
sampler = _fake_sampler(1_000.0, 1_000.0)
gov = _governor("enforce", rss_sampler=sampler)
@@ -680,7 +729,7 @@ class TestRewriteProfile:
)
def test_profile(self, _name, source, partition_col, expected_files, expected_count):
profile = rewrite_profile(_add_actions(), source, partition_col, ["id"])
- assert tuple(p.largest_file_bytes for p in profile.partitions) == expected_files
+ assert tuple(p.largest_row_group_bytes for p in profile.partitions) == expected_files
assert profile.files == expected_count
assert profile.table_files == len(_FILES)
# Each partition carries its share of the source, for the bytes its worker writes.
@@ -698,7 +747,7 @@ class TestRewriteProfile:
source = _source([("a", 5000), ("a", 5001), ("a", 5002)])
source = source.append_column("other", pa.array([1, 2, 3], pa.int64()))
profile = rewrite_profile(add_actions, source, _PART, primary_keys)
- assert [(p.largest_file_bytes, p.stored_bytes) for p in profile.partitions] == [((100, 50, 30), 180)]
+ assert [(p.largest_row_group_bytes, p.stored_bytes) for p in profile.partitions] == [((100, 50, 30), 180)]
@parameterized.expand(
[
@@ -720,7 +769,7 @@ class TestRewriteProfile:
def test_one_source_row_can_match_duplicate_keys_in_multiple_files(self):
files = [("a", 100, 100, 200), ("a", 50, 150, 250)]
profile = rewrite_profile(_add_actions(files), _source([("a", 175)]), _PART, ["id"])
- assert (profile.partitions[0].largest_file_bytes, profile.files) == ((100, 50), 2)
+ assert (profile.partitions[0].largest_row_group_bytes, profile.files) == ((100, 50), 2)
def test_string_keys_are_not_pruned(self):
# Delta truncates long string stats, so a string max proves nothing.
@@ -736,7 +785,7 @@ class TestRewriteProfile:
files = [("a", size, 0, 99) for size in range(1, 21)]
source = _source([("a", i) for i in range(20)])
shape = rewrite_profile(_add_actions(files), source, _PART, ["id"]).partitions[0]
- assert shape.largest_file_bytes == (20, 19, 18, 17, 16, 15, 14, 13)
+ assert shape.largest_row_group_bytes == (20, 19, 18, 17, 16, 15, 14, 13)
assert (shape.files, shape.stored_bytes) == (20, sum(range(1, 21)))
@parameterized.expand(
@@ -747,7 +796,7 @@ class TestRewriteProfile:
)
def test_nothing_to_rewrite(self, _name, files, source, expected_files, table_files):
profile = rewrite_profile(_add_actions(files), source, _PART, ["id"])
- assert tuple(p.largest_file_bytes for p in profile.partitions) == expected_files
+ assert tuple(p.largest_row_group_bytes for p in profile.partitions) == expected_files
assert (profile.files, profile.total_mb, profile.max_row_group_mb, profile.table_files) == (
0,
0.0,
@@ -781,6 +830,87 @@ class TestRewriteProfile:
assert (profile.files, profile.table_files, profile.target_file_size) == (2, 3, expected_target)
assert profile.max_row_group_mb == max(sizes[f"{_PART}=a"], sizes[f"{_PART}=b"]) / MB
+ @parameterized.expand(
+ [
+ # a/f0 holds row groups of 10 bytes at most; the other files keep their size.
+ ("footer_bounds_the_reader", {f"{_PART}=a/f0.parquet": 10}, (50, 30, 10), 1),
+ # A footer cannot report more than the file holds; the file size still bounds it.
+ ("never_above_the_file_size", {f"{_PART}=a/f0.parquet": 500}, (100, 50, 30), 1),
+ ("unread_files_keep_their_size", {"elsewhere.parquet": 1}, (100, 50, 30), 0),
+ ]
+ )
+ def test_row_groups_bound_the_readers(self, _name, bytes_by_path, expected, sized):
+ row_groups = RowGroupSizes(bytes_by_path=bytes_by_path, footers_read=2)
+ source = _source([("a", 5), ("a", 150), ("a", 250)])
+ profile = rewrite_profile(_add_actions(), source, _PART, ["id"], row_groups=row_groups)
+ shape = profile.partitions[0]
+ assert shape.largest_row_group_bytes == expected
+ assert (shape.files, shape.stored_bytes) == (3, 180)
+ assert (profile.row_group_sized_files, profile.footers_read) == (sized, 2)
+
+ @pytest.mark.parametrize(
+ "reader,wants,sized",
+ [
+ ("local", None, True),
+ # The setting off passes no reader: today's whole-file sizing.
+ (None, None, False),
+ # A plan that is already the widest gains nothing from footers, so none are read.
+ ("local", False, False),
+ ("broken", None, False),
+ ],
+ ids=["reads_the_footers", "no_reader", "widest_plan_reads_nothing", "unreadable_footers"],
+ )
+ def test_estimate_sizes_readers_by_their_row_groups(self, reader, wants, sized, tmp_path: Path):
+ existing = pa.table(
+ {
+ _PART: ["a"] * 2000 + ["b"] * 2000,
+ "id": pa.array(range(4000), pa.int64()),
+ "v": [f"value-{i:06d}" * 8 for i in range(4000)],
+ }
+ )
+ deltalake.write_deltalake(
+ str(tmp_path),
+ existing,
+ partition_by=_PART,
+ writer_properties=deltalake.WriterProperties(max_row_group_size=250, compression="UNCOMPRESSED"),
+ )
+ table = deltalake.DeltaTable(str(tmp_path))
+ sizes = table._table.get_add_file_sizes()
+ footers = [pq.read_metadata(tmp_path / path) for path in sizes]
+ largest_row_group = max(
+ sum(m.row_group(i).column(c).total_compressed_size for c in range(m.num_columns))
+ for m in footers
+ for i in range(m.num_row_groups)
+ )
+
+ def _broken(_sizes) -> pafs.FileSystem:
+ raise OSError("unreachable")
+
+ readers = {
+ "local": RowGroupSizeReader(str(tmp_path), cache=_FooterCache(10)),
+ "broken": RowGroupSizeReader(str(tmp_path), open_filesystem=_broken, cache=_FooterCache(10)),
+ None: None,
+ }
+ with patch.object(_mg, "_ROW_GROUP_PROBE_MIN_BYTES", 0):
+ profile = estimate_rewrite_profile(
+ table,
+ _source([("a", 5), ("b", 2500)]),
+ _PART,
+ ["id"],
+ readers[reader],
+ None if wants is None else lambda _profile: wants,
+ )
+
+ assert profile is not None
+ assert (profile.files, profile.total_mb) == (2, sum(sizes.values()) / MB)
+ if sized:
+ assert largest_row_group < max(sizes.values()) / 4
+ assert profile.max_row_group_mb == largest_row_group / MB
+ assert (profile.row_group_sized_files, profile.footers_read) == (2, 2)
+ else:
+ assert profile.max_row_group_mb == max(sizes.values()) / MB
+ assert (profile.row_group_sized_files, profile.footers_read) == (0, 0)
+
def test_estimate_unreadable_table_is_unknown(self):
broken = MagicMock()
broken.get_add_actions.side_effect = RuntimeError("no snapshot")
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_row_group_sizes.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_row_group_sizes.py
new file mode 100644
index 000000000000..088b940b91a6
--- /dev/null
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_row_group_sizes.py
@@ -0,0 +1,149 @@
+import threading
+from pathlib import Path
+
+import pytest
+from unittest.mock import patch
+
+import pyarrow as pa
+import pyarrow.fs as pafs
+import pyarrow.parquet as pq
+
+from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta import row_group_sizes
+from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.row_group_sizes import (
+ RowGroupSizeReader,
+ _FooterCache,
+ _ReadSlots,
+)
+
+
+def _write(path: Path, rows: int, row_group_rows: int) -> int:
+ table = pa.table({"id": pa.array(range(rows), pa.int64()), "v": [f"value-{i:06d}" * 4 for i in range(rows)]})
+ pq.write_table(table, path, row_group_size=row_group_rows, compression="none")
+ return path.stat().st_size
+
+
+def _expected(path: str) -> int:
+ metadata = pq.read_metadata(path)
+ return max(
+ sum(metadata.row_group(i).column(c).total_compressed_size for c in range(metadata.num_columns))
+ for i in range(metadata.num_row_groups)
+ )
+
+
+_READ_METADATA = pq.read_metadata
+
+
+class _Footers:
+ """Counts footer reads, and fails or stalls the reads of chosen files."""
+
+ def __init__(self, broken: frozenset[str] = frozenset(), stalled: frozenset[str] = frozenset()) -> None:
+ self.read: list[str] = []
+ self._broken = broken
+ self._stalled = stalled
+ self.release = threading.Event()
+ self._lock = threading.Lock()
+
+ def __call__(self, path: str, filesystem: pafs.FileSystem) -> pq.FileMetaData:
+ with self._lock:
+ self.read.append(path)
+ if path in self._broken:
+ raise OSError("unreachable")
+ if path in self._stalled:
+ self.release.wait(5)
+ return _READ_METADATA(path, filesystem=filesystem)
+
+
+def files(tmp_path: Path) -> dict[str, int]:
+ # Many small row groups, one row group, and a small file.
+ return {
+ str(tmp_path / name): _write(tmp_path / name, rows, row_group_rows)
+ for name, rows, row_group_rows in (("many", 4000, 500), ("one", 3000, 3000), ("small", 100, 100))
+ }
+
+
+def _read(sizes: dict[str, int], footers: _Footers, cache: _FooterCache | None = None, **kwargs):
+ reader = RowGroupSizeReader(
+ "table", open_filesystem=lambda _sizes: pafs.LocalFileSystem(), cache=cache or _FooterCache(100), **kwargs
+ )
+ with patch.object(row_group_sizes.pq, "read_metadata", side_effect=footers):
+ return reader.largest_row_groups(sizes)
+
+
+class TestRowGroupSizeReader:
+ def test_reads_the_largest_row_group_of_each_file(self, files: dict[str, int]) -> None:
+ result = _read(files, _Footers())
+
+ assert result.bytes_by_path == {path: _expected(path) for path in files}
+ assert result.footers_read == 3
+ many, one = (next(p for p in files if p.endswith(name)) for name in ("many", "one"))
+ assert result.bytes_by_path[many] < files[many] / 4
+ assert result.bytes_by_path[one] > files[one] / 2
+
+ def test_a_cached_footer_is_not_read_again(self, files: dict[str, int]) -> None:
+ cache = _FooterCache(100)
+ footers = _Footers()
+ first = _read(files, footers, cache)
+ second = _read(files, footers, cache)
+
+ assert second.bytes_by_path == first.bytes_by_path
+ assert (second.footers_read, len(footers.read)) == (0, 3)
+
+ def test_the_read_cap_keeps_the_largest_files(self, files: dict[str, int]) -> None:
+ footers = _Footers()
+ result = _read(files, footers, max_reads=2)
+
+ largest_two = sorted(sorted(files, key=files.__getitem__, reverse=True)[:2])
+ assert sorted(result.bytes_by_path) == largest_two
+ assert sorted(footers.read) == largest_two
+
+ @pytest.mark.parametrize("failure", ["broken", "stalled"])
+ def test_a_file_that_cannot_be_read_is_left_out(self, failure: str, files: dict[str, int]) -> None:
+ bad = next(p for p in files if p.endswith("many"))
+ footers = _Footers(**{failure: frozenset({bad})})
+ try:
+ result = _read(files, footers, timeout_s=0.5)
+ finally:
+ footers.release.set()
+
+ assert set(result.bytes_by_path) == set(files) - {bad}
+ assert result.footers_read == 2
+
+ def test_no_files_reads_nothing(self) -> None:
+ footers = _Footers()
+ assert _read({}, footers).bytes_by_path == {}
+ assert footers.read == []
+
+ def test_reads_a_uri_encoded_path_by_its_decoded_name(self, tmp_path: Path) -> None:
+ (tmp_path / "p=a b").mkdir()
+ size = _write(tmp_path / "p=a b" / "file.parquet", 2000, 500)
+ encoded = str(tmp_path / "p=a%20b" / "file.parquet")
+
+ result = _read({encoded: size}, _Footers())
+
+ assert result.bytes_by_path == {encoded: _expected(str(tmp_path / "p=a b" / "file.parquet"))}
+
+ def test_stalled_reads_hold_their_slots_so_later_calls_read_nothing(self, files: dict[str, int]) -> None:
+ many = next(p for p in files if p.endswith("many"))
+ slots = _ReadSlots(1)
+ footers = _Footers(stalled=frozenset({many}))
+ try:
+ stalled = _read(files, footers, slots=slots, timeout_s=0.2)
+ later = _read(files, footers, slots=slots)
+ finally:
+ footers.release.set()
+
+ assert (stalled.bytes_by_path, later.bytes_by_path) == ({}, {})
+ assert footers.read == [many]
+
+ def test_a_filesystem_that_cannot_open_frees_its_slots(self, files: dict[str, int]) -> None:
+ slots = _ReadSlots(len(files))
+
+ def broken(_sizes: object) -> pafs.FileSystem:
+ raise OSError("no credentials")
+
+ reader = RowGroupSizeReader("table", open_filesystem=broken, cache=_FooterCache(100), slots=slots)
+ with pytest.raises(OSError):
+ reader.largest_row_groups(files)
+
+ assert slots.acquire(len(files)) == len(files)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/writer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/writer.py
index 38fa23eba368..d484bd0092c3 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/writer.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/writer.py
@@ -1,6 +1,7 @@
import json
import time
import asyncio
+import functools
import contextlib
from collections.abc import Callable, Sequence
from typing import TYPE_CHECKING, Any, Literal
@@ -303,6 +304,9 @@ class DeltaWriter:
estimate_rewrite_profile,
get_governor,
)
+ from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.row_group_sizes import (
+ RowGroupSizeReader,
+ )
from products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.load.metrics import (
DELTALITE_WRITE_DURATION_SECONDS,
DELTALITE_WRITE_TOTAL,
@@ -325,12 +329,25 @@ class DeltaWriter:
# always writes — the governor never falls back to the delta-rs MERGE for capacity, because
# the MERGE is the *more* memory-hungry path. A source too big for its slice just runs with
# the smallest plan (governor logs a capacity_exceeded ops signal). The shape of the files
- # the merge can rewrite comes from the handle's add actions, so this costs no object-store
- # request.
+ # the merge can rewrite comes from the handle's add actions. Footers of large candidate
+ # files are read only when their whole sizes keep the plan narrow.
governor = get_governor()
+ row_group_reader = (
+ RowGroupSizeReader(uri, storage_options)
+ if settings.DATA_WAREHOUSE_DELTALITE_GOVERNOR_ROW_GROUP_SIZING
+ else None
+ )
rewrite = (
await asyncio.to_thread(
- estimate_rewrite_profile, existing_delta_table, data, partition_key, normalized_primary_keys
+ estimate_rewrite_profile,
+ existing_delta_table,
+ data,
+ partition_key,
+ normalized_primary_keys,
+ row_group_reader,
+ functools.partial(
+ governor.row_groups_can_widen, source_bytes=data.nbytes, n_partitions=n_partitions
+ ),
)
if governor.config.mode != "off"
else None
@@ -415,6 +432,8 @@ class DeltaWriter:
governor_mpp=adm.planned_mpp,
governor_mpf=adm.planned_mpf,
governor_max_row_group_mb=adm.max_row_group_mb,
+ governor_row_group_sized_files=adm.row_group_sized_files,
+ governor_footers_read=adm.footers_read,
governor_rewrite_total_mb=adm.rewrite_total_mb,
governor_rewrite_files=adm.rewrite_files,
governor_columns=adm.columns,