Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

perf(data-warehouse): stop copying deduped loader batches twice (#121891)

Tom Owersb041798b8ea6
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/arrow_utils.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/arrow_utils.py
index 52669084b96f..56b9cf4a37f1 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/arrow_utils.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/arrow_utils.py
@@ -1904,6 +1904,9 @@ def first_per_pk_table(pa_table: pa.Table, pk_columns: list[str], keep: Literal[
the latest occurrence of a key carries the freshest data. Either way the merge
receives at most one source row per key, avoiding ambiguous multi-match merge
semantics (and the duplicate inserts `when_not_matched_insert_all` would produce).
+
+ When no PK tuple repeats, the input table itself is returned, so the caller does not hold a
+ second copy of it.
"""
if not pk_columns or pa_table.num_rows == 0:
return pa_table
@@ -1923,6 +1926,10 @@ def first_per_pk_table(pa_table: pa.Table, pk_columns: list[str], keep: Literal[
# 2. Group by PK, keeping only one position per PK
grouped = indexed.group_by(pk_columns).aggregate([(idx_col_name, aggregate)])
+ # One group per row keeps every row in its place, and `take` would only copy the whole table.
+ if grouped.num_rows == pa_table.num_rows:
+ return pa_table
+
# 3. Sort those positions ascending so the output mirrors the input row order
kept_indices = np.sort(grouped.column(f"{idx_col_name}_{aggregate}").to_numpy())
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_writer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_writer.py
index 4c481b16b144..c844849864c9 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_writer.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_writer.py
@@ -57,6 +57,7 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.del
_deltalite_write_stats,
_merge_predicate_ops,
commit_covers_batch,
+ dedupe_incremental_batch,
)
_REPARTITION_MODULE = "products.warehouse_sources.backend.temporal.data_imports.pipelines.core.repartition"
@@ -755,17 +756,83 @@ class TestIncrementalBatchDeduplication:
@parameterized.expand(
[
- ("keep_first", "first", ["a1", "b1"]),
- ("keep_last", "last", ["a2", "b1"]),
+ ("unique_key", [{"id": [1, 2, 3], "k2": ["a", "a", "a"]}], ["id"], "last", [0, 1, 2]),
+ ("unique_composite_key", [{"id": [1, 1, 2], "k2": ["a", "b", "a"]}], ["id", "k2"], "last", [0, 1, 2]),
+ (
+ "unique_across_chunks",
+ [{"id": [1, 2], "k2": ["a", "a"]}, {"id": [3], "k2": ["a"]}],
+ ["id"],
+ "last",
+ [0, 1, 2],
+ ),
+ ("null_in_one_composite_part", [{"id": [1, 1], "k2": [None, "a"]}], ["id", "k2"], "last", [0, 1]),
+ ("repeated_key_keeps_last", [{"id": [1, 2, 1, 3], "k2": ["a", "a", "a", "a"]}], ["id"], "last", [1, 2, 3]),
+ (
+ "repeated_key_keeps_first",
+ [{"id": [1, 2, 1, 3], "k2": ["a", "a", "a", "a"]}],
+ ["id"],
+ "first",
+ [0, 1, 3],
+ ),
+ ("repeated_composite_key", [{"id": [1, 1, 1], "k2": ["a", "b", "a"]}], ["id", "k2"], "last", [1, 2]),
+ (
+ "repeated_across_chunks",
+ [{"id": [1, 2], "k2": ["a", "a"]}, {"id": [2, 3], "k2": ["a", "a"]}],
+ ["id"],
+ "last",
+ [0, 2, 3],
+ ),
+ ("null_keys_are_one_key", [{"id": [None, 1, None], "k2": ["a", "a", "a"]}], ["id"], "last", [1, 2]),
+ ]
+ )
+ def test_first_per_pk_table(
+ self,
+ _name: str,
+ chunks: list[dict[str, list[Any]]],
+ keys: list[str],
+ keep: Literal["first", "last"],
+ kept_positions: list[int],
+ ) -> None:
+ table = pa.concat_tables(
+ [pa.table({**chunk, "pos": range(offset, offset + len(chunk["id"]))}) for offset, chunk in _offsets(chunks)]
+ )
+
+ result = first_per_pk_table(table, keys, keep=keep)
+
+ assert result.column("pos").to_pylist() == kept_positions
+ # A batch without a repeated key must come back as it is, not as a copy of every row.
+ assert (result is table) == (len(kept_positions) == table.num_rows)
+
+ @parameterized.expand([("unique_keys", 0), ("one_repeated_key", 1)])
+ def test_dedupe_holds_a_second_copy_only_when_a_key_repeats(self, _name: str, repeats: int) -> None:
+ ids = list(range(2_000)) + list(range(repeats))
+ table = pa.table({"id": ids, "payload": [f"{i:0>200}" for i in range(len(ids))]})
+ before = pa.total_allocated_bytes()
+
+ result = first_per_pk_table(table, ["id"], keep="last")
+
+ copied = pa.total_allocated_bytes() - before
+ assert result.num_rows == 2_000
+ if repeats:
+ assert copied > table.nbytes // 2
+ else:
+ assert copied < table.nbytes // 10
+
+ @parameterized.expand(
+ [
+ ("one_partition", ["p0", "p0", "p0", "p0"]),
+ ("key_moves_partition", ["p0", "p1", "p0", "p1"]),
+ ("every_row_own_partition", ["p0", "p1", "p2", "p3"]),
]
)
- def test_first_per_pk_table_keep_modes(self, _name, keep, expected_names):
- table = pa.table({"id": [1, 1, 2], "name": ["a1", "a2", "b1"]})
+ def test_partition_keyed_pass_keeps_the_rows_of_a_key_only_pass(self, _name: str, partitions: list[str]) -> None:
+ table = pa.table({"id": [1, 1, 2, 1], PARTITION_KEY: partitions, "pos": [0, 1, 2, 3]})
- result = first_per_pk_table(table, ["id"], keep=keep).sort_by("id")
+ key_only, _ = dedupe_incremental_batch(table, ["id"], use_partitioning=False)
+ partition_keyed, _ = dedupe_incremental_batch(table, ["id"], use_partitioning=True)
+ both, _ = dedupe_incremental_batch(partition_keyed, ["id"], use_partitioning=False)
- assert result.column("id").to_pylist() == [1, 2]
- assert result.column("name").to_pylist() == expected_names
+ assert both.column("pos").to_pylist() == key_only.column("pos").to_pylist() == [2, 3]
@pytest.mark.asyncio
async def test_incremental_merge_dedupes_duplicate_source_rows(self, tmp_path: Path) -> None:
@@ -808,6 +875,15 @@ class TestIncrementalBatchDeduplication:
assert final.column("name").to_pylist() == ["second_copy"]
+def _offsets(chunks: list[dict[str, list[Any]]]) -> list[tuple[int, dict[str, list[Any]]]]:
+ offsets: list[tuple[int, dict[str, list[Any]]]] = []
+ start = 0
+ for chunk in chunks:
+ offsets.append((start, chunk))
+ start += len(chunk["id"])
+ return offsets
+
+
def _commit_operations(delta_path: str) -> list[str]:
return [commit["operation"] for commit in deltalake.DeltaTable(delta_path).history()]
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/writer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/writer.py
index 16ef8b7523ab..39a2e9ad8cb7 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/writer.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/writer.py
@@ -178,6 +178,22 @@ def commit_matches(commit: dict[str, Any], match: dict[str, str]) -> bool:
return any(all(layout.get(k) == v for k, v in match.items()) for layout in _commit_metadata_layouts(commit))
+def dedupe_incremental_batch(
+ data: pa.Table, primary_keys: Sequence[Any], use_partitioning: bool
+) -> tuple[pa.Table, list[str]]:
+ """Drop all but the last occurrence of each PK (+ partition) tuple in a batch.
+
+ Returns the batch and the keys it was deduplicated on. The batch comes back as it is when no key
+ repeats, or when none of `primary_keys` is a column of it.
+ """
+ dedupe_keys = [n for x in primary_keys if (n := normalize_column_name(x)) in data.column_names]
+ if not dedupe_keys:
+ return data, []
+ if use_partitioning:
+ dedupe_keys.append(PARTITION_KEY)
+ return first_per_pk_table(data, dedupe_keys, keep="last"), dedupe_keys
+
+
def commit_members_tag(members: Sequence[tuple[str, int]]) -> str:
"""The `members` commit tag for a set that spans runs: `run_uuid:batch_index` per member."""
return ",".join(f"{run_uuid}:{batch_index}" for run_uuid, batch_index in members)
@@ -235,14 +251,7 @@ class DeltaWriter:
async def _dedupe_incremental_batch(
self, data: pa.Table, primary_keys: Sequence[Any], use_partitioning: bool
) -> pa.Table:
- """Drop all but the last occurrence of each PK (+ partition) tuple in a batch."""
- dedupe_keys = [n for x in primary_keys if (n := normalize_column_name(x)) in data.column_names]
- if not dedupe_keys:
- return data
- if use_partitioning:
- dedupe_keys.append(PARTITION_KEY)
-
- deduped = first_per_pk_table(data, dedupe_keys, keep="last")
+ deduped, dedupe_keys = dedupe_incremental_batch(data, primary_keys, use_partitioning)
dropped = data.num_rows - deduped.num_rows
if dropped > 0:
await self._logger.awarning(
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/coalescing.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/coalescing.py
index cec6757dae40..822acd563140 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/coalescing.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/coalescing.py
@@ -24,9 +24,9 @@ if TYPE_CHECKING:
COALESCABLE_SYNC_TYPES: frozenset[str] = frozenset({"incremental", "append", "full_refresh"})
# A full refresh starts over at batch 0 and appends after it, so its runs never share a write.
CROSS_RUN_SYNC_TYPES: frozenset[str] = frozenset({"incremental", "append"})
-# The loader holds a set decoded in Arrow, then copies it once more while it deduplicates and
-# partitions, so the memory a set takes is about twice its Arrow size.
-_ARROW_COPIES_HELD = 2
+# The loader holds a set decoded in Arrow through the upsert. Deduplication copies the rows only when
+# a key repeats, and the loader frees the set as read before the upsert, so one copy is held.
+_ARROW_COPIES_HELD = 1
_MB = 1024 * 1024
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/processor.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/processor.py
index 6f56339a63cf..d3d341c74863 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/processor.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/processor.py
@@ -61,6 +61,7 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.del
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.writer import (
DeltaWriter,
commit_members_tag,
+ dedupe_incremental_batch,
)
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.hogql_schema import HogQLSchema
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.partitioning import (
@@ -369,6 +370,34 @@ def _apply_partitioning(
return pa_table
+def _dedupe_before_write(pa_table: pa.Table, primary_keys: list[str]) -> pa.Table:
+ """Drop the rows that `DeltaWriter.write` would drop, before the batch goes to the writer.
+
+ The writer deduplicates too, but its caller holds the batch until the write returns. If only
+ the writer deduplicated, the batch as read and its deduplicated copy would both stay in memory
+ for the whole upsert. The caller rebinds its batch to this result, so the batch as read is freed
+ first, and a set holds one Arrow copy of its rows through the write (see `_ARROW_COPIES_HELD`).
+
+ This keys on the partition column whenever the batch has one. The writer keys on it only when
+ the table is partitioned by it, so these keys are never fewer than the keys of the writer. The
+ last row of each writer key is also the last row of its key here, so the rows that land do not
+ change. When both use the same keys, the pass in the writer finds no repeated key and makes no
+ copy.
+ """
+ deduped, dedupe_keys = dedupe_incremental_batch(
+ pa_table, primary_keys, use_partitioning=PARTITION_KEY in pa_table.column_names
+ )
+ dropped = pa_table.num_rows - deduped.num_rows
+ if dropped > 0:
+ logger.warning(
+ "dropped_duplicate_primary_key_rows",
+ dropped=dropped,
+ dedupe_keys=dedupe_keys,
+ batch_rows=pa_table.num_rows,
+ )
+ return deduped
+
+
def _partial_data_loading_applies(export_signal: ExportSignalMessage, schema: ExternalDataSchema) -> bool:
"""Whether this batch feeds partial data loading (first-ever sync, Stripe only).
@@ -1537,6 +1566,9 @@ def _process_message_reported(
with timer.step("ownership_check"):
verify_ownership()
+ # The row metrics count the batch as read, before the dedupe below drops repeated keys.
+ source_rows = pa_table.num_rows
+
# The writer narrows this to `merge` for an upsert; an append stays `write` to its commit.
report_phase("write")
if cdc_write_mode == "scd2_append":
@@ -1586,6 +1618,8 @@ def _process_message_reported(
timer.step("write"),
DELTA_WRITE_DURATION_SECONDS.labels(write_type=write_type).time(),
):
+ if write_type == "incremental" and primary_keys:
+ pa_table = _dedupe_before_write(pa_table, primary_keys)
delta_writer = DeltaWriter(delta_table_ref)
delta_table = async_to_sync(delta_writer.write)(
data=pa_table,
@@ -1597,7 +1631,7 @@ def _process_message_reported(
)
deltalite_file_count_change = delta_writer.deltalite_file_count_change
- DELTA_ROWS_WRITTEN_TOTAL.inc(pa_table.num_rows)
+ DELTA_ROWS_WRITTEN_TOTAL.inc(source_rows)
# Marked as soon as the commit lands, before post-load: the final row of a run carries its own
# data now, so a post-load failure must send the retry down the post-load-only path rather
@@ -1640,7 +1674,7 @@ def _process_message_reported(
team_id=export_signal.team_id,
external_data_schema_id=export_signal.schema_id,
batch_index=export_signal.batch_index,
- rows_written=pa_table.num_rows,
+ rows_written=source_rows,
batch_count=len(members),
delta_version=delta_table_ref.latest_known_version(delta_table),
file_count=file_count,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/test_processor.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/test_processor.py
index 3793c93c237d..dfa47674020b 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/test_processor.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/test_processor.py
@@ -40,6 +40,7 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline
INCOMPLETE_RUN_ERROR,
IncompleteRunError,
_apply_partitioning,
+ _dedupe_before_write,
_enrich_cdc_rows,
_file_count_after_write,
_get_write_type,
@@ -1079,7 +1080,10 @@ class TestProcessMessages:
@patch(f"{_PROCESSOR}._trigger_ducklake_register_data_imports")
@patch(f"{_PROCESSOR}._mark_job_completed")
@patch(f"{_PROCESSOR}.run_post_load_operations", new_callable=AsyncMock, return_value=_POST_LOAD_RESULT)
- @patch(f"{_PROCESSOR}.read_parquet", return_value=pa.table({"id": [1]}))
+ @patch(
+ f"{_PROCESSOR}.read_parquet",
+ side_effect=lambda path: pa.table({"id": [int(path.rsplit("/", 1)[1].removesuffix(".parquet"))]}),
+ )
@patch(f"{_PROCESSOR}.is_batch_already_processed", return_value=False)
@patch(f"{_PROCESSOR}.DeltaWriter")
@patch(f"{_PROCESSOR}.DeltaTableRef")
@@ -1221,6 +1225,86 @@ class TestCrossRunKeyResolution:
assert not commit_covers_batch(set_commit, "run-2", 1)
+class TestSourceCopiesHeldThroughTheUpsert:
+ # Coalesced sets are sized by `_ARROW_COPIES_HELD`. If the loader keeps the set as read alive
+ # next to its deduplicated copy, a set takes twice the memory its cap allows.
+
+ _ROWS = 2_000
+
+ @staticmethod
+ def _batch(batch_no: int, ids: list[int]) -> pa.Table:
+ return pa.table({"id": ids, "payload": [f"{batch_no}:{pos:0>200}" for pos in range(len(ids))]})
+
+ @pytest.mark.parametrize(
+ "batch_ids",
+ [
+ [list(range(0, _ROWS)), list(range(_ROWS, 2 * _ROWS))],
+ [[*range(0, _ROWS), 0, 1], list(range(_ROWS, 2 * _ROWS))],
+ [list(range(0, _ROWS)), list(range(_ROWS // 2, _ROWS + _ROWS // 2))],
+ ],
+ ids=["unique_keys", "key_repeated_in_one_batch", "keys_repeated_across_the_set"],
+ )
+ def test_the_upsert_gets_one_copy_of_the_set_with_the_last_row_of_each_key(
+ self, batch_ids: list[list[int]], tmp_path: Path
+ ) -> None:
+ table_path = str(tmp_path / "table")
+ write_deltalake(table_path, self._batch(9, [0]))
+ paths = [f"s3://bucket/run-1/{3 + n}.parquet" for n in range(len(batch_ids))]
+ expected = pa.concat_tables([self._batch(n, ids) for n, ids in enumerate(batch_ids)]).to_pylist()
+ last_row_by_id = {row["id"]: row for row in expected}
+ expected_rows = sorted(last_row_by_id.values(), key=lambda row: expected.index(row))
+ one_copy_bytes = pa.concat_tables([self._batch(n, ids) for n, ids in enumerate(batch_ids)]).nbytes
+ seen: dict[str, Any] = {}
+
+ async def record_upsert_source(writer: DeltaWriter, **kwargs: Any) -> bool:
+ seen["held_bytes"] = pa.total_allocated_bytes() - baseline
+ seen["rows"] = kwargs["data"].select(["id", "payload"]).to_pylist()
+ return False
+
+ with ExitStack() as stack:
+ stack.enter_context(patch(f"{_PROCESSOR}.posthoganalytics"))
+ stack.enter_context(patch(f"{_PROCESSOR}.mark_batch_as_processed"))
+ stack.enter_context(patch(f"{_PROCESSOR}.is_batch_already_processed", return_value=False))
+ stack.enter_context(
+ patch(
+ f"{_PROCESSOR}.read_parquet",
+ side_effect=lambda path: self._batch(paths.index(path), batch_ids[paths.index(path)]),
+ )
+ )
+ stack.enter_context(
+ patch(f"{_PROCESSOR}.DeltaTableRef", side_effect=lambda **kwargs: make_local_table_ref(table_path))
+ )
+ stack.enter_context(patch.object(DeltaWriter, "_write_via_deltalite", record_upsert_source))
+ job_model = stack.enter_context(patch(f"{_PROCESSOR}.ExternalDataJob"))
+ job_model.objects.prefetch_related.return_value.get.return_value = MagicMock()
+ baseline = pa.total_allocated_bytes()
+ process_messages(_run_messages("run-1", start=3, count=len(batch_ids), final=False))
+
+ assert seen["rows"] == expected_rows
+ assert seen["held_bytes"] < one_copy_bytes * 1.5
+ landed = DeltaTable(table_path).to_pyarrow_table().select(["id", "payload"]).sort_by("id").to_pylist()
+ assert landed == sorted(expected_rows, key=lambda row: row["id"])
+
+ @parameterized.expand(
+ [
+ ("no_partition_column", None, [1, 2]),
+ ("key_in_one_partition", ["p0", "p0", "p0"], [1, 2]),
+ # The writer keys on the partition too, so a key that moved partition lands in both.
+ ("key_moved_partition", ["p0", "p0", "p1"], [0, 1, 2]),
+ ]
+ )
+ def test_dedupe_before_write_keeps_what_the_writer_keeps(
+ self, _name: str, partitions: list[str] | None, kept_positions: list[int]
+ ) -> None:
+ columns: dict[str, list[Any]] = {"id": [1, 2, 1], "pos": [0, 1, 2]}
+ if partitions is not None:
+ columns[PARTITION_KEY] = partitions
+
+ result = _dedupe_before_write(pa.table(columns), ["id"])
+
+ assert result.column("pos").to_pylist() == kept_positions
+
+
class TestPostImportTrigger:
"""The V3 hand-off to `data-import-post-import`: without it the load-dependent
post-import steps (signals, enrichment, statistics, table size, DuckLake copy)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_consumer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_consumer.py
index ee27058a656e..607df3eacc52 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_consumer.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_consumer.py
@@ -5032,20 +5032,20 @@ class TestCoalesceGroup:
@pytest.mark.parametrize(
"slot_mb,sizes,expected",
[
- # A 64 MiB slice at share 1.0 leaves 32 MiB of Arrow for the two copies the loader holds.
- (64, [(10 * _MIB, 1)] * 5, [[0, 1, 2], [3, 4]]),
+ # A 32 MiB slice at share 1.0 leaves 32 MiB of Arrow for the one copy the loader holds.
+ (32, [(10 * _MIB, 1)] * 5, [[0, 1, 2], [3, 4]]),
# Wide rows that compress badly join past the parquet cap while they fit the memory budget.
- (64, [(10 * _MIB, 30 * _MIB)] * 4, [[0, 1, 2], [3]]),
+ (32, [(10 * _MIB, 30 * _MIB)] * 4, [[0, 1, 2], [3]]),
# A batch bigger than the whole budget still loads, on its own.
- (64, [(1 * _MIB, 1), (100 * _MIB, 1), (1 * _MIB, 1)], [[0], [1], [2]]),
+ (32, [(1 * _MIB, 1), (100 * _MIB, 1), (1 * _MIB, 1)], [[0], [1], [2]]),
# A large slice is still bounded by the hard ceiling.
(100_000, [(10 * _MIB, 1)] * 5, [[0, 1, 2, 3], [4]]),
# Narrow batches still coalesce until the row cap.
- (64, [(1024, 1)] * 60, [list(range(50)), list(range(50, 60))]),
+ (32, [(1024, 1)] * 60, [list(range(50)), list(range(50, 60))]),
# One batch queued before the producer recorded its size puts the set back on the parquet cap.
- (64, [(10 * _MIB, 30 * _MIB), (None, 30 * _MIB), (10 * _MIB, 30 * _MIB)], [[0, 1], [2]]),
+ (32, [(10 * _MIB, 30 * _MIB), (None, 30 * _MIB), (10 * _MIB, 30 * _MIB)], [[0, 1], [2]]),
# The metadata is free-form JSON, so any other shape counts as unknown.
- (64, [("10", 30 * _MIB), (True, 30 * _MIB), (-1, 30 * _MIB)], [[0, 1], [2]]),
+ (32, [("10", 30 * _MIB), (True, 30 * _MIB), (-1, 30 * _MIB)], [[0, 1], [2]]),
# An unreadable pod limit gives no slice to size against.
(None, [(10 * _MIB, 30 * _MIB)] * 3, [[0, 1], [2]]),
# A slice that leaves no memory gives no Arrow budget.