Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(batch-exports): write parquet files with larger row groups (#119655)

Rossd7d698f0e540
diff --git a/posthog/settings/batch_exports.py b/posthog/settings/batch_exports.py
index ed9ad4c9ea9f..e8afe0d5472a 100644
--- a/posthog/settings/batch_exports.py
+++ b/posthog/settings/batch_exports.py
@@ -116,6 +116,10 @@ BATCH_EXPORTS_CLICKHOUSE_MAX_INSERT_BLOCK_SIZE_BYTES: int = get_from_env(
"BATCH_EXPORTS_CLICKHOUSE_MAX_INSERT_BLOCK_SIZE_BYTES", 64 * 1024 * 1024, type_cast=int
)
+BATCH_EXPORT_PARQUET_MAX_BUFFERED_ARROW_BYTES: int = get_from_env(
+ "BATCH_EXPORT_PARQUET_MAX_BUFFERED_ARROW_BYTES", 32 * 1024 * 1024, type_cast=int
+)
+
BATCH_EXPORT_OBJECT_STORAGE_ENDPOINT: str = os.getenv(
"BATCH_EXPORT_OBJECT_STORAGE_ENDPOINT", "http://objectstorage:19000"
)
diff --git a/posthog/temporal/common/asyncpa.py b/posthog/temporal/common/asyncpa.py
index fe1856abb287..f5e3a559dfaa 100644
--- a/posthog/temporal/common/asyncpa.py
+++ b/posthog/temporal/common/asyncpa.py
@@ -74,11 +74,15 @@ class AsyncMessageReader:
total_message_size = 8 + metadata_size + body_size
await self.read_until(total_message_size)
+ # A message parsed from a view of the buffer keeps the whole buffer alive, including bytes of later
+ # messages. Parse a copy of the message instead, so a held record batch only keeps its own bytes.
with memoryview(self._buffer) as buffer_view:
- loop = asyncio.get_running_loop()
- msg = await loop.run_in_executor(None, pa.ipc.read_message, buffer_view[:total_message_size])
+ message_bytes = bytes(buffer_view[:total_message_size])
+ del self._buffer[:total_message_size]
+
+ loop = asyncio.get_running_loop()
+ msg = await loop.run_in_executor(None, pa.ipc.read_message, message_bytes)
- self._buffer = self._buffer[total_message_size:]
self._bytes_consumed += total_message_size
return msg
diff --git a/posthog/temporal/tests/common/test_asyncpa.py b/posthog/temporal/tests/common/test_asyncpa.py
index 663db9f95a59..1c3ceda3c14f 100644
--- a/posthog/temporal/tests/common/test_asyncpa.py
+++ b/posthog/temporal/tests/common/test_asyncpa.py
@@ -1,7 +1,9 @@
+import gc
import io
import random
import string
import asyncio
+import tracemalloc
import collections.abc
import pytest
@@ -149,3 +151,25 @@ async def test_message_reader_stops_waiting_for_a_stalled_error_tail(monkeypatch
await asyncio.wait_for(reader.read_next_message(), timeout=5)
assert exc_info.value.unparsed == b"Code: 241. DB::Exception: an error"
+
+
+async def test_record_batch_reader_does_not_keep_read_buffers_alive():
+ batches = list(generate_record_batches(total_records=3_000, total_batches=300, target_size_bytes=100))
+
+ buffer = io.BytesIO()
+ with pa.ipc.new_stream(buffer, schema=batches[0].schema) as writer:
+ for batch in batches:
+ writer.write_batch(batch)
+ _ = buffer.seek(0)
+
+ gc.collect()
+ tracemalloc.start()
+ try:
+ held_batches = [batch async for batch in asyncpa.AsyncRecordBatchReader(AsyncWrapper(buffer, 128 * 1024))]
+ gc.collect()
+ held_bytes, _ = tracemalloc.get_traced_memory()
+ finally:
+ tracemalloc.stop()
+
+ assert held_batches == batches
+ assert held_bytes < 2 * sum(batch.nbytes for batch in held_batches)
diff --git a/products/batch_exports/backend/temporal/pipeline/transformer.py b/products/batch_exports/backend/temporal/pipeline/transformer.py
index 5d436ea23320..3347543ee31d 100644
--- a/products/batch_exports/backend/temporal/pipeline/transformer.py
+++ b/products/batch_exports/backend/temporal/pipeline/transformer.py
@@ -387,8 +387,29 @@ def replace_broken_unicode(obj):
return obj
+def get_parquet_column_paths(schema: pa.Schema, field_names: collections.abc.Iterable[str]) -> list[str]:
+ """Return the Parquet leaf column paths of the fields named `field_names` in `schema`.
+
+ `ParquetWriter` matches `use_dictionary` and `write_statistics` against leaf column paths, such as
+ `tags.list.element` for a list field, not against top-level field names.
+ """
+ buffer = io.BytesIO()
+ pq.write_table(pa.schema([schema.field(name) for name in field_names]).empty_table(), buffer)
+ parquet_schema = pq.ParquetFile(buffer).schema
+ return [parquet_schema.column(index).path for index in range(len(parquet_schema))]
+
+
class ParquetStreamTransformer:
- """A transformer to convert record batches into Parquet."""
+ """A transformer to convert record batches into Parquet.
+
+ Record batches are held until `max_buffered_arrow_bytes` of them are pending, then written as one
+ row group. The limit counts Arrow bytes, so the row group in the file is usually smaller.
+ """
+
+ # Close a file when less than this fraction of its max size is left. Otherwise, the last row
+ # group ends up being very small and we're also more likely to go over the max file size.
+ MIN_REMAINING_FILE_SIZE_FRACTION = 0.05
+ JSON_COLUMNS = frozenset({"properties", "person_properties", "set", "set_once"})
def __init__(
self,
@@ -396,70 +417,89 @@ class ParquetStreamTransformer:
compression_level: int | None = None,
include_inserted_at: bool = False,
max_file_size_bytes: int = 0,
+ max_buffered_arrow_bytes: int | None = None,
+ record_metrics: bool = True,
):
self.include_inserted_at = include_inserted_at
self.compression = compression
self.compression_level = compression_level
self.max_file_size_bytes = max_file_size_bytes
+ self.record_metrics = record_metrics
+
+ if max_buffered_arrow_bytes is None:
+ max_buffered_arrow_bytes = settings.BATCH_EXPORT_PARQUET_MAX_BUFFERED_ARROW_BYTES
+ if max_file_size_bytes:
+ max_buffered_arrow_bytes = min(max_buffered_arrow_bytes, max_file_size_bytes)
+ self.max_buffered_arrow_bytes = max_buffered_arrow_bytes
- # For Parquet, we need to handle schema and batching
self._parquet_writer: pq.ParquetWriter | None = None
self._parquet_buffer = io.BytesIO()
self._schema: pa.Schema | None = None
+ self._pending_record_batches: list[pa.RecordBatch] = []
+ self._pending_record_batches_bytes = 0
+ self._pending_record_batches_rows = 0
+ self._non_json_column_paths: list[str] = []
+ self._current_file_size = 0
+ # The compression ratio (Parquet bytes written per Arrow byte) achieved when writing the
+ # previous row group. We use this as an estimate for calculating how many bytes will be
+ # written next time. It starts at 1 because Parquet output is usually no larger than its
+ # Arrow input.
+ self._compression_ratio = 1.0
async def iter(
self, record_batches: collections.abc.AsyncIterable[pa.RecordBatch]
) -> collections.abc.AsyncIterator[Chunk]:
"""Iterate over record batches transforming them into chunks."""
- current_file_size = 0
-
async for record_batch in record_batches:
self.schema = record_batch.schema
+ self._add_pending_record_batch(record_batch)
- with ExecutionTimeRecorder(
- "parquet_stream_transformer_record_batch_transform_duration",
- description="Duration to transform a record batch into Parquet bytes.",
- log_message=(
- "Processed record batch with %(num_records)d records to parquet."
- " Record batch size: %(mb_processed).2f MB, process time:"
- " %(duration_seconds)d seconds, speed: %(mb_per_second).2f MB/s"
- ),
- log_attributes={"num_records": record_batch.num_rows},
- ) as recorder:
- recorder.add_bytes_processed(record_batch.nbytes)
- # Running write in a thread to yield control back to event loop.
- chunk = await asyncio.to_thread(self.write_record_batch, record_batch)
+ if not self._is_row_group_ready():
+ continue
- yield Chunk(chunk, False)
+ if self.record_metrics:
+ with ExecutionTimeRecorder(
+ "parquet_stream_transformer_record_batch_transform_duration",
+ description="Duration to write pending record batches as a Parquet row group.",
+ log_message=(
+ "Wrote row group with %(num_records)d records to parquet."
+ " Record batches size: %(mb_processed).2f MB, process time:"
+ " %(duration_seconds)d seconds, speed: %(mb_per_second).2f MB/s"
+ ),
+ log_attributes={"num_records": self._pending_record_batches_rows},
+ ) as recorder:
+ recorder.add_bytes_processed(self._pending_record_batches_bytes)
+ # Running write in a thread to yield control back to event loop.
+ chunk = await asyncio.to_thread(self._flush_row_group)
+ else:
+ chunk = await asyncio.to_thread(self._flush_row_group)
- if self.max_file_size_bytes and current_file_size + len(chunk) > self.max_file_size_bytes:
- footer = await asyncio.to_thread(self.finish_parquet_file)
+ yield Chunk(chunk, False)
+ self._current_file_size += len(chunk)
- yield Chunk(footer, True)
- current_file_size = 0
+ if self._is_current_file_full():
+ footer = await asyncio.to_thread(self._finish_parquet_file)
- else:
- current_file_size += len(chunk)
+ yield Chunk(footer, True)
- if not self._schema:
+ # Nothing is left to finish when no record batches arrived, or when the last file was finished in the loop.
+ if self._current_file_size == 0 and not self._pending_record_batches:
return
- footer = await asyncio.to_thread(self.finish_parquet_file)
+ footer = await asyncio.to_thread(self._finish_parquet_file)
yield Chunk(footer, True)
- @property
- def parquet_writer(self) -> pq.ParquetWriter:
+ def _get_parquet_writer(self) -> pq.ParquetWriter:
if self._parquet_writer is None:
- json_columns = {"properties", "person_properties", "set", "set_once"}
- # self.schema is set before getting a ParquetWriter.
- dicitionary_encoded_columns = [col for col in self.schema.names if col not in json_columns]
-
self._parquet_writer = pq.ParquetWriter(
self._parquet_buffer,
schema=self.schema,
compression="none" if self.compression is None else self.compression, # type: ignore
compression_level=self.compression_level,
- use_dictionary=dicitionary_encoded_columns, # type: ignore
+ use_dictionary=self._non_json_column_paths, # type: ignore
+ # Min/max statistics on JSON strings cannot help a reader skip row groups, so we do
+ # not write statistics for them to save on file size.
+ write_statistics=self._non_json_column_paths,
)
assert self._parquet_writer is not None
return self._parquet_writer
@@ -480,29 +520,75 @@ class ParquetStreamTransformer:
schema = schema.remove(index)
self._schema = schema
+ self._non_json_column_paths = get_parquet_column_paths(
+ schema, [name for name in schema.names if name not in self.JSON_COLUMNS]
+ )
- def finish_parquet_file(self) -> bytes:
- """Ensure underlying Parquet writer is closed before flushing buffer."""
- self.parquet_writer.close()
+ def _finish_parquet_file(self) -> bytes:
+ """Write any pending row group, close the file, and return the rest of its bytes."""
+ self._write_pending_row_group()
+ self._get_parquet_writer().close()
self._parquet_writer = None
+ self._current_file_size = 0
- final_data = self._parquet_buffer.getvalue()
+ return self._drain_parquet_buffer()
- self._parquet_buffer = io.BytesIO()
+ def _is_current_file_full(self) -> bool:
+ """Return whether the current file has too little space left for another row group."""
+ if not self.max_file_size_bytes:
+ return False
- return final_data
+ remaining_bytes = self.max_file_size_bytes - self._current_file_size
+ return remaining_bytes < self.max_file_size_bytes * self.MIN_REMAINING_FILE_SIZE_FRACTION
- def write_record_batch(self, record_batch: pa.RecordBatch) -> bytes:
- """Write record batch to buffer as Parquet."""
- column_names = self.schema.names
- self.parquet_writer.write_batch(record_batch.select(column_names))
- data = self._parquet_buffer.getvalue()
+ def _is_row_group_ready(self) -> bool:
+ """Return whether enough record batches are pending to write a row group."""
+ if not self.max_file_size_bytes:
+ return self._pending_record_batches_bytes >= self.max_buffered_arrow_bytes
+
+ # Near the end of a file, the limit is the space left in the file converted to Arrow bytes
+ # using our compression ratio estimate.
+ remaining_bytes = self.max_file_size_bytes - self._current_file_size
+ limit_bytes = min(self.max_buffered_arrow_bytes, remaining_bytes / self._compression_ratio)
+ return self._pending_record_batches_bytes >= limit_bytes
+ def _add_pending_record_batch(self, record_batch: pa.RecordBatch) -> None:
+ """Hold a record batch until enough record batches are pending to write a row group."""
+ record_batch = record_batch.select(self.schema.names)
+ self._pending_record_batches.append(record_batch)
+ self._pending_record_batches_bytes += record_batch.nbytes
+ self._pending_record_batches_rows += record_batch.num_rows
+
+ def _flush_row_group(self) -> bytes:
+ """Write all pending record batches as one row group, and return the bytes written."""
+ pending_record_batches_bytes = self._pending_record_batches_bytes
+ self._write_pending_row_group()
+ data = self._drain_parquet_buffer()
+
+ if pending_record_batches_bytes:
+ self._compression_ratio = len(data) / pending_record_batches_bytes
+
+ return data
+
+ def _drain_parquet_buffer(self) -> bytes:
+ """Return the bytes in the buffer, and empty it for the next write."""
+ data = self._parquet_buffer.getvalue()
self._parquet_buffer.seek(0)
self._parquet_buffer.truncate(0)
-
return data
+ def _write_pending_row_group(self) -> None:
+ """Write all pending record batches to buffer as a single Parquet row group."""
+ if not self._pending_record_batches:
+ return
+
+ table = pa.Table.from_batches(self._pending_record_batches, schema=self.schema)
+ self._get_parquet_writer().write_table(table, row_group_size=max(table.num_rows, 1))
+
+ self._pending_record_batches = []
+ self._pending_record_batches_bytes = 0
+ self._pending_record_batches_rows = 0
+
class RedshiftQueryStreamTransformer:
"""A transformer to convert record batches into a Redshift INSERT INTO query."""
diff --git a/products/batch_exports/backend/tests/temporal/pipeline/test_transformer.py b/products/batch_exports/backend/tests/temporal/pipeline/test_transformer.py
index 164f72e46d5d..0f3bb9e8655b 100644
--- a/products/batch_exports/backend/tests/temporal/pipeline/test_transformer.py
+++ b/products/batch_exports/backend/tests/temporal/pipeline/test_transformer.py
@@ -1,13 +1,17 @@
import io
import csv
import json
+import random
import typing
+import hashlib
import datetime as dt
import functools
+import collections.abc
import pytest
import pyarrow as pa
+import pyarrow.parquet as pq
from products.batch_exports.backend.temporal.pipeline.table import (
Field,
@@ -18,6 +22,7 @@ from products.batch_exports.backend.temporal.pipeline.table import (
from products.batch_exports.backend.temporal.pipeline.transformer import (
CSVStreamTransformer,
JSONLStreamTransformer,
+ ParquetStreamTransformer,
PipelineTransformer,
SchemaTransformer,
_ensure_curly_brackets_array,
@@ -149,6 +154,125 @@ async def test_csv_stream_transformer_writes_record_batches():
assert row == expected_row
+async def iterate_record_batches(
+ record_batches: list[pa.RecordBatch],
+) -> collections.abc.AsyncIterator[pa.RecordBatch]:
+ for record_batch in record_batches:
+ yield record_batch
+
+
+async def collect_parquet_files(
+ transformer: ParquetStreamTransformer, record_batches: list[pa.RecordBatch]
+) -> list[bytes]:
+ files: list[bytes] = []
+ current_file = b""
+ async for chunk in transformer.iter(iterate_record_batches(record_batches)):
+ current_file += chunk.data
+ if chunk.is_eof:
+ files.append(current_file)
+ current_file = b""
+ return files
+
+
+ "max_file_size_bytes,compression",
+ [(0, None), (256 * 1024, None), (256 * 1024, "zstd")],
+)
+async def test_parquet_stream_transformer_writes_small_record_batches_as_large_row_groups(
+ max_file_size_bytes, compression
+):
+ records = [
+ {
+ "uuid": f"uuid-{index}",
+ "properties": json.dumps(
+ {
+ "index": index,
+ "padding": "".join(hashlib.sha256(f"{index}-{n}".encode()).hexdigest() for n in range(16)),
+ }
+ ),
+ }
+ for index in range(1000)
+ ]
+ record_batches = pa.Table.from_pylist(records).to_batches(max_chunksize=10)
+
+ transformer = ParquetStreamTransformer(
+ compression=compression, max_file_size_bytes=max_file_size_bytes, max_buffered_arrow_bytes=64 * 1024
+ )
+ files = await collect_parquet_files(transformer, record_batches)
+
+ parquet_files = [pq.ParquetFile(io.BytesIO(file)) for file in files]
+ assert all(parquet_file.metadata.num_rows > 0 for parquet_file in parquet_files)
+ # Writing each record batch as its own row group would give one row group per batch. The exact count
+ # depends on where files split and on the learned compression ratio, so only check that batches are combined.
+ assert sum(parquet_file.metadata.num_row_groups for parquet_file in parquet_files) < len(record_batches) / 2
+ assert [row for parquet_file in parquet_files for row in parquet_file.read().to_pylist()] == records
+ # A file closes when less than 5% of its max size is left, and can go over by the error in the compression
+ # ratio, so full files land near the max but not on it. The last file holds the remaining rows, so it can be small.
+ if max_file_size_bytes:
+ assert len(files) > 1
+ assert all(0.9 * max_file_size_bytes <= len(file) <= 1.1 * max_file_size_bytes for file in files[:-1])
+
+
+async def test_parquet_stream_transformer_bounds_file_size_when_compression_gets_worse():
+ max_file_size_bytes = 1024 * 1024
+ rng = random.Random(0)
+ compressible = [
+ pa.RecordBatch.from_pydict({"properties": ['{"a": "' + "x" * 10_000 + '"}'] * 20}) for _ in range(30)
+ ]
+ incompressible = [
+ pa.RecordBatch.from_pydict({"properties": [rng.randbytes(5_000).hex() for _ in range(20)]}) for _ in range(40)
+ ]
+
+ transformer = ParquetStreamTransformer(compression="zstd", max_file_size_bytes=max_file_size_bytes)
+ files = await collect_parquet_files(transformer, compressible + incompressible)
+
+ assert max(len(file) for file in files) <= 2 * max_file_size_bytes
+
+
+async def test_parquet_stream_transformer_skips_statistics_and_dictionaries_only_for_json_columns():
+ record_batch = pa.RecordBatch.from_pylist(
+ [
+ {"uuid": "a", "tags": ["x", "y"], "person": {"name": "a"}, "properties": '{"a": 1}'},
+ {"uuid": "b", "tags": ["z"], "person": {"name": "b"}, "properties": '{"b": 2}'},
+ ]
+ )
+
+ transformer = ParquetStreamTransformer()
+ (data,) = await collect_parquet_files(transformer, [record_batch])
+
+ row_group = pq.ParquetFile(io.BytesIO(data)).metadata.row_group(0)
+ columns = [row_group.column(index) for index in range(row_group.num_columns)]
+ assert {
+ column.path_in_schema: column.statistics is not None and column.statistics.has_min_max for column in columns
+ } == {
+ "uuid": True,
+ "tags.list.element": True,
+ "person.name": True,
+ "properties": False,
+ }
+ assert {column.path_in_schema: "RLE_DICTIONARY" in column.encodings for column in columns} == {
+ "uuid": True,
+ "tags.list.element": True,
+ "person.name": True,
+ "properties": False,
+ }
+
+
+async def test_parquet_stream_transformer_without_metrics_runs_outside_temporal(settings):
+ # get_metric_meter only raises outside Temporal when DEBUG and TEST are both off.
+ settings.DEBUG = False
+ settings.TEST = False
+
+ transformer = ParquetStreamTransformer(record_metrics=False, max_buffered_arrow_bytes=1)
+ (data,) = await collect_parquet_files(transformer, [pa.RecordBatch.from_pylist([{"uuid": "a"}])])
+
+ assert pq.ParquetFile(io.BytesIO(data)).metadata.num_rows == 1
+
+
class FakeField(Field):
def __init__(self, name: str, data_type: pa.DataType):
self.name = name
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/azure_blob.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/azure_blob.py
index 9bd3b5b2ff70..41426d0d3e19 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/azure_blob.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/azure_blob.py
@@ -25,10 +25,9 @@ Connection handling comes from batch exports' Azure Blob destination:
`AzureBlobConsumer` itself is not used, even though it touches no Temporal metric meter and so
would run outside an activity. Its `consume_chunk` accumulates the whole file in a `bytearray`
before uploading, which is the memory behavior this writer exists to avoid. Instead the parquet
-is streamed a record batch at a time through `ParquetStreamTransformer`, whose
-`write_record_batch` and `finish_parquet_file` are pure, and handed to `upload_blob` as an
-async iterator. The SDK reads it in `max_block_size` blocks, so peak memory is one record batch
-plus a few blocks rather than ~200 MiB of Arrow and two copies of its parquet.
+is streamed a row group at a time through `ParquetStreamTransformer.iter`, and handed to
+`upload_blob` as an async iterator. The SDK reads it in `max_block_size` blocks, so peak memory
+is one row group plus a few blocks rather than ~200 MiB of Arrow and two copies of its parquet.
"""
from __future__ import annotations
@@ -111,29 +110,24 @@ class _WriteStats:
async def _parquet_byte_stream(
batches: AsyncIterator[pa.RecordBatch], compression: str | None, stats: _WriteStats
) -> AsyncIterator[bytes]:
- """Yield the parquet encoding of `batches`, one record batch at a time.
+ """Yield the parquet encoding of `batches` as one file, one row group at a time.
- `write_record_batch` returns only the bytes the writer produced for that batch and resets
- its buffer, so nothing accumulates. The pyarrow write runs in a thread to keep the event
- loop free for the upload it is feeding.
+ The transformer holds record batches until they fill a row group, so at most one row group
+ of Arrow data is in memory. A batch that later grows a column is projected back onto the
+ first schema, because one parquet file has one schema.
"""
- transformer = ParquetStreamTransformer(compression=compression, include_inserted_at=True)
+ # Delivery runs outside a Temporal activity, where there is no metric meter to record to.
+ transformer = ParquetStreamTransformer(compression=compression, include_inserted_at=True, record_metrics=False)
+
+ async for chunk in transformer.iter(_count_rows(batches, stats)):
+ stats.parquet_bytes += len(chunk.data)
+ yield chunk.data
- async for record_batch in batches:
- # No-op after the first batch. A batch that later grows a column is projected back onto
- # the first schema, because one parquet file has one schema.
- transformer.schema = record_batch.schema
- chunk = await asyncio.to_thread(transformer.write_record_batch, record_batch)
+async def _count_rows(batches: AsyncIterator[pa.RecordBatch], stats: _WriteStats) -> AsyncIterator[pa.RecordBatch]:
+ async for record_batch in batches:
stats.rows += record_batch.num_rows
- stats.parquet_bytes += len(chunk)
- if chunk:
- yield chunk
-
- footer = await asyncio.to_thread(transformer.finish_parquet_file)
- stats.parquet_bytes += len(footer)
- if footer:
- yield footer
+ yield record_batch
async def _chain(head: pa.RecordBatch, rest: AsyncIterator[pa.RecordBatch]) -> AsyncIterator[pa.RecordBatch]:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/s3.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/s3.py
index cce0706d86b8..82ba3ef1e574 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/s3.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/s3.py
@@ -22,14 +22,14 @@ Credentials, the client and its config come from batch exports' S3 destination:
roles, and `request_checksum_calculation="when_required"`. That last one is why R2, GCS and
MinIO accept our uploads at all: they reject AWS's newer checksum headers.
-What is not taken from there is the upload itself. `ConcurrentS3Consumer` times every part
-through `ExecutionTimeRecorder`, which resolves a Temporal metric meter and raises outside an
-activity. Delivery runs in the Postgres-queue load worker, not in an activity, so the parts are
-uploaded here instead, keeping their `RequestTimeout` retry behavior.
-
-Bytes are streamed a record batch at a time through `ParquetStreamTransformer`, whose
-`write_record_batch` and `finish_parquet_file` are pure. Collecting the whole staged batch
-first would hold ~200 MiB of Arrow plus two copies of its parquet, per destination.
+What is not taken from there is the upload itself. `ConcurrentS3Consumer` counts exported rows
+and bytes through Temporal activity metrics, which raise outside an activity. Delivery runs in the
+Postgres-queue load worker, not in an activity, so the parts are uploaded here instead, keeping
+their `RequestTimeout` retry behavior.
+
+Bytes are streamed a row group at a time through `ParquetStreamTransformer.iter`, which holds
+at most one row group of record batches. Collecting the whole staged batch first would hold
+~200 MiB of Arrow plus two copies of its parquet, per destination.
"""
from __future__ import annotations
@@ -151,29 +151,24 @@ class _WriteStats:
async def _parquet_byte_stream(
batches: AsyncIterator[pa.RecordBatch], compression: str | None, stats: _WriteStats
) -> AsyncIterator[bytes]:
- """Yield the parquet encoding of `batches`, one record batch at a time.
+ """Yield the parquet encoding of `batches` as one file, one row group at a time.
- `write_record_batch` returns only the bytes the writer produced for that batch and resets
- its buffer, so nothing accumulates. The pyarrow write runs in a thread to keep the event
- loop free for the upload it is feeding.
+ The transformer holds record batches until they fill a row group, so at most one row group
+ of Arrow data is in memory. A batch that later grows a column is projected back onto the
+ first schema, because one parquet file has one schema.
"""
- transformer = ParquetStreamTransformer(compression=compression, include_inserted_at=True)
+ # Delivery runs outside a Temporal activity, where there is no metric meter to record to.
+ transformer = ParquetStreamTransformer(compression=compression, include_inserted_at=True, record_metrics=False)
+
+ async for chunk in transformer.iter(_count_rows(batches, stats)):
+ stats.parquet_bytes += len(chunk.data)
+ yield chunk.data
- async for record_batch in batches:
- # No-op after the first batch. A batch that later grows a column is projected back onto
- # the first schema, because one parquet file has one schema.
- transformer.schema = record_batch.schema
- chunk = await asyncio.to_thread(transformer.write_record_batch, record_batch)
+async def _count_rows(batches: AsyncIterator[pa.RecordBatch], stats: _WriteStats) -> AsyncIterator[pa.RecordBatch]:
+ async for record_batch in batches:
stats.rows += record_batch.num_rows
- stats.parquet_bytes += len(chunk)
- if chunk:
- yield chunk
-
- footer = await asyncio.to_thread(transformer.finish_parquet_file)
- stats.parquet_bytes += len(footer)
- if footer:
- yield footer
+ yield record_batch
async def _chain(head: pa.RecordBatch, rest: AsyncIterator[pa.RecordBatch]) -> AsyncIterator[pa.RecordBatch]: