:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(batch-exports): write parquet files with larger row groups (#119655)
Rossd7d698f0e540
diff --git a/posthog/settings/batch_exports.py b/posthog/settings/batch_exports.pyindex ed9ad4c9ea9f..e8afe0d5472a 100644--- a/posthog/settings/batch_exports.py+++ b/posthog/settings/batch_exports.py@@ -116,6 +116,10 @@ BATCH_EXPORTS_CLICKHOUSE_MAX_INSERT_BLOCK_SIZE_BYTES: int = get_from_env("BATCH_EXPORTS_CLICKHOUSE_MAX_INSERT_BLOCK_SIZE_BYTES", 64 * 1024 * 1024, type_cast=int)+BATCH_EXPORT_PARQUET_MAX_BUFFERED_ARROW_BYTES: int = get_from_env(+ "BATCH_EXPORT_PARQUET_MAX_BUFFERED_ARROW_BYTES", 32 * 1024 * 1024, type_cast=int+)+BATCH_EXPORT_OBJECT_STORAGE_ENDPOINT: str = os.getenv("BATCH_EXPORT_OBJECT_STORAGE_ENDPOINT", "http://objectstorage:19000")diff --git a/posthog/temporal/common/asyncpa.py b/posthog/temporal/common/asyncpa.pyindex fe1856abb287..f5e3a559dfaa 100644--- a/posthog/temporal/common/asyncpa.py+++ b/posthog/temporal/common/asyncpa.py@@ -74,11 +74,15 @@ class AsyncMessageReader:total_message_size = 8 + metadata_size + body_sizeawait self.read_until(total_message_size)+ # A message parsed from a view of the buffer keeps the whole buffer alive, including bytes of later+ # messages. Parse a copy of the message instead, so a held record batch only keeps its own bytes.with memoryview(self._buffer) as buffer_view:- loop = asyncio.get_running_loop()- msg = await loop.run_in_executor(None, pa.ipc.read_message, buffer_view[:total_message_size])+ message_bytes = bytes(buffer_view[:total_message_size])+ del self._buffer[:total_message_size]++ loop = asyncio.get_running_loop()+ msg = await loop.run_in_executor(None, pa.ipc.read_message, message_bytes)- self._buffer = self._buffer[total_message_size:]self._bytes_consumed += total_message_sizereturn msgdiff --git a/posthog/temporal/tests/common/test_asyncpa.py b/posthog/temporal/tests/common/test_asyncpa.pyindex 663db9f95a59..1c3ceda3c14f 100644--- a/posthog/temporal/tests/common/test_asyncpa.py+++ b/posthog/temporal/tests/common/test_asyncpa.py@@ -1,7 +1,9 @@+import gcimport ioimport randomimport stringimport asyncio+import tracemallocimport collections.abcimport pytest@@ -149,3 +151,25 @@ async def test_message_reader_stops_waiting_for_a_stalled_error_tail(monkeypatchawait asyncio.wait_for(reader.read_next_message(), timeout=5)assert exc_info.value.unparsed == b"Code: 241. DB::Exception: an error"+++async def test_record_batch_reader_does_not_keep_read_buffers_alive():+ batches = list(generate_record_batches(total_records=3_000, total_batches=300, target_size_bytes=100))++ buffer = io.BytesIO()+ with pa.ipc.new_stream(buffer, schema=batches[0].schema) as writer:+ for batch in batches:+ writer.write_batch(batch)+ _ = buffer.seek(0)++ gc.collect()+ tracemalloc.start()+ try:+ held_batches = [batch async for batch in asyncpa.AsyncRecordBatchReader(AsyncWrapper(buffer, 128 * 1024))]+ gc.collect()+ held_bytes, _ = tracemalloc.get_traced_memory()+ finally:+ tracemalloc.stop()++ assert held_batches == batches+ assert held_bytes < 2 * sum(batch.nbytes for batch in held_batches)diff --git a/products/batch_exports/backend/temporal/pipeline/transformer.py b/products/batch_exports/backend/temporal/pipeline/transformer.pyindex 5d436ea23320..3347543ee31d 100644--- a/products/batch_exports/backend/temporal/pipeline/transformer.py+++ b/products/batch_exports/backend/temporal/pipeline/transformer.py@@ -387,8 +387,29 @@ def replace_broken_unicode(obj):return obj+def get_parquet_column_paths(schema: pa.Schema, field_names: collections.abc.Iterable[str]) -> list[str]:+ """Return the Parquet leaf column paths of the fields named `field_names` in `schema`.++ `ParquetWriter` matches `use_dictionary` and `write_statistics` against leaf column paths, such as+ `tags.list.element` for a list field, not against top-level field names.+ """+ buffer = io.BytesIO()+ pq.write_table(pa.schema([schema.field(name) for name in field_names]).empty_table(), buffer)+ parquet_schema = pq.ParquetFile(buffer).schema+ return [parquet_schema.column(index).path for index in range(len(parquet_schema))]++class ParquetStreamTransformer:- """A transformer to convert record batches into Parquet."""+ """A transformer to convert record batches into Parquet.++ Record batches are held until `max_buffered_arrow_bytes` of them are pending, then written as one+ row group. The limit counts Arrow bytes, so the row group in the file is usually smaller.+ """++ # Close a file when less than this fraction of its max size is left. Otherwise, the last row+ # group ends up being very small and we're also more likely to go over the max file size.+ MIN_REMAINING_FILE_SIZE_FRACTION = 0.05+ JSON_COLUMNS = frozenset({"properties", "person_properties", "set", "set_once"})def __init__(self,@@ -396,70 +417,89 @@ class ParquetStreamTransformer:compression_level: int | None = None,include_inserted_at: bool = False,max_file_size_bytes: int = 0,+ max_buffered_arrow_bytes: int | None = None,+ record_metrics: bool = True,):self.include_inserted_at = include_inserted_atself.compression = compressionself.compression_level = compression_levelself.max_file_size_bytes = max_file_size_bytes+ self.record_metrics = record_metrics++ if max_buffered_arrow_bytes is None:+ max_buffered_arrow_bytes = settings.BATCH_EXPORT_PARQUET_MAX_BUFFERED_ARROW_BYTES+ if max_file_size_bytes:+ max_buffered_arrow_bytes = min(max_buffered_arrow_bytes, max_file_size_bytes)+ self.max_buffered_arrow_bytes = max_buffered_arrow_bytes- # For Parquet, we need to handle schema and batchingself._parquet_writer: pq.ParquetWriter | None = Noneself._parquet_buffer = io.BytesIO()self._schema: pa.Schema | None = None+ self._pending_record_batches: list[pa.RecordBatch] = []+ self._pending_record_batches_bytes = 0+ self._pending_record_batches_rows = 0+ self._non_json_column_paths: list[str] = []+ self._current_file_size = 0+ # The compression ratio (Parquet bytes written per Arrow byte) achieved when writing the+ # previous row group. We use this as an estimate for calculating how many bytes will be+ # written next time. It starts at 1 because Parquet output is usually no larger than its+ # Arrow input.+ self._compression_ratio = 1.0async def iter(self, record_batches: collections.abc.AsyncIterable[pa.RecordBatch]) -> collections.abc.AsyncIterator[Chunk]:"""Iterate over record batches transforming them into chunks."""- current_file_size = 0-async for record_batch in record_batches:self.schema = record_batch.schema+ self._add_pending_record_batch(record_batch)- with ExecutionTimeRecorder(- "parquet_stream_transformer_record_batch_transform_duration",- description="Duration to transform a record batch into Parquet bytes.",- log_message=(- "Processed record batch with %(num_records)d records to parquet."- " Record batch size: %(mb_processed).2f MB, process time:"- " %(duration_seconds)d seconds, speed: %(mb_per_second).2f MB/s"- ),- log_attributes={"num_records": record_batch.num_rows},- ) as recorder:- recorder.add_bytes_processed(record_batch.nbytes)- # Running write in a thread to yield control back to event loop.- chunk = await asyncio.to_thread(self.write_record_batch, record_batch)+ if not self._is_row_group_ready():+ continue- yield Chunk(chunk, False)+ if self.record_metrics:+ with ExecutionTimeRecorder(+ "parquet_stream_transformer_record_batch_transform_duration",+ description="Duration to write pending record batches as a Parquet row group.",+ log_message=(+ "Wrote row group with %(num_records)d records to parquet."+ " Record batches size: %(mb_processed).2f MB, process time:"+ " %(duration_seconds)d seconds, speed: %(mb_per_second).2f MB/s"+ ),+ log_attributes={"num_records": self._pending_record_batches_rows},+ ) as recorder:+ recorder.add_bytes_processed(self._pending_record_batches_bytes)+ # Running write in a thread to yield control back to event loop.+ chunk = await asyncio.to_thread(self._flush_row_group)+ else:+ chunk = await asyncio.to_thread(self._flush_row_group)- if self.max_file_size_bytes and current_file_size + len(chunk) > self.max_file_size_bytes:- footer = await asyncio.to_thread(self.finish_parquet_file)+ yield Chunk(chunk, False)+ self._current_file_size += len(chunk)- yield Chunk(footer, True)- current_file_size = 0+ if self._is_current_file_full():+ footer = await asyncio.to_thread(self._finish_parquet_file)- else:- current_file_size += len(chunk)+ yield Chunk(footer, True)- if not self._schema:+ # Nothing is left to finish when no record batches arrived, or when the last file was finished in the loop.+ if self._current_file_size == 0 and not self._pending_record_batches:return- footer = await asyncio.to_thread(self.finish_parquet_file)+ footer = await asyncio.to_thread(self._finish_parquet_file)yield Chunk(footer, True)- @property- def parquet_writer(self) -> pq.ParquetWriter:+ def _get_parquet_writer(self) -> pq.ParquetWriter:if self._parquet_writer is None:- json_columns = {"properties", "person_properties", "set", "set_once"}- # self.schema is set before getting a ParquetWriter.- dicitionary_encoded_columns = [col for col in self.schema.names if col not in json_columns]-self._parquet_writer = pq.ParquetWriter(self._parquet_buffer,schema=self.schema,compression="none" if self.compression is None else self.compression, # type: ignorecompression_level=self.compression_level,- use_dictionary=dicitionary_encoded_columns, # type: ignore+ use_dictionary=self._non_json_column_paths, # type: ignore+ # Min/max statistics on JSON strings cannot help a reader skip row groups, so we do+ # not write statistics for them to save on file size.+ write_statistics=self._non_json_column_paths,)assert self._parquet_writer is not Nonereturn self._parquet_writer@@ -480,29 +520,75 @@ class ParquetStreamTransformer:schema = schema.remove(index)self._schema = schema+ self._non_json_column_paths = get_parquet_column_paths(+ schema, [name for name in schema.names if name not in self.JSON_COLUMNS]+ )- def finish_parquet_file(self) -> bytes:- """Ensure underlying Parquet writer is closed before flushing buffer."""- self.parquet_writer.close()+ def _finish_parquet_file(self) -> bytes:+ """Write any pending row group, close the file, and return the rest of its bytes."""+ self._write_pending_row_group()+ self._get_parquet_writer().close()self._parquet_writer = None+ self._current_file_size = 0- final_data = self._parquet_buffer.getvalue()+ return self._drain_parquet_buffer()- self._parquet_buffer = io.BytesIO()+ def _is_current_file_full(self) -> bool:+ """Return whether the current file has too little space left for another row group."""+ if not self.max_file_size_bytes:+ return False- return final_data+ remaining_bytes = self.max_file_size_bytes - self._current_file_size+ return remaining_bytes < self.max_file_size_bytes * self.MIN_REMAINING_FILE_SIZE_FRACTION- def write_record_batch(self, record_batch: pa.RecordBatch) -> bytes:- """Write record batch to buffer as Parquet."""- column_names = self.schema.names- self.parquet_writer.write_batch(record_batch.select(column_names))- data = self._parquet_buffer.getvalue()+ def _is_row_group_ready(self) -> bool:+ """Return whether enough record batches are pending to write a row group."""+ if not self.max_file_size_bytes:+ return self._pending_record_batches_bytes >= self.max_buffered_arrow_bytes++ # Near the end of a file, the limit is the space left in the file converted to Arrow bytes+ # using our compression ratio estimate.+ remaining_bytes = self.max_file_size_bytes - self._current_file_size+ limit_bytes = min(self.max_buffered_arrow_bytes, remaining_bytes / self._compression_ratio)+ return self._pending_record_batches_bytes >= limit_bytes+ def _add_pending_record_batch(self, record_batch: pa.RecordBatch) -> None:+ """Hold a record batch until enough record batches are pending to write a row group."""+ record_batch = record_batch.select(self.schema.names)+ self._pending_record_batches.append(record_batch)+ self._pending_record_batches_bytes += record_batch.nbytes+ self._pending_record_batches_rows += record_batch.num_rows++ def _flush_row_group(self) -> bytes:+ """Write all pending record batches as one row group, and return the bytes written."""+ pending_record_batches_bytes = self._pending_record_batches_bytes+ self._write_pending_row_group()+ data = self._drain_parquet_buffer()++ if pending_record_batches_bytes:+ self._compression_ratio = len(data) / pending_record_batches_bytes++ return data++ def _drain_parquet_buffer(self) -> bytes:+ """Return the bytes in the buffer, and empty it for the next write."""+ data = self._parquet_buffer.getvalue()self._parquet_buffer.seek(0)self._parquet_buffer.truncate(0)-return data+ def _write_pending_row_group(self) -> None:+ """Write all pending record batches to buffer as a single Parquet row group."""+ if not self._pending_record_batches:+ return++ table = pa.Table.from_batches(self._pending_record_batches, schema=self.schema)+ self._get_parquet_writer().write_table(table, row_group_size=max(table.num_rows, 1))++ self._pending_record_batches = []+ self._pending_record_batches_bytes = 0+ self._pending_record_batches_rows = 0+class RedshiftQueryStreamTransformer:"""A transformer to convert record batches into a Redshift INSERT INTO query."""diff --git a/products/batch_exports/backend/tests/temporal/pipeline/test_transformer.py b/products/batch_exports/backend/tests/temporal/pipeline/test_transformer.pyindex 164f72e46d5d..0f3bb9e8655b 100644--- a/products/batch_exports/backend/tests/temporal/pipeline/test_transformer.py+++ b/products/batch_exports/backend/tests/temporal/pipeline/test_transformer.py@@ -1,13 +1,17 @@import ioimport csvimport json+import randomimport typing+import hashlibimport datetime as dtimport functools+import collections.abcimport pytestimport pyarrow as pa+import pyarrow.parquet as pqfrom products.batch_exports.backend.temporal.pipeline.table import (Field,@@ -18,6 +22,7 @@ from products.batch_exports.backend.temporal.pipeline.table import (from products.batch_exports.backend.temporal.pipeline.transformer import (CSVStreamTransformer,JSONLStreamTransformer,+ ParquetStreamTransformer,PipelineTransformer,SchemaTransformer,_ensure_curly_brackets_array,@@ -149,6 +154,125 @@ async def test_csv_stream_transformer_writes_record_batches():assert row == expected_row+async def iterate_record_batches(+ record_batches: list[pa.RecordBatch],+) -> collections.abc.AsyncIterator[pa.RecordBatch]:+ for record_batch in record_batches:+ yield record_batch+++async def collect_parquet_files(+ transformer: ParquetStreamTransformer, record_batches: list[pa.RecordBatch]+) -> list[bytes]:+ files: list[bytes] = []+ current_file = b""+ async for chunk in transformer.iter(iterate_record_batches(record_batches)):+ current_file += chunk.data+ if chunk.is_eof:+ files.append(current_file)+ current_file = b""+ return files+++ "max_file_size_bytes,compression",+ [(0, None), (256 * 1024, None), (256 * 1024, "zstd")],+)+async def test_parquet_stream_transformer_writes_small_record_batches_as_large_row_groups(+ max_file_size_bytes, compression+):+ records = [+ {+ "uuid": f"uuid-{index}",+ "properties": json.dumps(+ {+ "index": index,+ "padding": "".join(hashlib.sha256(f"{index}-{n}".encode()).hexdigest() for n in range(16)),+ }+ ),+ }+ for index in range(1000)+ ]+ record_batches = pa.Table.from_pylist(records).to_batches(max_chunksize=10)++ transformer = ParquetStreamTransformer(+ compression=compression, max_file_size_bytes=max_file_size_bytes, max_buffered_arrow_bytes=64 * 1024+ )+ files = await collect_parquet_files(transformer, record_batches)++ parquet_files = [pq.ParquetFile(io.BytesIO(file)) for file in files]+ assert all(parquet_file.metadata.num_rows > 0 for parquet_file in parquet_files)+ # Writing each record batch as its own row group would give one row group per batch. The exact count+ # depends on where files split and on the learned compression ratio, so only check that batches are combined.+ assert sum(parquet_file.metadata.num_row_groups for parquet_file in parquet_files) < len(record_batches) / 2+ assert [row for parquet_file in parquet_files for row in parquet_file.read().to_pylist()] == records+ # A file closes when less than 5% of its max size is left, and can go over by the error in the compression+ # ratio, so full files land near the max but not on it. The last file holds the remaining rows, so it can be small.+ if max_file_size_bytes:+ assert len(files) > 1+ assert all(0.9 * max_file_size_bytes <= len(file) <= 1.1 * max_file_size_bytes for file in files[:-1])+++async def test_parquet_stream_transformer_bounds_file_size_when_compression_gets_worse():+ max_file_size_bytes = 1024 * 1024+ rng = random.Random(0)+ compressible = [+ pa.RecordBatch.from_pydict({"properties": ['{"a": "' + "x" * 10_000 + '"}'] * 20}) for _ in range(30)+ ]+ incompressible = [+ pa.RecordBatch.from_pydict({"properties": [rng.randbytes(5_000).hex() for _ in range(20)]}) for _ in range(40)+ ]++ transformer = ParquetStreamTransformer(compression="zstd", max_file_size_bytes=max_file_size_bytes)+ files = await collect_parquet_files(transformer, compressible + incompressible)++ assert max(len(file) for file in files) <= 2 * max_file_size_bytes+++async def test_parquet_stream_transformer_skips_statistics_and_dictionaries_only_for_json_columns():+ record_batch = pa.RecordBatch.from_pylist(+ [+ {"uuid": "a", "tags": ["x", "y"], "person": {"name": "a"}, "properties": '{"a": 1}'},+ {"uuid": "b", "tags": ["z"], "person": {"name": "b"}, "properties": '{"b": 2}'},+ ]+ )++ transformer = ParquetStreamTransformer()+ (data,) = await collect_parquet_files(transformer, [record_batch])++ row_group = pq.ParquetFile(io.BytesIO(data)).metadata.row_group(0)+ columns = [row_group.column(index) for index in range(row_group.num_columns)]+ assert {+ column.path_in_schema: column.statistics is not None and column.statistics.has_min_max for column in columns+ } == {+ "uuid": True,+ "tags.list.element": True,+ "person.name": True,+ "properties": False,+ }+ assert {column.path_in_schema: "RLE_DICTIONARY" in column.encodings for column in columns} == {+ "uuid": True,+ "tags.list.element": True,+ "person.name": True,+ "properties": False,+ }+++async def test_parquet_stream_transformer_without_metrics_runs_outside_temporal(settings):+ # get_metric_meter only raises outside Temporal when DEBUG and TEST are both off.+ settings.DEBUG = False+ settings.TEST = False++ transformer = ParquetStreamTransformer(record_metrics=False, max_buffered_arrow_bytes=1)+ (data,) = await collect_parquet_files(transformer, [pa.RecordBatch.from_pylist([{"uuid": "a"}])])++ assert pq.ParquetFile(io.BytesIO(data)).metadata.num_rows == 1++class FakeField(Field):def __init__(self, name: str, data_type: pa.DataType):self.name = namediff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/azure_blob.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/azure_blob.pyindex 9bd3b5b2ff70..41426d0d3e19 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/azure_blob.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/azure_blob.py@@ -25,10 +25,9 @@ Connection handling comes from batch exports' Azure Blob destination:`AzureBlobConsumer` itself is not used, even though it touches no Temporal metric meter and sowould run outside an activity. Its `consume_chunk` accumulates the whole file in a `bytearray`before uploading, which is the memory behavior this writer exists to avoid. Instead the parquet-is streamed a record batch at a time through `ParquetStreamTransformer`, whose-`write_record_batch` and `finish_parquet_file` are pure, and handed to `upload_blob` as an-async iterator. The SDK reads it in `max_block_size` blocks, so peak memory is one record batch-plus a few blocks rather than ~200 MiB of Arrow and two copies of its parquet.+is streamed a row group at a time through `ParquetStreamTransformer.iter`, and handed to+`upload_blob` as an async iterator. The SDK reads it in `max_block_size` blocks, so peak memory+is one row group plus a few blocks rather than ~200 MiB of Arrow and two copies of its parquet."""from __future__ import annotations@@ -111,29 +110,24 @@ class _WriteStats:async def _parquet_byte_stream(batches: AsyncIterator[pa.RecordBatch], compression: str | None, stats: _WriteStats) -> AsyncIterator[bytes]:- """Yield the parquet encoding of `batches`, one record batch at a time.+ """Yield the parquet encoding of `batches` as one file, one row group at a time.- `write_record_batch` returns only the bytes the writer produced for that batch and resets- its buffer, so nothing accumulates. The pyarrow write runs in a thread to keep the event- loop free for the upload it is feeding.+ The transformer holds record batches until they fill a row group, so at most one row group+ of Arrow data is in memory. A batch that later grows a column is projected back onto the+ first schema, because one parquet file has one schema."""- transformer = ParquetStreamTransformer(compression=compression, include_inserted_at=True)+ # Delivery runs outside a Temporal activity, where there is no metric meter to record to.+ transformer = ParquetStreamTransformer(compression=compression, include_inserted_at=True, record_metrics=False)++ async for chunk in transformer.iter(_count_rows(batches, stats)):+ stats.parquet_bytes += len(chunk.data)+ yield chunk.data- async for record_batch in batches:- # No-op after the first batch. A batch that later grows a column is projected back onto- # the first schema, because one parquet file has one schema.- transformer.schema = record_batch.schema- chunk = await asyncio.to_thread(transformer.write_record_batch, record_batch)+async def _count_rows(batches: AsyncIterator[pa.RecordBatch], stats: _WriteStats) -> AsyncIterator[pa.RecordBatch]:+ async for record_batch in batches:stats.rows += record_batch.num_rows- stats.parquet_bytes += len(chunk)- if chunk:- yield chunk-- footer = await asyncio.to_thread(transformer.finish_parquet_file)- stats.parquet_bytes += len(footer)- if footer:- yield footer+ yield record_batchasync def _chain(head: pa.RecordBatch, rest: AsyncIterator[pa.RecordBatch]) -> AsyncIterator[pa.RecordBatch]:diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/s3.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/s3.pyindex cce0706d86b8..82ba3ef1e574 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/s3.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/s3.py@@ -22,14 +22,14 @@ Credentials, the client and its config come from batch exports' S3 destination:roles, and `request_checksum_calculation="when_required"`. That last one is why R2, GCS andMinIO accept our uploads at all: they reject AWS's newer checksum headers.-What is not taken from there is the upload itself. `ConcurrentS3Consumer` times every part-through `ExecutionTimeRecorder`, which resolves a Temporal metric meter and raises outside an-activity. Delivery runs in the Postgres-queue load worker, not in an activity, so the parts are-uploaded here instead, keeping their `RequestTimeout` retry behavior.--Bytes are streamed a record batch at a time through `ParquetStreamTransformer`, whose-`write_record_batch` and `finish_parquet_file` are pure. Collecting the whole staged batch-first would hold ~200 MiB of Arrow plus two copies of its parquet, per destination.+What is not taken from there is the upload itself. `ConcurrentS3Consumer` counts exported rows+and bytes through Temporal activity metrics, which raise outside an activity. Delivery runs in the+Postgres-queue load worker, not in an activity, so the parts are uploaded here instead, keeping+their `RequestTimeout` retry behavior.++Bytes are streamed a row group at a time through `ParquetStreamTransformer.iter`, which holds+at most one row group of record batches. Collecting the whole staged batch first would hold+~200 MiB of Arrow plus two copies of its parquet, per destination."""from __future__ import annotations@@ -151,29 +151,24 @@ class _WriteStats:async def _parquet_byte_stream(batches: AsyncIterator[pa.RecordBatch], compression: str | None, stats: _WriteStats) -> AsyncIterator[bytes]:- """Yield the parquet encoding of `batches`, one record batch at a time.+ """Yield the parquet encoding of `batches` as one file, one row group at a time.- `write_record_batch` returns only the bytes the writer produced for that batch and resets- its buffer, so nothing accumulates. The pyarrow write runs in a thread to keep the event- loop free for the upload it is feeding.+ The transformer holds record batches until they fill a row group, so at most one row group+ of Arrow data is in memory. A batch that later grows a column is projected back onto the+ first schema, because one parquet file has one schema."""- transformer = ParquetStreamTransformer(compression=compression, include_inserted_at=True)+ # Delivery runs outside a Temporal activity, where there is no metric meter to record to.+ transformer = ParquetStreamTransformer(compression=compression, include_inserted_at=True, record_metrics=False)++ async for chunk in transformer.iter(_count_rows(batches, stats)):+ stats.parquet_bytes += len(chunk.data)+ yield chunk.data- async for record_batch in batches:- # No-op after the first batch. A batch that later grows a column is projected back onto- # the first schema, because one parquet file has one schema.- transformer.schema = record_batch.schema- chunk = await asyncio.to_thread(transformer.write_record_batch, record_batch)+async def _count_rows(batches: AsyncIterator[pa.RecordBatch], stats: _WriteStats) -> AsyncIterator[pa.RecordBatch]:+ async for record_batch in batches:stats.rows += record_batch.num_rows- stats.parquet_bytes += len(chunk)- if chunk:- yield chunk-- footer = await asyncio.to_thread(transformer.finish_parquet_file)- stats.parquet_bytes += len(footer)- if footer:- yield footer+ yield record_batchasync def _chain(head: pa.RecordBatch, rest: AsyncIterator[pa.RecordBatch]) -> AsyncIterator[pa.RecordBatch]: