Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): size v3 load sets by decoded arrow memory (#119683)

Tom Owers4b86951a717e
diff --git a/posthog/settings/data_warehouse.py b/posthog/settings/data_warehouse.py
index 2501f6dda2d0..6667f5127747 100644
--- a/posthog/settings/data_warehouse.py
+++ b/posthog/settings/data_warehouse.py
@@ -227,13 +227,20 @@ WAREHOUSE_SOURCES_DATABASE_URL: str = (
# so a set must stay inside the memory slice the deltalite governor sizes per concurrent upsert
# (memory_governor.py): with a 29 GiB pod, 16 groups in flight and a 2 GiB reserve, that slice is
# about 1.3 GiB, of which 750 MiB goes to four partition workers. The row cap equals one extraction
-# chunk, the largest single batch the loader already takes, and the byte cap counts parquet on disk,
-# which decodes to Arrow several times its size. The count cap is only a backstop.
+# chunk, the largest single batch the loader already takes. The count cap is only a backstop.
+# When each batch records its decoded Arrow size and the pod limit is known, the copies of a set the
+# loader holds may use SLOT_SHARE of that slice, and the set's Arrow size never exceeds
+# MAX_ARROW_BYTES. Otherwise MAX_BYTES caps the parquet on disk, which decodes to Arrow several
+# times its size.
DATA_WAREHOUSE_V3_COALESCE_MAX_BATCHES = get_from_env("DATA_WAREHOUSE_V3_COALESCE_MAX_BATCHES", 128, type_cast=int)
DATA_WAREHOUSE_V3_COALESCE_MAX_ROWS = get_from_env("DATA_WAREHOUSE_V3_COALESCE_MAX_ROWS", 500_000, type_cast=int)
DATA_WAREHOUSE_V3_COALESCE_MAX_BYTES = get_from_env(
"DATA_WAREHOUSE_V3_COALESCE_MAX_BYTES", 64 * 1024 * 1024, type_cast=int
)
+DATA_WAREHOUSE_V3_COALESCE_SLOT_SHARE = get_from_env("DATA_WAREHOUSE_V3_COALESCE_SLOT_SHARE", 0.4, type_cast=float)
+DATA_WAREHOUSE_V3_COALESCE_MAX_ARROW_BYTES = get_from_env(
+ "DATA_WAREHOUSE_V3_COALESCE_MAX_ARROW_BYTES", 512 * 1024 * 1024, type_cast=int
+)
# Consecutive configuration errors (bad credentials, unknown database, unreachable host) after which
# PostHog pauses a destination instead of letting every scheduled run fail against it.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/coalescing.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/coalescing.py
index e21ca92dc4e5..2e825678cdef 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/coalescing.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/coalescing.py
@@ -24,6 +24,10 @@ if TYPE_CHECKING:
COALESCABLE_SYNC_TYPES: frozenset[str] = frozenset({"incremental", "append", "full_refresh"})
# A full refresh starts over at batch 0 and appends after it, so its runs never share a write.
CROSS_RUN_SYNC_TYPES: frozenset[str] = frozenset({"incremental", "append"})
+# The loader holds a set decoded in Arrow, then copies it once more while it deduplicates and
+# partitions, so the memory a set takes is about twice its Arrow size.
+_ARROW_COPIES_HELD = 2
+_MB = 1024 * 1024
@frozen
@@ -62,6 +66,8 @@ class CoalesceMember:
is_resume: bool
row_count: int
byte_size: int
+ # Decoded Arrow size the producer recorded. None for a batch queued without it.
+ arrow_bytes: int | None
shape: LoadShape
@classmethod
@@ -76,6 +82,7 @@ class CoalesceMember:
is_resume=batch.is_resume,
row_count=batch.row_count,
byte_size=batch.byte_size,
+ arrow_bytes=_arrow_bytes(metadata.get("arrow_bytes")),
shape=LoadShape(
sync_type=batch.sync_type,
resource_name=batch.resource_name,
@@ -102,6 +109,8 @@ class CoalesceMember:
is_resume=signal.is_resume,
row_count=signal.row_count,
byte_size=signal.byte_size,
+ # The sink checks only the join rules, which never read a size.
+ arrow_bytes=None,
shape=LoadShape(
sync_type=signal.sync_type,
resource_name=signal.resource_name,
@@ -120,21 +129,43 @@ class CoalesceMember:
@frozen
class CoalesceCaps:
- """The size a set may grow to. Rows and bytes are the real bounds; the count is a backstop."""
+ """The size a set may grow to. Rows and bytes are the real bounds; the count is a backstop.
+
+ `max_arrow_bytes` bounds the decoded size of a set whose members all recorded it. A set with a
+ member of unknown size, or a pod whose memory slice is unknown, is bounded by `max_bytes` of
+ parquet on disk instead.
+ """
max_batches: int
max_rows: int
max_bytes: int
+ max_arrow_bytes: int | None
@classmethod
- def from_settings(cls) -> CoalesceCaps:
+ def from_settings(cls, slot_budget_mb: float | None) -> CoalesceCaps:
+ """`slot_budget_mb` is the memory slice the governor gives one upsert, or None when unknown."""
return cls(
max_batches=settings.DATA_WAREHOUSE_V3_COALESCE_MAX_BATCHES,
max_rows=settings.DATA_WAREHOUSE_V3_COALESCE_MAX_ROWS,
max_bytes=settings.DATA_WAREHOUSE_V3_COALESCE_MAX_BYTES,
+ max_arrow_bytes=_arrow_budget(slot_budget_mb),
)
+def _arrow_budget(slot_budget_mb: float | None) -> int | None:
+ if slot_budget_mb is None or slot_budget_mb <= 0:
+ return None
+ held_bytes = slot_budget_mb * _MB * settings.DATA_WAREHOUSE_V3_COALESCE_SLOT_SHARE
+ return min(int(held_bytes / _ARROW_COPIES_HELD), settings.DATA_WAREHOUSE_V3_COALESCE_MAX_ARROW_BYTES)
+
+
+def _arrow_bytes(value: object) -> int | None:
+ # The metadata is free-form JSON, so a value of any other shape counts as unknown.
+ if isinstance(value, bool) or not isinstance(value, int) or value < 0:
+ return None
+ return value
+
+
def _key_tuple(keys: Sequence[str] | None) -> tuple[str, ...] | None:
return None if keys is None else tuple(keys)
@@ -192,9 +223,13 @@ def extends_set(current: Sequence[CoalesceMember], candidate: CoalesceMember, ca
return False
if len(current) >= caps.max_batches:
return False
- rows = sum(member.row_count for member in current) + candidate.row_count
- size = sum(member.byte_size for member in current) + candidate.byte_size
- return rows <= caps.max_rows and size <= caps.max_bytes
+ members = [*current, candidate]
+ if sum(member.row_count for member in members) > caps.max_rows:
+ return False
+ arrow_sizes = [member.arrow_bytes for member in members if member.arrow_bytes is not None]
+ if caps.max_arrow_bytes is not None and len(arrow_sizes) == len(members):
+ return sum(arrow_sizes) <= caps.max_arrow_bytes
+ return sum(member.byte_size for member in members) <= caps.max_bytes
def set_violation(members: Sequence[CoalesceMember]) -> str | None:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/consumer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/consumer.py
index a0bec5eb2831..1a7a36cdde1d 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/consumer.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/consumer.py
@@ -36,6 +36,7 @@ from products.warehouse_sources.backend.temporal.data_imports.metrics import (
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.auto_widen_resync import (
COLUMN_TYPE_WIDENED_KEY,
)
+from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.memory_governor import get_governor
from products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.coalescing import (
CoalesceCaps,
CoalesceMember,
@@ -1050,7 +1051,7 @@ class DeltaBatchConsumerAdapter:
Only adjacent batches join, so every set keeps the claim order: the order the loader would
have taken the batches in one at a time.
"""
- caps = CoalesceCaps.from_settings()
+ caps = CoalesceCaps.from_settings(slot_budget_mb=get_governor().slot_budget_mb())
sets: list[list[PendingBatch]] = []
current: list[PendingBatch] = []
current_members: list[CoalesceMember] = []
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/producer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/producer.py
index c3a5541f0cff..5382fee5b43f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/producer.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/producer.py
@@ -316,6 +316,8 @@ class PostgresProducer:
if self._workflow_run_id is not None:
metadata["workflow_run_id"] = self._workflow_run_id
metadata["timestamp_ns"] = batch_result.timestamp_ns
+ if batch_result.arrow_bytes is not None:
+ metadata["arrow_bytes"] = batch_result.arrow_bytes
if self._external_destination_ids is not None:
metadata["external_destination_ids"] = self._external_destination_ids
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_consumer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_consumer.py
index 4d208067ab8d..94407679fdaf 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_consumer.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_consumer.py
@@ -1,6 +1,8 @@
import math
import time
import asyncio
+import dataclasses
+from collections.abc import Iterator
from datetime import UTC, datetime
from typing import Any, NamedTuple, cast
@@ -22,6 +24,12 @@ from products.warehouse_sources.backend.temporal.data_imports.metrics import LOC
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.errors import (
TransientObjectStoreError,
)
+from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.memory_governor import (
+ GovernorConfig,
+ MemoryGovernor,
+ PodMemory,
+ reset_governor_for_tests,
+)
from products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.destinations_load.errors import (
DestinationConfigurationError,
DestinationDeliveryError,
@@ -4218,10 +4226,42 @@ def _run_batches(count: int, *, run_uuid: str = "run-1", start: int = 0, **overr
]
+_MIB = 1024 * 1024
+
+
+class _UnlimitedPod(PodMemory):
+ def limit_mb(self) -> float | None:
+ return None
+
+
+def _use_pod_slot(slot_mb: float | None) -> None:
+ # One upsert per pod with no safety margin or reserve, so the slice equals the pod limit.
+ config = GovernorConfig(max_concurrent=1, safety=1.0, reserve_mb=0.0, rss_sample_ms=0.0)
+ pod = _UnlimitedPod() if slot_mb is None else PodMemory(limit_override_mb=slot_mb)
+ reset_governor_for_tests(MemoryGovernor(config=config, pod=pod))
+
+
+def _sized_batches(sizes: list[tuple[object, int]], **overrides: Any) -> list[PendingBatch]:
+ batches = _run_batches(len(sizes), **overrides)
+ return [
+ dataclasses.replace(
+ batch,
+ byte_size=byte_size,
+ metadata={} if arrow_bytes is None else {"arrow_bytes": arrow_bytes},
+ )
+ for batch, (arrow_bytes, byte_size) in zip(batches, sizes)
+ ]
+
+
class TestCoalesceGroup:
"""Which consecutive batches the Delta sink may load as one write. A set that crosses a run, skips
an index or grows past the caps would merge rows the writer must never see together."""
+ @pytest.fixture(autouse=True)
+ def _restore_governor(self) -> Iterator[None]:
+ yield
+ reset_governor_for_tests(None)
+
def _sets(self, batches: list[PendingBatch]) -> list[list[int]]:
return [[b.batch_index for b in s] for s in DeltaBatchConsumerAdapter().coalesce_group(batches)]
@@ -4375,6 +4415,69 @@ class TestCoalesceGroup:
with override_settings(**settings_overrides):
assert self._sets(_run_batches(count, **overrides)) == expected
+ @pytest.mark.parametrize(
+ "slot_mb,sizes,expected",
+ [
+ # A 64 MiB slice at share 1.0 leaves 32 MiB of Arrow for the two copies the loader holds.
+ (64, [(10 * _MIB, 1)] * 5, [[0, 1, 2], [3, 4]]),
+ # Wide rows that compress badly join past the parquet cap while they fit the memory budget.
+ (64, [(10 * _MIB, 30 * _MIB)] * 4, [[0, 1, 2], [3]]),
+ # A batch bigger than the whole budget still loads, on its own.
+ (64, [(1 * _MIB, 1), (100 * _MIB, 1), (1 * _MIB, 1)], [[0], [1], [2]]),
+ # A large slice is still bounded by the hard ceiling.
+ (100_000, [(10 * _MIB, 1)] * 5, [[0, 1, 2, 3], [4]]),
+ # Narrow batches still coalesce until the row cap.
+ (64, [(1024, 1)] * 60, [list(range(50)), list(range(50, 60))]),
+ # One batch queued before the producer recorded its size puts the set back on the parquet cap.
+ (64, [(10 * _MIB, 30 * _MIB), (None, 30 * _MIB), (10 * _MIB, 30 * _MIB)], [[0, 1], [2]]),
+ # The metadata is free-form JSON, so any other shape counts as unknown.
+ (64, [("10", 30 * _MIB), (True, 30 * _MIB), (-1, 30 * _MIB)], [[0, 1], [2]]),
+ # An unreadable pod limit gives no slice to size against.
+ (None, [(10 * _MIB, 30 * _MIB)] * 3, [[0, 1], [2]]),
+ # A slice that leaves no memory gives no Arrow budget.
+ (0, [(10 * _MIB, 30 * _MIB)] * 3, [[0, 1], [2]]),
+ ],
+ ids=[
+ "wide_stops_at_memory_budget",
+ "wide_joins_past_parquet_cap",
+ "oversized_batch_alone",
+ "ceiling_binds",
+ "narrow_stops_at_row_cap",
+ "unknown_batch_size_falls_back",
+ "invalid_batch_size_falls_back",
+ "unknown_slot_falls_back",
+ "empty_slot_falls_back",
+ ],
+ )
+ def test_sets_are_capped_by_arrow_memory(
+ self, slot_mb: float | None, sizes: list[tuple[object, int]], expected: list[list[int]]
+ ):
+ _use_pod_slot(slot_mb)
+ with override_settings(
+ DATA_WAREHOUSE_V3_COALESCE_SLOT_SHARE=1.0, DATA_WAREHOUSE_V3_COALESCE_MAX_ARROW_BYTES=45 * _MIB
+ ):
+ assert self._sets(_sized_batches(sizes, row_count=10_000)) == expected
+
+ @pytest.mark.parametrize(
+ "batches,expected",
+ [
+ (_sized_batches([(_MIB, 1)] * 2) + _run_batches(1, start=3, metadata={"arrow_bytes": _MIB}), [[0, 1], [3]]),
+ (
+ _sized_batches([(_MIB, 1)] * 2) + _run_batches(2, run_uuid="run-2", metadata={"arrow_bytes": _MIB}),
+ [[0, 1], [0, 1]],
+ ),
+ (
+ _sized_batches([(_MIB, 1)] * 2)
+ + _run_batches(2, run_uuid="run-2", is_resume=True, metadata={"arrow_bytes": _MIB}),
+ [[0, 1, 0, 1]],
+ ),
+ ],
+ ids=["gap_splits", "fresh_run_splits", "resume_joins"],
+ )
+ def test_arrow_budget_keeps_the_join_rules(self, batches: list[PendingBatch], expected: list[list[int]]):
+ _use_pod_slot(1024)
+ assert self._sets(batches) == expected
+
class _BatchStatus(NamedTuple):
batch_id: str
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_producer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_producer.py
index 244a0542b668..343ab6cf73eb 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_producer.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_producer.py
@@ -1,4 +1,5 @@
import json
+import dataclasses
from typing import Any
import pytest
@@ -109,13 +110,14 @@ class TestPostgresProducerSendBatch:
cdc_write_mode="upsert",
cdc_table_mode="merge",
)
- batch_result = _make_batch_result(batch_index=1)
+ batch_result = dataclasses.replace(_make_batch_result(batch_index=1), arrow_bytes=4096)
producer.send_batch_notification(batch_result)
mock = _mock_conn(producer)
params = mock.execute.call_args[0][1]
metadata = json.loads(params["metadata"])
+ assert metadata["arrow_bytes"] == 4096
assert metadata["primary_keys"] == ["id"]
assert metadata["partition_count"] == 4
assert metadata["cdc_write_mode"] == "upsert"
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/s3/common.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/s3/common.py
index b37137e2d794..8115fc5d140e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/s3/common.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/s3/common.py
@@ -13,7 +13,7 @@ from products.warehouse_sources.backend.temporal.data_imports.util import NonRet
logger = structlog.get_logger(__name__)
-@dataclass
+@dataclass(frozen=False)
class BatchWriteResult:
"""Result of writing a batch to S3."""
@@ -22,6 +22,8 @@ class BatchWriteResult:
byte_size: int
batch_index: int
timestamp_ns: int = field(default_factory=time.time_ns)
+ # Decoded size of the batch in Arrow, which the loader uses to size the sets it writes at once.
+ arrow_bytes: int | None = None
def strip_s3_protocol(s3_path: str) -> str:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/s3/test_writer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/s3/test_writer.py
index b96e5d3f6385..1615ed5cdb6e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/s3/test_writer.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/s3/test_writer.py
@@ -6,6 +6,7 @@ import pytest
from unittest.mock import MagicMock, patch
import pyarrow as pa
+import pyarrow.parquet as pq
import botocore.exceptions
from fsspec.implementations.memory import MemoryFileSystem
from parameterized import parameterized
@@ -149,9 +150,14 @@ class TestBatchByteSize:
result = writer.write_batch(table, 0)
- stored_size = store.info(result.s3_path.split("://", 1)[-1])["size"]
+ stored_path = result.s3_path.split("://", 1)[-1]
+ stored_size = store.info(stored_path)["size"]
assert result.byte_size == stored_size
assert result.byte_size > 0
+ # The loader reads the file back into Arrow, so the recorded size must match that, not the parquet.
+ with store.open(stored_path, "rb") as f:
+ loaded_bytes = pq.read_table(f).nbytes
+ assert result.arrow_bytes == pytest.approx(loaded_bytes, rel=0.05, abs=1024)
class TestWriteBatchPermissionDenied:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/s3/writer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/s3/writer.py
index 1f41bcc9d247..57baec568703 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/s3/writer.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/s3/writer.py
@@ -193,6 +193,7 @@ class S3BatchWriter:
byte_size=byte_size,
batch_index=batch_index,
timestamp_ns=timestamp_ns,
+ arrow_bytes=pa_table.nbytes,
)
def write_schema(self) -> str | None: