:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): size v3 load sets by decoded arrow memory (#119683)
Tom Owers4b86951a717e
diff --git a/posthog/settings/data_warehouse.py b/posthog/settings/data_warehouse.pyindex 2501f6dda2d0..6667f5127747 100644--- a/posthog/settings/data_warehouse.py+++ b/posthog/settings/data_warehouse.py@@ -227,13 +227,20 @@ WAREHOUSE_SOURCES_DATABASE_URL: str = (# so a set must stay inside the memory slice the deltalite governor sizes per concurrent upsert# (memory_governor.py): with a 29 GiB pod, 16 groups in flight and a 2 GiB reserve, that slice is# about 1.3 GiB, of which 750 MiB goes to four partition workers. The row cap equals one extraction-# chunk, the largest single batch the loader already takes, and the byte cap counts parquet on disk,-# which decodes to Arrow several times its size. The count cap is only a backstop.+# chunk, the largest single batch the loader already takes. The count cap is only a backstop.+# When each batch records its decoded Arrow size and the pod limit is known, the copies of a set the+# loader holds may use SLOT_SHARE of that slice, and the set's Arrow size never exceeds+# MAX_ARROW_BYTES. Otherwise MAX_BYTES caps the parquet on disk, which decodes to Arrow several+# times its size.DATA_WAREHOUSE_V3_COALESCE_MAX_BATCHES = get_from_env("DATA_WAREHOUSE_V3_COALESCE_MAX_BATCHES", 128, type_cast=int)DATA_WAREHOUSE_V3_COALESCE_MAX_ROWS = get_from_env("DATA_WAREHOUSE_V3_COALESCE_MAX_ROWS", 500_000, type_cast=int)DATA_WAREHOUSE_V3_COALESCE_MAX_BYTES = get_from_env("DATA_WAREHOUSE_V3_COALESCE_MAX_BYTES", 64 * 1024 * 1024, type_cast=int)+DATA_WAREHOUSE_V3_COALESCE_SLOT_SHARE = get_from_env("DATA_WAREHOUSE_V3_COALESCE_SLOT_SHARE", 0.4, type_cast=float)+DATA_WAREHOUSE_V3_COALESCE_MAX_ARROW_BYTES = get_from_env(+ "DATA_WAREHOUSE_V3_COALESCE_MAX_ARROW_BYTES", 512 * 1024 * 1024, type_cast=int+)# Consecutive configuration errors (bad credentials, unknown database, unreachable host) after which# PostHog pauses a destination instead of letting every scheduled run fail against it.diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/coalescing.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/coalescing.pyindex e21ca92dc4e5..2e825678cdef 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/coalescing.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/coalescing.py@@ -24,6 +24,10 @@ if TYPE_CHECKING:COALESCABLE_SYNC_TYPES: frozenset[str] = frozenset({"incremental", "append", "full_refresh"})# A full refresh starts over at batch 0 and appends after it, so its runs never share a write.CROSS_RUN_SYNC_TYPES: frozenset[str] = frozenset({"incremental", "append"})+# The loader holds a set decoded in Arrow, then copies it once more while it deduplicates and+# partitions, so the memory a set takes is about twice its Arrow size.+_ARROW_COPIES_HELD = 2+_MB = 1024 * 1024@frozen@@ -62,6 +66,8 @@ class CoalesceMember:is_resume: boolrow_count: intbyte_size: int+ # Decoded Arrow size the producer recorded. None for a batch queued without it.+ arrow_bytes: int | Noneshape: LoadShape@classmethod@@ -76,6 +82,7 @@ class CoalesceMember:is_resume=batch.is_resume,row_count=batch.row_count,byte_size=batch.byte_size,+ arrow_bytes=_arrow_bytes(metadata.get("arrow_bytes")),shape=LoadShape(sync_type=batch.sync_type,resource_name=batch.resource_name,@@ -102,6 +109,8 @@ class CoalesceMember:is_resume=signal.is_resume,row_count=signal.row_count,byte_size=signal.byte_size,+ # The sink checks only the join rules, which never read a size.+ arrow_bytes=None,shape=LoadShape(sync_type=signal.sync_type,resource_name=signal.resource_name,@@ -120,21 +129,43 @@ class CoalesceMember:@frozenclass CoalesceCaps:- """The size a set may grow to. Rows and bytes are the real bounds; the count is a backstop."""+ """The size a set may grow to. Rows and bytes are the real bounds; the count is a backstop.++ `max_arrow_bytes` bounds the decoded size of a set whose members all recorded it. A set with a+ member of unknown size, or a pod whose memory slice is unknown, is bounded by `max_bytes` of+ parquet on disk instead.+ """max_batches: intmax_rows: intmax_bytes: int+ max_arrow_bytes: int | None@classmethod- def from_settings(cls) -> CoalesceCaps:+ def from_settings(cls, slot_budget_mb: float | None) -> CoalesceCaps:+ """`slot_budget_mb` is the memory slice the governor gives one upsert, or None when unknown."""return cls(max_batches=settings.DATA_WAREHOUSE_V3_COALESCE_MAX_BATCHES,max_rows=settings.DATA_WAREHOUSE_V3_COALESCE_MAX_ROWS,max_bytes=settings.DATA_WAREHOUSE_V3_COALESCE_MAX_BYTES,+ max_arrow_bytes=_arrow_budget(slot_budget_mb),)+def _arrow_budget(slot_budget_mb: float | None) -> int | None:+ if slot_budget_mb is None or slot_budget_mb <= 0:+ return None+ held_bytes = slot_budget_mb * _MB * settings.DATA_WAREHOUSE_V3_COALESCE_SLOT_SHARE+ return min(int(held_bytes / _ARROW_COPIES_HELD), settings.DATA_WAREHOUSE_V3_COALESCE_MAX_ARROW_BYTES)+++def _arrow_bytes(value: object) -> int | None:+ # The metadata is free-form JSON, so a value of any other shape counts as unknown.+ if isinstance(value, bool) or not isinstance(value, int) or value < 0:+ return None+ return value++def _key_tuple(keys: Sequence[str] | None) -> tuple[str, ...] | None:return None if keys is None else tuple(keys)@@ -192,9 +223,13 @@ def extends_set(current: Sequence[CoalesceMember], candidate: CoalesceMember, careturn Falseif len(current) >= caps.max_batches:return False- rows = sum(member.row_count for member in current) + candidate.row_count- size = sum(member.byte_size for member in current) + candidate.byte_size- return rows <= caps.max_rows and size <= caps.max_bytes+ members = [*current, candidate]+ if sum(member.row_count for member in members) > caps.max_rows:+ return False+ arrow_sizes = [member.arrow_bytes for member in members if member.arrow_bytes is not None]+ if caps.max_arrow_bytes is not None and len(arrow_sizes) == len(members):+ return sum(arrow_sizes) <= caps.max_arrow_bytes+ return sum(member.byte_size for member in members) <= caps.max_bytesdef set_violation(members: Sequence[CoalesceMember]) -> str | None:diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/consumer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/consumer.pyindex a0bec5eb2831..1a7a36cdde1d 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/consumer.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/consumer.py@@ -36,6 +36,7 @@ from products.warehouse_sources.backend.temporal.data_imports.metrics import (from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.auto_widen_resync import (COLUMN_TYPE_WIDENED_KEY,)+from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.memory_governor import get_governorfrom products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.coalescing import (CoalesceCaps,CoalesceMember,@@ -1050,7 +1051,7 @@ class DeltaBatchConsumerAdapter:Only adjacent batches join, so every set keeps the claim order: the order the loader wouldhave taken the batches in one at a time."""- caps = CoalesceCaps.from_settings()+ caps = CoalesceCaps.from_settings(slot_budget_mb=get_governor().slot_budget_mb())sets: list[list[PendingBatch]] = []current: list[PendingBatch] = []current_members: list[CoalesceMember] = []diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/producer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/producer.pyindex c3a5541f0cff..5382fee5b43f 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/producer.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/producer.py@@ -316,6 +316,8 @@ class PostgresProducer:if self._workflow_run_id is not None:metadata["workflow_run_id"] = self._workflow_run_idmetadata["timestamp_ns"] = batch_result.timestamp_ns+ if batch_result.arrow_bytes is not None:+ metadata["arrow_bytes"] = batch_result.arrow_bytesif self._external_destination_ids is not None:metadata["external_destination_ids"] = self._external_destination_idsdiff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_consumer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_consumer.pyindex 4d208067ab8d..94407679fdaf 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_consumer.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_consumer.py@@ -1,6 +1,8 @@import mathimport timeimport asyncio+import dataclasses+from collections.abc import Iteratorfrom datetime import UTC, datetimefrom typing import Any, NamedTuple, cast@@ -22,6 +24,12 @@ from products.warehouse_sources.backend.temporal.data_imports.metrics import LOCfrom products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.errors import (TransientObjectStoreError,)+from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.memory_governor import (+ GovernorConfig,+ MemoryGovernor,+ PodMemory,+ reset_governor_for_tests,+)from products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.destinations_load.errors import (DestinationConfigurationError,DestinationDeliveryError,@@ -4218,10 +4226,42 @@ def _run_batches(count: int, *, run_uuid: str = "run-1", start: int = 0, **overr]+_MIB = 1024 * 1024+++class _UnlimitedPod(PodMemory):+ def limit_mb(self) -> float | None:+ return None+++def _use_pod_slot(slot_mb: float | None) -> None:+ # One upsert per pod with no safety margin or reserve, so the slice equals the pod limit.+ config = GovernorConfig(max_concurrent=1, safety=1.0, reserve_mb=0.0, rss_sample_ms=0.0)+ pod = _UnlimitedPod() if slot_mb is None else PodMemory(limit_override_mb=slot_mb)+ reset_governor_for_tests(MemoryGovernor(config=config, pod=pod))+++def _sized_batches(sizes: list[tuple[object, int]], **overrides: Any) -> list[PendingBatch]:+ batches = _run_batches(len(sizes), **overrides)+ return [+ dataclasses.replace(+ batch,+ byte_size=byte_size,+ metadata={} if arrow_bytes is None else {"arrow_bytes": arrow_bytes},+ )+ for batch, (arrow_bytes, byte_size) in zip(batches, sizes)+ ]++class TestCoalesceGroup:"""Which consecutive batches the Delta sink may load as one write. A set that crosses a run, skipsan index or grows past the caps would merge rows the writer must never see together."""+ @pytest.fixture(autouse=True)+ def _restore_governor(self) -> Iterator[None]:+ yield+ reset_governor_for_tests(None)+def _sets(self, batches: list[PendingBatch]) -> list[list[int]]:return [[b.batch_index for b in s] for s in DeltaBatchConsumerAdapter().coalesce_group(batches)]@@ -4375,6 +4415,69 @@ class TestCoalesceGroup:with override_settings(**settings_overrides):assert self._sets(_run_batches(count, **overrides)) == expected+ @pytest.mark.parametrize(+ "slot_mb,sizes,expected",+ [+ # A 64 MiB slice at share 1.0 leaves 32 MiB of Arrow for the two copies the loader holds.+ (64, [(10 * _MIB, 1)] * 5, [[0, 1, 2], [3, 4]]),+ # Wide rows that compress badly join past the parquet cap while they fit the memory budget.+ (64, [(10 * _MIB, 30 * _MIB)] * 4, [[0, 1, 2], [3]]),+ # A batch bigger than the whole budget still loads, on its own.+ (64, [(1 * _MIB, 1), (100 * _MIB, 1), (1 * _MIB, 1)], [[0], [1], [2]]),+ # A large slice is still bounded by the hard ceiling.+ (100_000, [(10 * _MIB, 1)] * 5, [[0, 1, 2, 3], [4]]),+ # Narrow batches still coalesce until the row cap.+ (64, [(1024, 1)] * 60, [list(range(50)), list(range(50, 60))]),+ # One batch queued before the producer recorded its size puts the set back on the parquet cap.+ (64, [(10 * _MIB, 30 * _MIB), (None, 30 * _MIB), (10 * _MIB, 30 * _MIB)], [[0, 1], [2]]),+ # The metadata is free-form JSON, so any other shape counts as unknown.+ (64, [("10", 30 * _MIB), (True, 30 * _MIB), (-1, 30 * _MIB)], [[0, 1], [2]]),+ # An unreadable pod limit gives no slice to size against.+ (None, [(10 * _MIB, 30 * _MIB)] * 3, [[0, 1], [2]]),+ # A slice that leaves no memory gives no Arrow budget.+ (0, [(10 * _MIB, 30 * _MIB)] * 3, [[0, 1], [2]]),+ ],+ ids=[+ "wide_stops_at_memory_budget",+ "wide_joins_past_parquet_cap",+ "oversized_batch_alone",+ "ceiling_binds",+ "narrow_stops_at_row_cap",+ "unknown_batch_size_falls_back",+ "invalid_batch_size_falls_back",+ "unknown_slot_falls_back",+ "empty_slot_falls_back",+ ],+ )+ def test_sets_are_capped_by_arrow_memory(+ self, slot_mb: float | None, sizes: list[tuple[object, int]], expected: list[list[int]]+ ):+ _use_pod_slot(slot_mb)+ with override_settings(+ DATA_WAREHOUSE_V3_COALESCE_SLOT_SHARE=1.0, DATA_WAREHOUSE_V3_COALESCE_MAX_ARROW_BYTES=45 * _MIB+ ):+ assert self._sets(_sized_batches(sizes, row_count=10_000)) == expected++ @pytest.mark.parametrize(+ "batches,expected",+ [+ (_sized_batches([(_MIB, 1)] * 2) + _run_batches(1, start=3, metadata={"arrow_bytes": _MIB}), [[0, 1], [3]]),+ (+ _sized_batches([(_MIB, 1)] * 2) + _run_batches(2, run_uuid="run-2", metadata={"arrow_bytes": _MIB}),+ [[0, 1], [0, 1]],+ ),+ (+ _sized_batches([(_MIB, 1)] * 2)+ + _run_batches(2, run_uuid="run-2", is_resume=True, metadata={"arrow_bytes": _MIB}),+ [[0, 1, 0, 1]],+ ),+ ],+ ids=["gap_splits", "fresh_run_splits", "resume_joins"],+ )+ def test_arrow_budget_keeps_the_join_rules(self, batches: list[PendingBatch], expected: list[list[int]]):+ _use_pod_slot(1024)+ assert self._sets(batches) == expected+class _BatchStatus(NamedTuple):batch_id: strdiff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_producer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_producer.pyindex 244a0542b668..343ab6cf73eb 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_producer.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_producer.py@@ -1,4 +1,5 @@import json+import dataclassesfrom typing import Anyimport pytest@@ -109,13 +110,14 @@ class TestPostgresProducerSendBatch:cdc_write_mode="upsert",cdc_table_mode="merge",)- batch_result = _make_batch_result(batch_index=1)+ batch_result = dataclasses.replace(_make_batch_result(batch_index=1), arrow_bytes=4096)producer.send_batch_notification(batch_result)mock = _mock_conn(producer)params = mock.execute.call_args[0][1]metadata = json.loads(params["metadata"])+ assert metadata["arrow_bytes"] == 4096assert metadata["primary_keys"] == ["id"]assert metadata["partition_count"] == 4assert metadata["cdc_write_mode"] == "upsert"diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/s3/common.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/s3/common.pyindex b37137e2d794..8115fc5d140e 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/s3/common.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/s3/common.py@@ -13,7 +13,7 @@ from products.warehouse_sources.backend.temporal.data_imports.util import NonRetlogger = structlog.get_logger(__name__)-@dataclass+@dataclass(frozen=False)class BatchWriteResult:"""Result of writing a batch to S3."""@@ -22,6 +22,8 @@ class BatchWriteResult:byte_size: intbatch_index: inttimestamp_ns: int = field(default_factory=time.time_ns)+ # Decoded size of the batch in Arrow, which the loader uses to size the sets it writes at once.+ arrow_bytes: int | None = Nonedef strip_s3_protocol(s3_path: str) -> str:diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/s3/test_writer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/s3/test_writer.pyindex b96e5d3f6385..1615ed5cdb6e 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/s3/test_writer.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/s3/test_writer.py@@ -6,6 +6,7 @@ import pytestfrom unittest.mock import MagicMock, patchimport pyarrow as pa+import pyarrow.parquet as pqimport botocore.exceptionsfrom fsspec.implementations.memory import MemoryFileSystemfrom parameterized import parameterized@@ -149,9 +150,14 @@ class TestBatchByteSize:result = writer.write_batch(table, 0)- stored_size = store.info(result.s3_path.split("://", 1)[-1])["size"]+ stored_path = result.s3_path.split("://", 1)[-1]+ stored_size = store.info(stored_path)["size"]assert result.byte_size == stored_sizeassert result.byte_size > 0+ # The loader reads the file back into Arrow, so the recorded size must match that, not the parquet.+ with store.open(stored_path, "rb") as f:+ loaded_bytes = pq.read_table(f).nbytes+ assert result.arrow_bytes == pytest.approx(loaded_bytes, rel=0.05, abs=1024)class TestWriteBatchPermissionDenied:diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/s3/writer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/s3/writer.pyindex 1f41bcc9d247..57baec568703 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/s3/writer.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/s3/writer.py@@ -193,6 +193,7 @@ class S3BatchWriter:byte_size=byte_size,batch_index=batch_index,timestamp_ns=timestamp_ns,+ arrow_bytes=pa_table.nbytes,)def write_schema(self) -> str | None: