:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
perf(data-warehouse): stop copying deduped loader batches twice (#121891)
Tom Owersb041798b8ea6
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/arrow_utils.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/arrow_utils.pyindex 52669084b96f..56b9cf4a37f1 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/arrow_utils.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/arrow_utils.py@@ -1904,6 +1904,9 @@ def first_per_pk_table(pa_table: pa.Table, pk_columns: list[str], keep: Literal[the latest occurrence of a key carries the freshest data. Either way the mergereceives at most one source row per key, avoiding ambiguous multi-match mergesemantics (and the duplicate inserts `when_not_matched_insert_all` would produce).++ When no PK tuple repeats, the input table itself is returned, so the caller does not hold a+ second copy of it."""if not pk_columns or pa_table.num_rows == 0:return pa_table@@ -1923,6 +1926,10 @@ def first_per_pk_table(pa_table: pa.Table, pk_columns: list[str], keep: Literal[# 2. Group by PK, keeping only one position per PKgrouped = indexed.group_by(pk_columns).aggregate([(idx_col_name, aggregate)])+ # One group per row keeps every row in its place, and `take` would only copy the whole table.+ if grouped.num_rows == pa_table.num_rows:+ return pa_table+# 3. Sort those positions ascending so the output mirrors the input row orderkept_indices = np.sort(grouped.column(f"{idx_col_name}_{aggregate}").to_numpy())diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_writer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_writer.pyindex 4c481b16b144..c844849864c9 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_writer.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_writer.py@@ -57,6 +57,7 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.del_deltalite_write_stats,_merge_predicate_ops,commit_covers_batch,+ dedupe_incremental_batch,)_REPARTITION_MODULE = "products.warehouse_sources.backend.temporal.data_imports.pipelines.core.repartition"@@ -755,17 +756,83 @@ class TestIncrementalBatchDeduplication:@parameterized.expand([- ("keep_first", "first", ["a1", "b1"]),- ("keep_last", "last", ["a2", "b1"]),+ ("unique_key", [{"id": [1, 2, 3], "k2": ["a", "a", "a"]}], ["id"], "last", [0, 1, 2]),+ ("unique_composite_key", [{"id": [1, 1, 2], "k2": ["a", "b", "a"]}], ["id", "k2"], "last", [0, 1, 2]),+ (+ "unique_across_chunks",+ [{"id": [1, 2], "k2": ["a", "a"]}, {"id": [3], "k2": ["a"]}],+ ["id"],+ "last",+ [0, 1, 2],+ ),+ ("null_in_one_composite_part", [{"id": [1, 1], "k2": [None, "a"]}], ["id", "k2"], "last", [0, 1]),+ ("repeated_key_keeps_last", [{"id": [1, 2, 1, 3], "k2": ["a", "a", "a", "a"]}], ["id"], "last", [1, 2, 3]),+ (+ "repeated_key_keeps_first",+ [{"id": [1, 2, 1, 3], "k2": ["a", "a", "a", "a"]}],+ ["id"],+ "first",+ [0, 1, 3],+ ),+ ("repeated_composite_key", [{"id": [1, 1, 1], "k2": ["a", "b", "a"]}], ["id", "k2"], "last", [1, 2]),+ (+ "repeated_across_chunks",+ [{"id": [1, 2], "k2": ["a", "a"]}, {"id": [2, 3], "k2": ["a", "a"]}],+ ["id"],+ "last",+ [0, 2, 3],+ ),+ ("null_keys_are_one_key", [{"id": [None, 1, None], "k2": ["a", "a", "a"]}], ["id"], "last", [1, 2]),+ ]+ )+ def test_first_per_pk_table(+ self,+ _name: str,+ chunks: list[dict[str, list[Any]]],+ keys: list[str],+ keep: Literal["first", "last"],+ kept_positions: list[int],+ ) -> None:+ table = pa.concat_tables(+ [pa.table({**chunk, "pos": range(offset, offset + len(chunk["id"]))}) for offset, chunk in _offsets(chunks)]+ )++ result = first_per_pk_table(table, keys, keep=keep)++ assert result.column("pos").to_pylist() == kept_positions+ # A batch without a repeated key must come back as it is, not as a copy of every row.+ assert (result is table) == (len(kept_positions) == table.num_rows)++ @parameterized.expand([("unique_keys", 0), ("one_repeated_key", 1)])+ def test_dedupe_holds_a_second_copy_only_when_a_key_repeats(self, _name: str, repeats: int) -> None:+ ids = list(range(2_000)) + list(range(repeats))+ table = pa.table({"id": ids, "payload": [f"{i:0>200}" for i in range(len(ids))]})+ before = pa.total_allocated_bytes()++ result = first_per_pk_table(table, ["id"], keep="last")++ copied = pa.total_allocated_bytes() - before+ assert result.num_rows == 2_000+ if repeats:+ assert copied > table.nbytes // 2+ else:+ assert copied < table.nbytes // 10++ @parameterized.expand(+ [+ ("one_partition", ["p0", "p0", "p0", "p0"]),+ ("key_moves_partition", ["p0", "p1", "p0", "p1"]),+ ("every_row_own_partition", ["p0", "p1", "p2", "p3"]),])- def test_first_per_pk_table_keep_modes(self, _name, keep, expected_names):- table = pa.table({"id": [1, 1, 2], "name": ["a1", "a2", "b1"]})+ def test_partition_keyed_pass_keeps_the_rows_of_a_key_only_pass(self, _name: str, partitions: list[str]) -> None:+ table = pa.table({"id": [1, 1, 2, 1], PARTITION_KEY: partitions, "pos": [0, 1, 2, 3]})- result = first_per_pk_table(table, ["id"], keep=keep).sort_by("id")+ key_only, _ = dedupe_incremental_batch(table, ["id"], use_partitioning=False)+ partition_keyed, _ = dedupe_incremental_batch(table, ["id"], use_partitioning=True)+ both, _ = dedupe_incremental_batch(partition_keyed, ["id"], use_partitioning=False)- assert result.column("id").to_pylist() == [1, 2]- assert result.column("name").to_pylist() == expected_names+ assert both.column("pos").to_pylist() == key_only.column("pos").to_pylist() == [2, 3]@pytest.mark.asyncioasync def test_incremental_merge_dedupes_duplicate_source_rows(self, tmp_path: Path) -> None:@@ -808,6 +875,15 @@ class TestIncrementalBatchDeduplication:assert final.column("name").to_pylist() == ["second_copy"]+def _offsets(chunks: list[dict[str, list[Any]]]) -> list[tuple[int, dict[str, list[Any]]]]:+ offsets: list[tuple[int, dict[str, list[Any]]]] = []+ start = 0+ for chunk in chunks:+ offsets.append((start, chunk))+ start += len(chunk["id"])+ return offsets++def _commit_operations(delta_path: str) -> list[str]:return [commit["operation"] for commit in deltalake.DeltaTable(delta_path).history()]diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/writer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/writer.pyindex 16ef8b7523ab..39a2e9ad8cb7 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/writer.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/writer.py@@ -178,6 +178,22 @@ def commit_matches(commit: dict[str, Any], match: dict[str, str]) -> bool:return any(all(layout.get(k) == v for k, v in match.items()) for layout in _commit_metadata_layouts(commit))+def dedupe_incremental_batch(+ data: pa.Table, primary_keys: Sequence[Any], use_partitioning: bool+) -> tuple[pa.Table, list[str]]:+ """Drop all but the last occurrence of each PK (+ partition) tuple in a batch.++ Returns the batch and the keys it was deduplicated on. The batch comes back as it is when no key+ repeats, or when none of `primary_keys` is a column of it.+ """+ dedupe_keys = [n for x in primary_keys if (n := normalize_column_name(x)) in data.column_names]+ if not dedupe_keys:+ return data, []+ if use_partitioning:+ dedupe_keys.append(PARTITION_KEY)+ return first_per_pk_table(data, dedupe_keys, keep="last"), dedupe_keys++def commit_members_tag(members: Sequence[tuple[str, int]]) -> str:"""The `members` commit tag for a set that spans runs: `run_uuid:batch_index` per member."""return ",".join(f"{run_uuid}:{batch_index}" for run_uuid, batch_index in members)@@ -235,14 +251,7 @@ class DeltaWriter:async def _dedupe_incremental_batch(self, data: pa.Table, primary_keys: Sequence[Any], use_partitioning: bool) -> pa.Table:- """Drop all but the last occurrence of each PK (+ partition) tuple in a batch."""- dedupe_keys = [n for x in primary_keys if (n := normalize_column_name(x)) in data.column_names]- if not dedupe_keys:- return data- if use_partitioning:- dedupe_keys.append(PARTITION_KEY)-- deduped = first_per_pk_table(data, dedupe_keys, keep="last")+ deduped, dedupe_keys = dedupe_incremental_batch(data, primary_keys, use_partitioning)dropped = data.num_rows - deduped.num_rowsif dropped > 0:await self._logger.awarning(diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/coalescing.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/coalescing.pyindex cec6757dae40..822acd563140 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/coalescing.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/coalescing.py@@ -24,9 +24,9 @@ if TYPE_CHECKING:COALESCABLE_SYNC_TYPES: frozenset[str] = frozenset({"incremental", "append", "full_refresh"})# A full refresh starts over at batch 0 and appends after it, so its runs never share a write.CROSS_RUN_SYNC_TYPES: frozenset[str] = frozenset({"incremental", "append"})-# The loader holds a set decoded in Arrow, then copies it once more while it deduplicates and-# partitions, so the memory a set takes is about twice its Arrow size.-_ARROW_COPIES_HELD = 2+# The loader holds a set decoded in Arrow through the upsert. Deduplication copies the rows only when+# a key repeats, and the loader frees the set as read before the upsert, so one copy is held.+_ARROW_COPIES_HELD = 1_MB = 1024 * 1024diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/processor.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/processor.pyindex 6f56339a63cf..d3d341c74863 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/processor.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/processor.py@@ -61,6 +61,7 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delfrom products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.writer import (DeltaWriter,commit_members_tag,+ dedupe_incremental_batch,)from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.hogql_schema import HogQLSchemafrom products.warehouse_sources.backend.temporal.data_imports.pipelines.core.partitioning import (@@ -369,6 +370,34 @@ def _apply_partitioning(return pa_table+def _dedupe_before_write(pa_table: pa.Table, primary_keys: list[str]) -> pa.Table:+ """Drop the rows that `DeltaWriter.write` would drop, before the batch goes to the writer.++ The writer deduplicates too, but its caller holds the batch until the write returns. If only+ the writer deduplicated, the batch as read and its deduplicated copy would both stay in memory+ for the whole upsert. The caller rebinds its batch to this result, so the batch as read is freed+ first, and a set holds one Arrow copy of its rows through the write (see `_ARROW_COPIES_HELD`).++ This keys on the partition column whenever the batch has one. The writer keys on it only when+ the table is partitioned by it, so these keys are never fewer than the keys of the writer. The+ last row of each writer key is also the last row of its key here, so the rows that land do not+ change. When both use the same keys, the pass in the writer finds no repeated key and makes no+ copy.+ """+ deduped, dedupe_keys = dedupe_incremental_batch(+ pa_table, primary_keys, use_partitioning=PARTITION_KEY in pa_table.column_names+ )+ dropped = pa_table.num_rows - deduped.num_rows+ if dropped > 0:+ logger.warning(+ "dropped_duplicate_primary_key_rows",+ dropped=dropped,+ dedupe_keys=dedupe_keys,+ batch_rows=pa_table.num_rows,+ )+ return deduped++def _partial_data_loading_applies(export_signal: ExportSignalMessage, schema: ExternalDataSchema) -> bool:"""Whether this batch feeds partial data loading (first-ever sync, Stripe only).@@ -1537,6 +1566,9 @@ def _process_message_reported(with timer.step("ownership_check"):verify_ownership()+ # The row metrics count the batch as read, before the dedupe below drops repeated keys.+ source_rows = pa_table.num_rows+# The writer narrows this to `merge` for an upsert; an append stays `write` to its commit.report_phase("write")if cdc_write_mode == "scd2_append":@@ -1586,6 +1618,8 @@ def _process_message_reported(timer.step("write"),DELTA_WRITE_DURATION_SECONDS.labels(write_type=write_type).time(),):+ if write_type == "incremental" and primary_keys:+ pa_table = _dedupe_before_write(pa_table, primary_keys)delta_writer = DeltaWriter(delta_table_ref)delta_table = async_to_sync(delta_writer.write)(data=pa_table,@@ -1597,7 +1631,7 @@ def _process_message_reported()deltalite_file_count_change = delta_writer.deltalite_file_count_change- DELTA_ROWS_WRITTEN_TOTAL.inc(pa_table.num_rows)+ DELTA_ROWS_WRITTEN_TOTAL.inc(source_rows)# Marked as soon as the commit lands, before post-load: the final row of a run carries its own# data now, so a post-load failure must send the retry down the post-load-only path rather@@ -1640,7 +1674,7 @@ def _process_message_reported(team_id=export_signal.team_id,external_data_schema_id=export_signal.schema_id,batch_index=export_signal.batch_index,- rows_written=pa_table.num_rows,+ rows_written=source_rows,batch_count=len(members),delta_version=delta_table_ref.latest_known_version(delta_table),file_count=file_count,diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/test_processor.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/test_processor.pyindex 3793c93c237d..dfa47674020b 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/test_processor.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/test_processor.py@@ -40,6 +40,7 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.pipelineINCOMPLETE_RUN_ERROR,IncompleteRunError,_apply_partitioning,+ _dedupe_before_write,_enrich_cdc_rows,_file_count_after_write,_get_write_type,@@ -1079,7 +1080,10 @@ class TestProcessMessages:@patch(f"{_PROCESSOR}._trigger_ducklake_register_data_imports")@patch(f"{_PROCESSOR}._mark_job_completed")@patch(f"{_PROCESSOR}.run_post_load_operations", new_callable=AsyncMock, return_value=_POST_LOAD_RESULT)- @patch(f"{_PROCESSOR}.read_parquet", return_value=pa.table({"id": [1]}))+ @patch(+ f"{_PROCESSOR}.read_parquet",+ side_effect=lambda path: pa.table({"id": [int(path.rsplit("/", 1)[1].removesuffix(".parquet"))]}),+ )@patch(f"{_PROCESSOR}.is_batch_already_processed", return_value=False)@patch(f"{_PROCESSOR}.DeltaWriter")@patch(f"{_PROCESSOR}.DeltaTableRef")@@ -1221,6 +1225,86 @@ class TestCrossRunKeyResolution:assert not commit_covers_batch(set_commit, "run-2", 1)+class TestSourceCopiesHeldThroughTheUpsert:+ # Coalesced sets are sized by `_ARROW_COPIES_HELD`. If the loader keeps the set as read alive+ # next to its deduplicated copy, a set takes twice the memory its cap allows.++ _ROWS = 2_000++ @staticmethod+ def _batch(batch_no: int, ids: list[int]) -> pa.Table:+ return pa.table({"id": ids, "payload": [f"{batch_no}:{pos:0>200}" for pos in range(len(ids))]})++ @pytest.mark.parametrize(+ "batch_ids",+ [+ [list(range(0, _ROWS)), list(range(_ROWS, 2 * _ROWS))],+ [[*range(0, _ROWS), 0, 1], list(range(_ROWS, 2 * _ROWS))],+ [list(range(0, _ROWS)), list(range(_ROWS // 2, _ROWS + _ROWS // 2))],+ ],+ ids=["unique_keys", "key_repeated_in_one_batch", "keys_repeated_across_the_set"],+ )+ def test_the_upsert_gets_one_copy_of_the_set_with_the_last_row_of_each_key(+ self, batch_ids: list[list[int]], tmp_path: Path+ ) -> None:+ table_path = str(tmp_path / "table")+ write_deltalake(table_path, self._batch(9, [0]))+ paths = [f"s3://bucket/run-1/{3 + n}.parquet" for n in range(len(batch_ids))]+ expected = pa.concat_tables([self._batch(n, ids) for n, ids in enumerate(batch_ids)]).to_pylist()+ last_row_by_id = {row["id"]: row for row in expected}+ expected_rows = sorted(last_row_by_id.values(), key=lambda row: expected.index(row))+ one_copy_bytes = pa.concat_tables([self._batch(n, ids) for n, ids in enumerate(batch_ids)]).nbytes+ seen: dict[str, Any] = {}++ async def record_upsert_source(writer: DeltaWriter, **kwargs: Any) -> bool:+ seen["held_bytes"] = pa.total_allocated_bytes() - baseline+ seen["rows"] = kwargs["data"].select(["id", "payload"]).to_pylist()+ return False++ with ExitStack() as stack:+ stack.enter_context(patch(f"{_PROCESSOR}.posthoganalytics"))+ stack.enter_context(patch(f"{_PROCESSOR}.mark_batch_as_processed"))+ stack.enter_context(patch(f"{_PROCESSOR}.is_batch_already_processed", return_value=False))+ stack.enter_context(+ patch(+ f"{_PROCESSOR}.read_parquet",+ side_effect=lambda path: self._batch(paths.index(path), batch_ids[paths.index(path)]),+ )+ )+ stack.enter_context(+ patch(f"{_PROCESSOR}.DeltaTableRef", side_effect=lambda **kwargs: make_local_table_ref(table_path))+ )+ stack.enter_context(patch.object(DeltaWriter, "_write_via_deltalite", record_upsert_source))+ job_model = stack.enter_context(patch(f"{_PROCESSOR}.ExternalDataJob"))+ job_model.objects.prefetch_related.return_value.get.return_value = MagicMock()+ baseline = pa.total_allocated_bytes()+ process_messages(_run_messages("run-1", start=3, count=len(batch_ids), final=False))++ assert seen["rows"] == expected_rows+ assert seen["held_bytes"] < one_copy_bytes * 1.5+ landed = DeltaTable(table_path).to_pyarrow_table().select(["id", "payload"]).sort_by("id").to_pylist()+ assert landed == sorted(expected_rows, key=lambda row: row["id"])++ @parameterized.expand(+ [+ ("no_partition_column", None, [1, 2]),+ ("key_in_one_partition", ["p0", "p0", "p0"], [1, 2]),+ # The writer keys on the partition too, so a key that moved partition lands in both.+ ("key_moved_partition", ["p0", "p0", "p1"], [0, 1, 2]),+ ]+ )+ def test_dedupe_before_write_keeps_what_the_writer_keeps(+ self, _name: str, partitions: list[str] | None, kept_positions: list[int]+ ) -> None:+ columns: dict[str, list[Any]] = {"id": [1, 2, 1], "pos": [0, 1, 2]}+ if partitions is not None:+ columns[PARTITION_KEY] = partitions++ result = _dedupe_before_write(pa.table(columns), ["id"])++ assert result.column("pos").to_pylist() == kept_positions++class TestPostImportTrigger:"""The V3 hand-off to `data-import-post-import`: without it the load-dependentpost-import steps (signals, enrichment, statistics, table size, DuckLake copy)diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_consumer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_consumer.pyindex ee27058a656e..607df3eacc52 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_consumer.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_consumer.py@@ -5032,20 +5032,20 @@ class TestCoalesceGroup:@pytest.mark.parametrize("slot_mb,sizes,expected",[- # A 64 MiB slice at share 1.0 leaves 32 MiB of Arrow for the two copies the loader holds.- (64, [(10 * _MIB, 1)] * 5, [[0, 1, 2], [3, 4]]),+ # A 32 MiB slice at share 1.0 leaves 32 MiB of Arrow for the one copy the loader holds.+ (32, [(10 * _MIB, 1)] * 5, [[0, 1, 2], [3, 4]]),# Wide rows that compress badly join past the parquet cap while they fit the memory budget.- (64, [(10 * _MIB, 30 * _MIB)] * 4, [[0, 1, 2], [3]]),+ (32, [(10 * _MIB, 30 * _MIB)] * 4, [[0, 1, 2], [3]]),# A batch bigger than the whole budget still loads, on its own.- (64, [(1 * _MIB, 1), (100 * _MIB, 1), (1 * _MIB, 1)], [[0], [1], [2]]),+ (32, [(1 * _MIB, 1), (100 * _MIB, 1), (1 * _MIB, 1)], [[0], [1], [2]]),# A large slice is still bounded by the hard ceiling.(100_000, [(10 * _MIB, 1)] * 5, [[0, 1, 2, 3], [4]]),# Narrow batches still coalesce until the row cap.- (64, [(1024, 1)] * 60, [list(range(50)), list(range(50, 60))]),+ (32, [(1024, 1)] * 60, [list(range(50)), list(range(50, 60))]),# One batch queued before the producer recorded its size puts the set back on the parquet cap.- (64, [(10 * _MIB, 30 * _MIB), (None, 30 * _MIB), (10 * _MIB, 30 * _MIB)], [[0, 1], [2]]),+ (32, [(10 * _MIB, 30 * _MIB), (None, 30 * _MIB), (10 * _MIB, 30 * _MIB)], [[0, 1], [2]]),# The metadata is free-form JSON, so any other shape counts as unknown.- (64, [("10", 30 * _MIB), (True, 30 * _MIB), (-1, 30 * _MIB)], [[0, 1], [2]]),+ (32, [("10", 30 * _MIB), (True, 30 * _MIB), (-1, 30 * _MIB)], [[0, 1], [2]]),# An unreadable pod limit gives no slice to size against.(None, [(10 * _MIB, 30 * _MIB)] * 3, [[0, 1], [2]]),# A slice that leaves no memory gives no Arrow budget.