Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-modeling): cap deltalite row groups on incremental model writes (#120449)

Tom Owers769074f83423
diff --git a/posthog/temporal/data_modeling/activities/incremental_write.py b/posthog/temporal/data_modeling/activities/incremental_write.py
index b3ef8aeb704a..bb0aa248cb4a 100644
--- a/posthog/temporal/data_modeling/activities/incremental_write.py
+++ b/posthog/temporal/data_modeling/activities/incremental_write.py
@@ -22,6 +22,8 @@ stays on delta-rs.
from typing import Any, Optional
+from django.conf import settings
+
import pyarrow as pa
import pyarrow.compute as pc
@@ -163,12 +165,14 @@ def upsert_batch(
"""
import deltalite
+ max_row_group_bytes = settings.DATA_WAREHOUSE_DELTALITE_MAX_ROW_GROUP_BYTES
lite = deltalite.DeltaLiteTable.open(table_uri, storage_options)
return lite.upsert(
pa.Table.from_batches([batch]),
list(unique_key),
None,
commit_metadata=commit_metadata,
+ max_row_group_bytes=max_row_group_bytes if max_row_group_bytes > 0 else None,
)
diff --git a/posthog/temporal/tests/data_modeling/test_incremental_write.py b/posthog/temporal/tests/data_modeling/test_incremental_write.py
index 61d64184cb8c..676f9910adfc 100644
--- a/posthog/temporal/tests/data_modeling/test_incremental_write.py
+++ b/posthog/temporal/tests/data_modeling/test_incremental_write.py
@@ -6,10 +6,17 @@ column missing from the output).
"""
import pytest
+from unittest.mock import MagicMock, patch
+
+from django.test import override_settings
import pyarrow as pa
-from posthog.temporal.data_modeling.activities.incremental_write import IncrementalWriteError, UniqueKeyTracker
+from posthog.temporal.data_modeling.activities.incremental_write import (
+ IncrementalWriteError,
+ UniqueKeyTracker,
+ upsert_batch,
+)
def _batch(ids: list[int | None], names: list[str | None]) -> pa.RecordBatch:
@@ -70,3 +77,28 @@ def test_exceeding_the_memory_cap_fails_loudly() -> None:
with pytest.raises(IncrementalWriteError, match="too many unique keys"):
tracker.check(_batch([1, 2, 3, 4], ["a", "b", "c", "d"]))
+
+
+ "row_group_cap,expected",
+ [
+ pytest.param(16 * 1024 * 1024, 16 * 1024 * 1024, id="cap_on"),
+ pytest.param(0, None, id="zero_is_off"),
+ pytest.param(-1, None, id="negative_is_off"),
+ ],
+)
+def test_upsert_batch_passes_the_row_group_cap(row_group_cap: int, expected: int | None) -> None:
+ fake_table = MagicMock()
+ fake_deltalite = MagicMock()
+ fake_deltalite.DeltaLiteTable.open.return_value = fake_table
+
+ with (
+ patch.dict("sys.modules", {"deltalite": fake_deltalite}),
+ override_settings(DATA_WAREHOUSE_DELTALITE_MAX_ROW_GROUP_BYTES=row_group_cap),
+ ):
+ upsert_batch("s3://b/t", {}, _batch([1], ["a"]), ("id",), commit_metadata={"run_uuid": "abc"})
+
+ fake_table.upsert.assert_called_once()
+ kwargs = fake_table.upsert.call_args.kwargs
+ assert kwargs["max_row_group_bytes"] == expected
+ assert kwargs["commit_metadata"] == {"run_uuid": "abc"}