:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-modeling): cap deltalite row groups on incremental model writes (#120449)
Tom Owers769074f83423
diff --git a/posthog/temporal/data_modeling/activities/incremental_write.py b/posthog/temporal/data_modeling/activities/incremental_write.pyindex b3ef8aeb704a..bb0aa248cb4a 100644--- a/posthog/temporal/data_modeling/activities/incremental_write.py+++ b/posthog/temporal/data_modeling/activities/incremental_write.py@@ -22,6 +22,8 @@ stays on delta-rs.from typing import Any, Optional+from django.conf import settings+import pyarrow as paimport pyarrow.compute as pc@@ -163,12 +165,14 @@ def upsert_batch("""import deltalite+ max_row_group_bytes = settings.DATA_WAREHOUSE_DELTALITE_MAX_ROW_GROUP_BYTESlite = deltalite.DeltaLiteTable.open(table_uri, storage_options)return lite.upsert(pa.Table.from_batches([batch]),list(unique_key),None,commit_metadata=commit_metadata,+ max_row_group_bytes=max_row_group_bytes if max_row_group_bytes > 0 else None,)diff --git a/posthog/temporal/tests/data_modeling/test_incremental_write.py b/posthog/temporal/tests/data_modeling/test_incremental_write.pyindex 61d64184cb8c..676f9910adfc 100644--- a/posthog/temporal/tests/data_modeling/test_incremental_write.py+++ b/posthog/temporal/tests/data_modeling/test_incremental_write.py@@ -6,10 +6,17 @@ column missing from the output)."""import pytest+from unittest.mock import MagicMock, patch++from django.test import override_settingsimport pyarrow as pa-from posthog.temporal.data_modeling.activities.incremental_write import IncrementalWriteError, UniqueKeyTracker+from posthog.temporal.data_modeling.activities.incremental_write import (+ IncrementalWriteError,+ UniqueKeyTracker,+ upsert_batch,+)def _batch(ids: list[int | None], names: list[str | None]) -> pa.RecordBatch:@@ -70,3 +77,28 @@ def test_exceeding_the_memory_cap_fails_loudly() -> None:with pytest.raises(IncrementalWriteError, match="too many unique keys"):tracker.check(_batch([1, 2, 3, 4], ["a", "b", "c", "d"]))+++ "row_group_cap,expected",+ [+ pytest.param(16 * 1024 * 1024, 16 * 1024 * 1024, id="cap_on"),+ pytest.param(0, None, id="zero_is_off"),+ pytest.param(-1, None, id="negative_is_off"),+ ],+)+def test_upsert_batch_passes_the_row_group_cap(row_group_cap: int, expected: int | None) -> None:+ fake_table = MagicMock()+ fake_deltalite = MagicMock()+ fake_deltalite.DeltaLiteTable.open.return_value = fake_table++ with (+ patch.dict("sys.modules", {"deltalite": fake_deltalite}),+ override_settings(DATA_WAREHOUSE_DELTALITE_MAX_ROW_GROUP_BYTES=row_group_cap),+ ):+ upsert_batch("s3://b/t", {}, _batch([1], ["a"]), ("id",), commit_metadata={"run_uuid": "abc"})++ fake_table.upsert.assert_called_once()+ kwargs = fake_table.upsert.call_args.kwargs+ assert kwargs["max_row_group_bytes"] == expected+ assert kwargs["commit_metadata"] == {"run_uuid": "abc"}