:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): cap deltalite row groups on wide merge tables (#120340)
Tom Owers1aed95cf023a
diff --git a/posthog/settings/data_warehouse.py b/posthog/settings/data_warehouse.pyindex 30d7929a66dd..dbca08805e55 100644--- a/posthog/settings/data_warehouse.py+++ b/posthog/settings/data_warehouse.py@@ -165,6 +165,14 @@ DATA_WAREHOUSE_FULL_VACUUM_INTERVAL_HOURS = get_from_env(# `optimize.compact`. deltalite streams each bin under the load slot's memory budget. A deltalite build# without `compact`, a table that deltalite refuses, or a deltalite failure falls back to delta-rs.DATA_WAREHOUSE_DELTALITE_COMPACTION = get_from_env("DATA_WAREHOUSE_DELTALITE_COMPACTION", False, type_cast=str_to_bool)+# Row-group cap for the files deltalite writes. A merge upsert closes a row group at this many encoded+# bytes, and a deltalite compaction closes one at this many decoded bytes, so compaction does not join+# the row groups again. A reader fetches one whole row group before it decodes it, so smaller row groups+# lower upsert and query memory on tables with wide rows. 0 turns the cap off and keeps deltalite's+# defaults.+DATA_WAREHOUSE_DELTALITE_MAX_ROW_GROUP_BYTES = get_from_env(+ "DATA_WAREHOUSE_DELTALITE_MAX_ROW_GROUP_BYTES", 16 * 1024 * 1024, type_cast=int+)# A rebuild run (first sync, reset, full refresh) compacts its own files once a compaction would remove# this many, so a long rebuild does not leave one post-load pass that outlasts the stuck-batch timeout.DATA_WAREHOUSE_REBUILD_COMPACT_MIN_REMOVABLE_FILES = get_from_env(diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/maintenance.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/maintenance.pyindex cca8c1c6c0dd..be182b8d6390 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/maintenance.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/maintenance.py@@ -587,6 +587,11 @@ class DeltaMaintenance:"max_bins_per_commit": COMPACT_MAX_BINS_PER_COMMIT,"partitions": partitions,}+ # Without this, compaction writes row groups up to deltalite's 128 MiB default and so joins again+ # the row groups that the upsert cap split.+ max_row_group_bytes = settings.DATA_WAREHOUSE_DELTALITE_MAX_ROW_GROUP_BYTES+ if max_row_group_bytes > 0:+ compact_kwargs["max_row_group_decoded_bytes"] = max_row_group_bytesdef _run() -> dict[str, Any]:if identity is None:@@ -605,6 +610,7 @@ class DeltaMaintenance:compact_target_size=target_size,compact_max_parallel_bins=compact_kwargs["max_parallel_bins"],compact_slot_budget_mb=round(slot_budget_mb, 1) if slot_budget_mb is not None else None,+ compact_max_row_group_decoded_bytes=compact_kwargs.get("max_row_group_decoded_bytes"),pod_memory_mb=pod_mb_before,peak_rss_mb=_peak_rss_mb(),)diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_maintenance.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_maintenance.pyindex 8e47bb00f072..9131bfebf3f3 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_maintenance.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_maintenance.py@@ -629,10 +629,47 @@ class TestDeltaliteCompaction:@parameterized.expand([- ("committed", 1, 152, [152], {}, DEFAULT_COMPACT_TARGET_SIZE_BYTES),- ("nothing_to_rewrite", 0, 150, [], {}, DEFAULT_COMPACT_TARGET_SIZE_BYTES),- ("table_target", 1, 152, [152], {"delta.targetFileSize": str(32 * _MB)}, 32 * _MB),- ("invalid_table_target", 1, 152, [152], {"delta.targetFileSize": "-1"}, DEFAULT_COMPACT_TARGET_SIZE_BYTES),+ (+ "committed",+ 1,+ 152,+ [152],+ {},+ DEFAULT_COMPACT_TARGET_SIZE_BYTES,+ 16 * _MB,+ {"max_row_group_decoded_bytes": 16 * _MB},+ ),+ (+ "nothing_to_rewrite",+ 0,+ 150,+ [],+ {},+ DEFAULT_COMPACT_TARGET_SIZE_BYTES,+ 16 * _MB,+ {"max_row_group_decoded_bytes": 16 * _MB},+ ),+ (+ "table_target",+ 1,+ 152,+ [152],+ {"delta.targetFileSize": str(32 * _MB)},+ 32 * _MB,+ 16 * _MB,+ {"max_row_group_decoded_bytes": 16 * _MB},+ ),+ (+ "invalid_table_target",+ 1,+ 152,+ [152],+ {"delta.targetFileSize": "-1"},+ DEFAULT_COMPACT_TARGET_SIZE_BYTES,+ 16 * _MB,+ {"max_row_group_decoded_bytes": 16 * _MB},+ ),+ ("row_group_cap_off", 1, 152, [152], {}, DEFAULT_COMPACT_TARGET_SIZE_BYTES, 0, {}),])@pytest.mark.asyncio@@ -644,13 +681,18 @@ class TestDeltaliteCompaction:expected_noted: list[int],configuration: dict[str, str],expected_target: int,+ row_group_cap: int,+ expected_row_group_kwargs: dict[str, int],) -> None:compact = MagicMock(return_value={"commits": commits, "version": version, "numFilesAdded": 3, "numFilesRemoved": 300})maintenance, table_ref, fake_class = self._setup(compact, configuration=configuration)- with patch(f"{_MAINTENANCE_MODULE}._sample_compression_ratio", side_effect=AssertionError("sampled")):+ with (+ patch(f"{_MAINTENANCE_MODULE}._sample_compression_ratio", side_effect=AssertionError("sampled")),+ override_settings(DATA_WAREHOUSE_DELTALITE_MAX_ROW_GROUP_BYTES=row_group_cap),+ ):ran, delta_rs_compact, plan, capture = await self._run(maintenance, fake_class)assert ran is True@@ -666,6 +708,7 @@ class TestDeltaliteCompaction:min_partition_removable_files=1,max_bins_per_commit=COMPACT_MAX_BINS_PER_COMMIT,partitions=None,+ **expected_row_group_kwargs,)assert [call.args[0] for call in table_ref.note_deltalite_commit.call_args_list] == expected_noted[done] = self._info_calls(table_ref, "compact: done")diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_writer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_writer.pyindex 2ca0d9467acd..9429f7a6bd9b 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_writer.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_writer.py@@ -1,5 +1,6 @@import osimport json+import randomfrom decimal import Decimalfrom pathlib import Pathfrom types import SimpleNamespace@@ -8,9 +9,12 @@ from typing import Any, Literal, castimport pytestfrom unittest.mock import AsyncMock, MagicMock, patch+from django.test import override_settings+import pyarrow as paimport deltalakeimport pyarrow.compute as pc+import pyarrow.parquet as pqfrom parameterized import parameterizedfrom pytest_mock import MockerFixture@@ -1287,8 +1291,17 @@ class TestDeltaliteWritePath:assert _committed_version(stats) is Noneassert _committed_file_count_change(stats) is None+ @parameterized.expand(+ [+ ("row_group_cap_on", 16 * 1024 * 1024, 16 * 1024 * 1024),+ ("row_group_cap_off", 0, None),+ ("negative_cap_is_off", -1, None),+ ]+ )@pytest.mark.asyncio- async def test_writes_via_deltalite_when_enabled(self):+ async def test_writes_via_deltalite_when_enabled(+ self, _name: str, row_group_cap: int, expected_max_row_group_bytes: int | None+ ):logger = make_logger() # captured so we can inspect the structured log without hitting the typed attrhelper = DeltaTableRef(resource_name="t", job=MagicMock(team_id=2, schema_id="sch-1"), logger=logger)existing = MagicMock()@@ -1305,6 +1318,7 @@ class TestDeltaliteWritePath:patch.dict("sys.modules", {"deltalite": fake_deltalite}),patch.object(helper, "_get_delta_table_uri", AsyncMock(return_value="s3://b/t")),patch.object(helper, "_get_credentials", return_value={"AWS_REGION": "us-east-1"}),+ override_settings(DATA_WAREHOUSE_DELTALITE_MAX_ROW_GROUP_BYTES=row_group_cap),):wrote = await DeltaWriter(helper)._write_via_deltalite(existing_delta_table=existing,@@ -1316,6 +1330,8 @@ class TestDeltaliteWritePath:assert wrote is Truefake_deltalite.DeltaLiteTable.open.assert_called_once_with("s3://b/t", {"AWS_REGION": "us-east-1"})fake_table.upsert.assert_called_once()+ assert fake_table.upsert.call_args.kwargs["max_row_group_bytes"] == expected_max_row_group_bytes+ assert "isolate_source_rows" not in fake_table.upsert.call_args.kwargs# PARTITION_KEY is passed as the partition arg when the table is partitioned.assert fake_table.upsert.call_args.args[2] == PARTITION_KEY# The delta-rs handle is not refreshed on the write path (a log listing per batch); the ref@@ -1330,6 +1346,49 @@ class TestDeltaliteWritePath:assert log_kwargs["partitions_touched"] == 1assert "duration_ms" in log_kwargs+ @pytest.mark.parametrize("row_group_cap", [256 * 1024, 0], ids=["row_group_cap_on", "row_group_cap_off"])+ @pytest.mark.asyncio+ async def test_row_group_cap_reaches_the_real_upsert_and_compaction(+ self, row_group_cap: int, tmp_path: Path+ ) -> None:+ payload = random.Random(0).randbytes+ delta_path = str(tmp_path / "table")+ deltalake.write_deltalake(delta_path, pa.table({"id": pa.array([0], pa.int64()), "v": [payload(32 * 1024)]}))+ helper = make_local_table_ref(delta_path)+ batch = pa.table({"id": pa.array(range(1, 49), pa.int64()), "v": [payload(32 * 1024) for _ in range(48)]})++ def row_group_sizes() -> list[int]:+ sizes = []+ for uri in deltalake.DeltaTable(delta_path).file_uris():+ metadata = pq.ParquetFile(uri).metadata+ for index in range(metadata.num_row_groups):+ row_group = metadata.row_group(index)+ sizes.append(sum(row_group.column(c).total_compressed_size for c in range(row_group.num_columns)))+ return sizes++ with override_settings(DATA_WAREHOUSE_DELTALITE_MAX_ROW_GROUP_BYTES=row_group_cap):+ wrote = await DeltaWriter(helper)._write_via_deltalite(+ existing_delta_table=deltalake.DeltaTable(delta_path),+ data=batch,+ normalized_primary_keys=["id"],+ use_partitioning=False,+ commit_metadata=None,+ )+ assert wrote is True+ after_upsert = row_group_sizes()+ compacted = await DeltaMaintenance(helper)._compact_with_deltalite(deltalake.DeltaTable(delta_path))+ assert compacted is True+ after_compaction = row_group_sizes()++ batch_bytes = 48 * 32 * 1024+ if row_group_cap:+ # The cap is checked between written batches, so a row group passes it by at most one batch.+ for sizes in (after_upsert, after_compaction):+ assert len(sizes) > 2 and max(sizes) <= 2 * row_group_cap+ else:+ assert max(after_upsert) >= batch_bytes and len(after_compaction) == 1+ assert deltalake.DeltaTable(delta_path).to_pyarrow_table().num_rows == 49+@pytest.mark.asyncioasync def test_sizes_the_upsert_by_the_files_it_rewrites(self, tmp_path: Path):existing_data = pa.table(diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/writer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/writer.pyindex 2f3e42b4d94b..38fa23eba368 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/writer.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/writer.py@@ -5,6 +5,8 @@ import contextlibfrom collections.abc import Callable, Sequencefrom typing import TYPE_CHECKING, Any, Literal+from django.conf import settings+import pyarrow as paimport deltalakeimport pyarrow.compute as pc@@ -333,6 +335,7 @@ class DeltaWriter:if governor.config.mode != "off"else None)+ max_row_group_bytes = settings.DATA_WAREHOUSE_DELTALITE_MAX_ROW_GROUP_BYTESasync with governor.admit(source_bytes=data.nbytes, n_partitions=n_partitions, rewrite=rewrite) as adm:def _run_upsert(table: Any, upsert_kwargs: dict[str, int]) -> Any:@@ -341,6 +344,7 @@ class DeltaWriter:list(normalized_primary_keys),partition_key,commit_metadata=commit_metadata,+ max_row_group_bytes=max_row_group_bytes if max_row_group_bytes > 0 else None,**upsert_kwargs,)@@ -424,6 +428,7 @@ class DeltaWriter:governor_rss_samples=adm.rss.samples if adm.rss else None,governor_concurrent_upserts=adm.concurrent_upserts,governor_max_concurrent_upserts=adm.rss.max_concurrent if adm.rss else None,+ row_group_cap_bytes=max_row_group_bytes,**_deltalite_write_stats(stats),)DELTALITE_WRITE_TOTAL.labels(outcome="written").inc()diff --git a/pyproject.toml b/pyproject.tomlindex 65749b80b297..2c461053b109 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -214,7 +214,7 @@ dependencies = ["braintrust-core>=0.0.59,<1","wrapt==1.17.3","limits==5.8.0",- "deltalite==0.1.11",+ "deltalite==0.1.12","posthog-hogland==0.4.1","requests-oauthlib~=2.0.0",]diff --git a/uv.lock b/uv.lockindex 390c4454d298..a78bf098ac85 100644--- a/uv.lock+++ b/uv.lock@@ -25,19 +25,19 @@ conflicts = [[]][options]-exclude-newer = "2026-10-09T14:35:12.109977Z"+exclude-newer = "0001-01-01T00:00:00Z" # This has no effect and is included for backwards compatibility when using relative exclude-newer values.exclude-newer-span = "P7D"[options.exclude-newer-package]-pixelhog = false-modal = false-posthoganalytics = false+deltalite = falsedjango = falsegoogle-ads = false-deltalite = falsehogql-parser = false-posthog-hogland = falsehogql-parser-rs = false+modal = false+pixelhog = false+posthog-hogland = false+posthoganalytics = false[manifest]members = [@@ -1829,16 +1829,16 @@ pyarrow = [[[package]]name = "deltalite"-version = "0.1.11"+version = "0.1.12"source = { registry = "https://pypi.org/simple" }-sdist = { url = "https://files.pythonhosted.org/packages/f9/c1/85d795c164d60ed1a3395db326fd43e50d3506bbfc685372a1d46a84ca67/deltalite-0.1.11.tar.gz", hash = "sha256:69a6720829a94d8ba078b8ae08657172bf85ad6b84e044c5d56e98efcb45d090", size = 295605, upload-time = "2026-10-07T10:25:29.142Z" }+sdist = { url = "https://files.pythonhosted.org/packages/21/ad/bcf1451d9513de77d187e1a6d11c9528cf539675bb421bc68061da57c112/deltalite-0.1.12.tar.gz", hash = "sha256:ac62d8186e4db265d5ffca20df66dc2fc1d624e05492148599e868499cc6a3cc", size = 300564, upload-time = "2026-10-20T13:57:57.531Z" }wheels = [- { url = "https://files.pythonhosted.org/packages/b0/4a/3f6b4d8e10c4024652322f824847728d6720b5bfa562d6d477af87379238/deltalite-0.1.11-cp312-abi3-macosx_10_12_x86_64.whl", hash = "sha256:41cf4ca5af5adfca9ab27e57faa977fff98a42547bb42b757acad7f37f2fd35c", size = 18463623, upload-time = "2026-10-07T10:25:10.442Z" },- { url = "https://files.pythonhosted.org/packages/38/78/03a4faf78885264882a9f0ec09ce6b465adb7020e887e736e2de3a58b80f/deltalite-0.1.11-cp312-abi3-macosx_11_0_arm64.whl", hash = "sha256:788d7e4b2004a286df42be17788c06e920dfb890f3d007050ce44688adf4aa84", size = 17938315, upload-time = "2026-10-07T10:25:13.266Z" },- { url = "https://files.pythonhosted.org/packages/79/7c/abab0f6814046ba998dd44ad19a7bdb3f8fca06087f28582b857f4b63bcd/deltalite-0.1.11-cp312-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:27c7603529dfd8220fc23940f42acdec424b810a1f161cb1a97859cb839bc0de", size = 64511410, upload-time = "2026-10-07T10:25:16.391Z" },- { url = "https://files.pythonhosted.org/packages/c1/87/a290b5b16967fe6609fa83e37a3854b6a7b0133aeda25d40c07762f7483a/deltalite-0.1.11-cp312-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:81f3d808eb8d628c211696a55b9b6ebf8b51f91df1d13112e0d9a0ea708308f3", size = 67403553, upload-time = "2026-10-07T10:25:19.82Z" },- { url = "https://files.pythonhosted.org/packages/75/fb/25d649290699b9398130a3f4fa3a22adf6850874e34fd6a10cb36c0ee444/deltalite-0.1.11-cp312-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:8bc04caa0b3b494180b7fdff8c3082641c9cee6b974613be9e1502873d4723f7", size = 64684529, upload-time = "2026-10-07T10:25:23.122Z" },- { url = "https://files.pythonhosted.org/packages/5f/19/dbe6170af468c81a3b4d65ec268a7983c15a29a1c2cae2b1e9cc8da72153/deltalite-0.1.11-cp312-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:b92a2b957c489336526901e5d2a63b98c620d5a82738851271c6452d72e4fcf1", size = 65706707, upload-time = "2026-10-07T10:25:26.804Z" },+ { url = "https://files.pythonhosted.org/packages/38/d6/21308bf1d8c8c775373eaa94ac481df68e8a66a0985d232b522070988d0a/deltalite-0.1.12-cp312-abi3-macosx_10_12_x86_64.whl", hash = "sha256:9c74b350479b83239351d837023c4e81bc6b31d4e59e5da02b7d2dd7d25a5210", size = 18459027, upload-time = "2026-10-20T13:57:36.784Z" },+ { url = "https://files.pythonhosted.org/packages/59/a9/e049492f0d63c22b0ef402a162fed670674f4d343af4f4c0992b8d036132/deltalite-0.1.12-cp312-abi3-macosx_11_0_arm64.whl", hash = "sha256:27b704274b8cac4975320cabba25667e16cd562faf45d683051eafeaa16a5c52", size = 17924216, upload-time = "2026-10-20T13:57:39.66Z" },+ { url = "https://files.pythonhosted.org/packages/79/35/6b3fc36a43d08da33a9a9f906fbbca87ef03ce3a271d3e5914746a9a1eef/deltalite-0.1.12-cp312-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:143f15bb98641b8fc174f569e9bbd929f517dc02c4b0c6f3191d30fdb7767ae8", size = 64522984, upload-time = "2026-10-20T13:57:42.734Z" },+ { url = "https://files.pythonhosted.org/packages/a5/7b/3209dfec1c9b7d8dc9eac26e84c4e65fa7b259c98e3b4bb12734d753dfff/deltalite-0.1.12-cp312-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:5e37ae14d860256645ef6048e4684a1baf2c8605a267414a9514efa8c1f22b9e", size = 67427768, upload-time = "2026-10-20T13:57:46.922Z" },+ { url = "https://files.pythonhosted.org/packages/2d/80/f0e19f43ad1f0a0ac44deee50b77c6bfad7e5b9d32be485c398f5f0462e6/deltalite-0.1.12-cp312-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:d5edebf257ba63329cc19b098f9943f1c7c1b22188d5088ac020be26b255215b", size = 64668300, upload-time = "2026-10-20T13:57:50.886Z" },+ { url = "https://files.pythonhosted.org/packages/9d/31/afc82195f4c41e7f30218e8d591fbcdd70d95a2ab93e02c7534b88c5889b/deltalite-0.1.12-cp312-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:5e9ca5091aa30b822b06aca13e75b0c21ba495fca2060ed09337c13065c10416", size = 65700242, upload-time = "2026-10-20T13:57:54.816Z" },][[package]]@@ -6154,7 +6154,7 @@ requires-dist = [{ name = "databricks-sql-connector", specifier = "~=4.3.0" },{ name = "defusedxml", specifier = "~=0.7.1" },{ name = "deltalake", extras = ["pyarrow"], specifier = "==1.6.1" },- { name = "deltalite", specifier = "==0.1.11" },+ { name = "deltalite", specifier = "==0.1.12" },{ name = "disposable-email-domains", specifier = "~=0.0.140" },{ name = "dj-database-url", specifier = "~=3.0.0" },{ name = "django", specifier = "~=5.2.0" },