Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): cap deltalite row groups on wide merge tables (#120340)

Tom Owers1aed95cf023a
diff --git a/posthog/settings/data_warehouse.py b/posthog/settings/data_warehouse.py
index 30d7929a66dd..dbca08805e55 100644
--- a/posthog/settings/data_warehouse.py
+++ b/posthog/settings/data_warehouse.py
@@ -165,6 +165,14 @@ DATA_WAREHOUSE_FULL_VACUUM_INTERVAL_HOURS = get_from_env(
# `optimize.compact`. deltalite streams each bin under the load slot's memory budget. A deltalite build
# without `compact`, a table that deltalite refuses, or a deltalite failure falls back to delta-rs.
DATA_WAREHOUSE_DELTALITE_COMPACTION = get_from_env("DATA_WAREHOUSE_DELTALITE_COMPACTION", False, type_cast=str_to_bool)
+# Row-group cap for the files deltalite writes. A merge upsert closes a row group at this many encoded
+# bytes, and a deltalite compaction closes one at this many decoded bytes, so compaction does not join
+# the row groups again. A reader fetches one whole row group before it decodes it, so smaller row groups
+# lower upsert and query memory on tables with wide rows. 0 turns the cap off and keeps deltalite's
+# defaults.
+DATA_WAREHOUSE_DELTALITE_MAX_ROW_GROUP_BYTES = get_from_env(
+ "DATA_WAREHOUSE_DELTALITE_MAX_ROW_GROUP_BYTES", 16 * 1024 * 1024, type_cast=int
+)
# A rebuild run (first sync, reset, full refresh) compacts its own files once a compaction would remove
# this many, so a long rebuild does not leave one post-load pass that outlasts the stuck-batch timeout.
DATA_WAREHOUSE_REBUILD_COMPACT_MIN_REMOVABLE_FILES = get_from_env(
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/maintenance.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/maintenance.py
index cca8c1c6c0dd..be182b8d6390 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/maintenance.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/maintenance.py
@@ -587,6 +587,11 @@ class DeltaMaintenance:
"max_bins_per_commit": COMPACT_MAX_BINS_PER_COMMIT,
"partitions": partitions,
}
+ # Without this, compaction writes row groups up to deltalite's 128 MiB default and so joins again
+ # the row groups that the upsert cap split.
+ max_row_group_bytes = settings.DATA_WAREHOUSE_DELTALITE_MAX_ROW_GROUP_BYTES
+ if max_row_group_bytes > 0:
+ compact_kwargs["max_row_group_decoded_bytes"] = max_row_group_bytes
def _run() -> dict[str, Any]:
if identity is None:
@@ -605,6 +610,7 @@ class DeltaMaintenance:
compact_target_size=target_size,
compact_max_parallel_bins=compact_kwargs["max_parallel_bins"],
compact_slot_budget_mb=round(slot_budget_mb, 1) if slot_budget_mb is not None else None,
+ compact_max_row_group_decoded_bytes=compact_kwargs.get("max_row_group_decoded_bytes"),
pod_memory_mb=pod_mb_before,
peak_rss_mb=_peak_rss_mb(),
)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_maintenance.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_maintenance.py
index 8e47bb00f072..9131bfebf3f3 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_maintenance.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_maintenance.py
@@ -629,10 +629,47 @@ class TestDeltaliteCompaction:
@parameterized.expand(
[
- ("committed", 1, 152, [152], {}, DEFAULT_COMPACT_TARGET_SIZE_BYTES),
- ("nothing_to_rewrite", 0, 150, [], {}, DEFAULT_COMPACT_TARGET_SIZE_BYTES),
- ("table_target", 1, 152, [152], {"delta.targetFileSize": str(32 * _MB)}, 32 * _MB),
- ("invalid_table_target", 1, 152, [152], {"delta.targetFileSize": "-1"}, DEFAULT_COMPACT_TARGET_SIZE_BYTES),
+ (
+ "committed",
+ 1,
+ 152,
+ [152],
+ {},
+ DEFAULT_COMPACT_TARGET_SIZE_BYTES,
+ 16 * _MB,
+ {"max_row_group_decoded_bytes": 16 * _MB},
+ ),
+ (
+ "nothing_to_rewrite",
+ 0,
+ 150,
+ [],
+ {},
+ DEFAULT_COMPACT_TARGET_SIZE_BYTES,
+ 16 * _MB,
+ {"max_row_group_decoded_bytes": 16 * _MB},
+ ),
+ (
+ "table_target",
+ 1,
+ 152,
+ [152],
+ {"delta.targetFileSize": str(32 * _MB)},
+ 32 * _MB,
+ 16 * _MB,
+ {"max_row_group_decoded_bytes": 16 * _MB},
+ ),
+ (
+ "invalid_table_target",
+ 1,
+ 152,
+ [152],
+ {"delta.targetFileSize": "-1"},
+ DEFAULT_COMPACT_TARGET_SIZE_BYTES,
+ 16 * _MB,
+ {"max_row_group_decoded_bytes": 16 * _MB},
+ ),
+ ("row_group_cap_off", 1, 152, [152], {}, DEFAULT_COMPACT_TARGET_SIZE_BYTES, 0, {}),
]
)
@pytest.mark.asyncio
@@ -644,13 +681,18 @@ class TestDeltaliteCompaction:
expected_noted: list[int],
configuration: dict[str, str],
expected_target: int,
+ row_group_cap: int,
+ expected_row_group_kwargs: dict[str, int],
) -> None:
compact = MagicMock(
return_value={"commits": commits, "version": version, "numFilesAdded": 3, "numFilesRemoved": 300}
)
maintenance, table_ref, fake_class = self._setup(compact, configuration=configuration)
- with patch(f"{_MAINTENANCE_MODULE}._sample_compression_ratio", side_effect=AssertionError("sampled")):
+ with (
+ patch(f"{_MAINTENANCE_MODULE}._sample_compression_ratio", side_effect=AssertionError("sampled")),
+ override_settings(DATA_WAREHOUSE_DELTALITE_MAX_ROW_GROUP_BYTES=row_group_cap),
+ ):
ran, delta_rs_compact, plan, capture = await self._run(maintenance, fake_class)
assert ran is True
@@ -666,6 +708,7 @@ class TestDeltaliteCompaction:
min_partition_removable_files=1,
max_bins_per_commit=COMPACT_MAX_BINS_PER_COMMIT,
partitions=None,
+ **expected_row_group_kwargs,
)
assert [call.args[0] for call in table_ref.note_deltalite_commit.call_args_list] == expected_noted
[done] = self._info_calls(table_ref, "compact: done")
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_writer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_writer.py
index 2ca0d9467acd..9429f7a6bd9b 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_writer.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_writer.py
@@ -1,5 +1,6 @@
import os
import json
+import random
from decimal import Decimal
from pathlib import Path
from types import SimpleNamespace
@@ -8,9 +9,12 @@ from typing import Any, Literal, cast
import pytest
from unittest.mock import AsyncMock, MagicMock, patch
+from django.test import override_settings
+
import pyarrow as pa
import deltalake
import pyarrow.compute as pc
+import pyarrow.parquet as pq
from parameterized import parameterized
from pytest_mock import MockerFixture
@@ -1287,8 +1291,17 @@ class TestDeltaliteWritePath:
assert _committed_version(stats) is None
assert _committed_file_count_change(stats) is None
+ @parameterized.expand(
+ [
+ ("row_group_cap_on", 16 * 1024 * 1024, 16 * 1024 * 1024),
+ ("row_group_cap_off", 0, None),
+ ("negative_cap_is_off", -1, None),
+ ]
+ )
@pytest.mark.asyncio
- async def test_writes_via_deltalite_when_enabled(self):
+ async def test_writes_via_deltalite_when_enabled(
+ self, _name: str, row_group_cap: int, expected_max_row_group_bytes: int | None
+ ):
logger = make_logger() # captured so we can inspect the structured log without hitting the typed attr
helper = DeltaTableRef(resource_name="t", job=MagicMock(team_id=2, schema_id="sch-1"), logger=logger)
existing = MagicMock()
@@ -1305,6 +1318,7 @@ class TestDeltaliteWritePath:
patch.dict("sys.modules", {"deltalite": fake_deltalite}),
patch.object(helper, "_get_delta_table_uri", AsyncMock(return_value="s3://b/t")),
patch.object(helper, "_get_credentials", return_value={"AWS_REGION": "us-east-1"}),
+ override_settings(DATA_WAREHOUSE_DELTALITE_MAX_ROW_GROUP_BYTES=row_group_cap),
):
wrote = await DeltaWriter(helper)._write_via_deltalite(
existing_delta_table=existing,
@@ -1316,6 +1330,8 @@ class TestDeltaliteWritePath:
assert wrote is True
fake_deltalite.DeltaLiteTable.open.assert_called_once_with("s3://b/t", {"AWS_REGION": "us-east-1"})
fake_table.upsert.assert_called_once()
+ assert fake_table.upsert.call_args.kwargs["max_row_group_bytes"] == expected_max_row_group_bytes
+ assert "isolate_source_rows" not in fake_table.upsert.call_args.kwargs
# PARTITION_KEY is passed as the partition arg when the table is partitioned.
assert fake_table.upsert.call_args.args[2] == PARTITION_KEY
# The delta-rs handle is not refreshed on the write path (a log listing per batch); the ref
@@ -1330,6 +1346,49 @@ class TestDeltaliteWritePath:
assert log_kwargs["partitions_touched"] == 1
assert "duration_ms" in log_kwargs
+ @pytest.mark.parametrize("row_group_cap", [256 * 1024, 0], ids=["row_group_cap_on", "row_group_cap_off"])
+ @pytest.mark.asyncio
+ async def test_row_group_cap_reaches_the_real_upsert_and_compaction(
+ self, row_group_cap: int, tmp_path: Path
+ ) -> None:
+ payload = random.Random(0).randbytes
+ delta_path = str(tmp_path / "table")
+ deltalake.write_deltalake(delta_path, pa.table({"id": pa.array([0], pa.int64()), "v": [payload(32 * 1024)]}))
+ helper = make_local_table_ref(delta_path)
+ batch = pa.table({"id": pa.array(range(1, 49), pa.int64()), "v": [payload(32 * 1024) for _ in range(48)]})
+
+ def row_group_sizes() -> list[int]:
+ sizes = []
+ for uri in deltalake.DeltaTable(delta_path).file_uris():
+ metadata = pq.ParquetFile(uri).metadata
+ for index in range(metadata.num_row_groups):
+ row_group = metadata.row_group(index)
+ sizes.append(sum(row_group.column(c).total_compressed_size for c in range(row_group.num_columns)))
+ return sizes
+
+ with override_settings(DATA_WAREHOUSE_DELTALITE_MAX_ROW_GROUP_BYTES=row_group_cap):
+ wrote = await DeltaWriter(helper)._write_via_deltalite(
+ existing_delta_table=deltalake.DeltaTable(delta_path),
+ data=batch,
+ normalized_primary_keys=["id"],
+ use_partitioning=False,
+ commit_metadata=None,
+ )
+ assert wrote is True
+ after_upsert = row_group_sizes()
+ compacted = await DeltaMaintenance(helper)._compact_with_deltalite(deltalake.DeltaTable(delta_path))
+ assert compacted is True
+ after_compaction = row_group_sizes()
+
+ batch_bytes = 48 * 32 * 1024
+ if row_group_cap:
+ # The cap is checked between written batches, so a row group passes it by at most one batch.
+ for sizes in (after_upsert, after_compaction):
+ assert len(sizes) > 2 and max(sizes) <= 2 * row_group_cap
+ else:
+ assert max(after_upsert) >= batch_bytes and len(after_compaction) == 1
+ assert deltalake.DeltaTable(delta_path).to_pyarrow_table().num_rows == 49
+
@pytest.mark.asyncio
async def test_sizes_the_upsert_by_the_files_it_rewrites(self, tmp_path: Path):
existing_data = pa.table(
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/writer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/writer.py
index 2f3e42b4d94b..38fa23eba368 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/writer.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/writer.py
@@ -5,6 +5,8 @@ import contextlib
from collections.abc import Callable, Sequence
from typing import TYPE_CHECKING, Any, Literal
+from django.conf import settings
+
import pyarrow as pa
import deltalake
import pyarrow.compute as pc
@@ -333,6 +335,7 @@ class DeltaWriter:
if governor.config.mode != "off"
else None
)
+ max_row_group_bytes = settings.DATA_WAREHOUSE_DELTALITE_MAX_ROW_GROUP_BYTES
async with governor.admit(source_bytes=data.nbytes, n_partitions=n_partitions, rewrite=rewrite) as adm:
def _run_upsert(table: Any, upsert_kwargs: dict[str, int]) -> Any:
@@ -341,6 +344,7 @@ class DeltaWriter:
list(normalized_primary_keys),
partition_key,
commit_metadata=commit_metadata,
+ max_row_group_bytes=max_row_group_bytes if max_row_group_bytes > 0 else None,
**upsert_kwargs,
)
@@ -424,6 +428,7 @@ class DeltaWriter:
governor_rss_samples=adm.rss.samples if adm.rss else None,
governor_concurrent_upserts=adm.concurrent_upserts,
governor_max_concurrent_upserts=adm.rss.max_concurrent if adm.rss else None,
+ row_group_cap_bytes=max_row_group_bytes,
**_deltalite_write_stats(stats),
)
DELTALITE_WRITE_TOTAL.labels(outcome="written").inc()
diff --git a/pyproject.toml b/pyproject.toml
index 65749b80b297..2c461053b109 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -214,7 +214,7 @@ dependencies = [
"braintrust-core>=0.0.59,<1",
"wrapt==1.17.3",
"limits==5.8.0",
- "deltalite==0.1.11",
+ "deltalite==0.1.12",
"posthog-hogland==0.4.1",
"requests-oauthlib~=2.0.0",
]
diff --git a/uv.lock b/uv.lock
index 390c4454d298..a78bf098ac85 100644
--- a/uv.lock
+++ b/uv.lock
@@ -25,19 +25,19 @@ conflicts = [[
]]
[options]
-exclude-newer = "2026-10-09T14:35:12.109977Z"
+exclude-newer = "0001-01-01T00:00:00Z" # This has no effect and is included for backwards compatibility when using relative exclude-newer values.
exclude-newer-span = "P7D"
[options.exclude-newer-package]
-pixelhog = false
-modal = false
-posthoganalytics = false
+deltalite = false
django = false
google-ads = false
-deltalite = false
hogql-parser = false
-posthog-hogland = false
hogql-parser-rs = false
+modal = false
+pixelhog = false
+posthog-hogland = false
+posthoganalytics = false
[manifest]
members = [
@@ -1829,16 +1829,16 @@ pyarrow = [
[[package]]
name = "deltalite"
-version = "0.1.11"
+version = "0.1.12"
source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/f9/c1/85d795c164d60ed1a3395db326fd43e50d3506bbfc685372a1d46a84ca67/deltalite-0.1.11.tar.gz", hash = "sha256:69a6720829a94d8ba078b8ae08657172bf85ad6b84e044c5d56e98efcb45d090", size = 295605, upload-time = "2026-10-07T10:25:29.142Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/21/ad/bcf1451d9513de77d187e1a6d11c9528cf539675bb421bc68061da57c112/deltalite-0.1.12.tar.gz", hash = "sha256:ac62d8186e4db265d5ffca20df66dc2fc1d624e05492148599e868499cc6a3cc", size = 300564, upload-time = "2026-10-20T13:57:57.531Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/b0/4a/3f6b4d8e10c4024652322f824847728d6720b5bfa562d6d477af87379238/deltalite-0.1.11-cp312-abi3-macosx_10_12_x86_64.whl", hash = "sha256:41cf4ca5af5adfca9ab27e57faa977fff98a42547bb42b757acad7f37f2fd35c", size = 18463623, upload-time = "2026-10-07T10:25:10.442Z" },
- { url = "https://files.pythonhosted.org/packages/38/78/03a4faf78885264882a9f0ec09ce6b465adb7020e887e736e2de3a58b80f/deltalite-0.1.11-cp312-abi3-macosx_11_0_arm64.whl", hash = "sha256:788d7e4b2004a286df42be17788c06e920dfb890f3d007050ce44688adf4aa84", size = 17938315, upload-time = "2026-10-07T10:25:13.266Z" },
- { url = "https://files.pythonhosted.org/packages/79/7c/abab0f6814046ba998dd44ad19a7bdb3f8fca06087f28582b857f4b63bcd/deltalite-0.1.11-cp312-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:27c7603529dfd8220fc23940f42acdec424b810a1f161cb1a97859cb839bc0de", size = 64511410, upload-time = "2026-10-07T10:25:16.391Z" },
- { url = "https://files.pythonhosted.org/packages/c1/87/a290b5b16967fe6609fa83e37a3854b6a7b0133aeda25d40c07762f7483a/deltalite-0.1.11-cp312-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:81f3d808eb8d628c211696a55b9b6ebf8b51f91df1d13112e0d9a0ea708308f3", size = 67403553, upload-time = "2026-10-07T10:25:19.82Z" },
- { url = "https://files.pythonhosted.org/packages/75/fb/25d649290699b9398130a3f4fa3a22adf6850874e34fd6a10cb36c0ee444/deltalite-0.1.11-cp312-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:8bc04caa0b3b494180b7fdff8c3082641c9cee6b974613be9e1502873d4723f7", size = 64684529, upload-time = "2026-10-07T10:25:23.122Z" },
- { url = "https://files.pythonhosted.org/packages/5f/19/dbe6170af468c81a3b4d65ec268a7983c15a29a1c2cae2b1e9cc8da72153/deltalite-0.1.11-cp312-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:b92a2b957c489336526901e5d2a63b98c620d5a82738851271c6452d72e4fcf1", size = 65706707, upload-time = "2026-10-07T10:25:26.804Z" },
+ { url = "https://files.pythonhosted.org/packages/38/d6/21308bf1d8c8c775373eaa94ac481df68e8a66a0985d232b522070988d0a/deltalite-0.1.12-cp312-abi3-macosx_10_12_x86_64.whl", hash = "sha256:9c74b350479b83239351d837023c4e81bc6b31d4e59e5da02b7d2dd7d25a5210", size = 18459027, upload-time = "2026-10-20T13:57:36.784Z" },
+ { url = "https://files.pythonhosted.org/packages/59/a9/e049492f0d63c22b0ef402a162fed670674f4d343af4f4c0992b8d036132/deltalite-0.1.12-cp312-abi3-macosx_11_0_arm64.whl", hash = "sha256:27b704274b8cac4975320cabba25667e16cd562faf45d683051eafeaa16a5c52", size = 17924216, upload-time = "2026-10-20T13:57:39.66Z" },
+ { url = "https://files.pythonhosted.org/packages/79/35/6b3fc36a43d08da33a9a9f906fbbca87ef03ce3a271d3e5914746a9a1eef/deltalite-0.1.12-cp312-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:143f15bb98641b8fc174f569e9bbd929f517dc02c4b0c6f3191d30fdb7767ae8", size = 64522984, upload-time = "2026-10-20T13:57:42.734Z" },
+ { url = "https://files.pythonhosted.org/packages/a5/7b/3209dfec1c9b7d8dc9eac26e84c4e65fa7b259c98e3b4bb12734d753dfff/deltalite-0.1.12-cp312-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:5e37ae14d860256645ef6048e4684a1baf2c8605a267414a9514efa8c1f22b9e", size = 67427768, upload-time = "2026-10-20T13:57:46.922Z" },
+ { url = "https://files.pythonhosted.org/packages/2d/80/f0e19f43ad1f0a0ac44deee50b77c6bfad7e5b9d32be485c398f5f0462e6/deltalite-0.1.12-cp312-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:d5edebf257ba63329cc19b098f9943f1c7c1b22188d5088ac020be26b255215b", size = 64668300, upload-time = "2026-10-20T13:57:50.886Z" },
+ { url = "https://files.pythonhosted.org/packages/9d/31/afc82195f4c41e7f30218e8d591fbcdd70d95a2ab93e02c7534b88c5889b/deltalite-0.1.12-cp312-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:5e9ca5091aa30b822b06aca13e75b0c21ba495fca2060ed09337c13065c10416", size = 65700242, upload-time = "2026-10-20T13:57:54.816Z" },
]
[[package]]
@@ -6154,7 +6154,7 @@ requires-dist = [
{ name = "databricks-sql-connector", specifier = "~=4.3.0" },
{ name = "defusedxml", specifier = "~=0.7.1" },
{ name = "deltalake", extras = ["pyarrow"], specifier = "==1.6.1" },
- { name = "deltalite", specifier = "==0.1.11" },
+ { name = "deltalite", specifier = "==0.1.12" },
{ name = "disposable-email-domains", specifier = "~=0.0.140" },
{ name = "dj-database-url", specifier = "~=3.0.0" },
{ name = "django", specifier = "~=5.2.0" },