Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): use deltalite 0.1.14 probe key cache (#121246)

Tom Owersb416ed0883d9
diff --git a/posthog/settings/data_warehouse.py b/posthog/settings/data_warehouse.py
index 967f1b05b025..a2e823b6724f 100644
--- a/posthog/settings/data_warehouse.py
+++ b/posthog/settings/data_warehouse.py
@@ -191,6 +191,11 @@ DATA_WAREHOUSE_DELTALITE_MAX_ROW_GROUP_BYTES = get_from_env(
DATA_WAREHOUSE_DELTALITE_GOVERNOR_ROW_GROUP_SIZING = get_from_env(
"DATA_WAREHOUSE_DELTALITE_GOVERNOR_ROW_GROUP_SIZING", True, type_cast=str_to_bool
)
+# deltalite keeps the primary keys of the data files that a merge reads or writes in a per-process cache,
+# so a later merge into the same table decides most files without a read from object storage. The cache
+# size comes from the DELTALITE_PROBE_CACHE_BYTES environment variable (default 128 MiB per process). False makes every merge read each
+# candidate file again, and needs a deltalite build that has the probe cache.
+DATA_WAREHOUSE_DELTALITE_PROBE_CACHE = get_from_env("DATA_WAREHOUSE_DELTALITE_PROBE_CACHE", True, type_cast=str_to_bool)
# A rebuild run (first sync, reset, full refresh) compacts its own files once a compaction would remove
# this many, so a long rebuild does not leave one post-load pass that outlasts the stuck-batch timeout.
DATA_WAREHOUSE_REBUILD_COMPACT_MIN_REMOVABLE_FILES = get_from_env(
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_writer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_writer.py
index 9429f7a6bd9b..7bbe24d1278a 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_writer.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_writer.py
@@ -1293,14 +1293,15 @@ class TestDeltaliteWritePath:
@parameterized.expand(
[
- ("row_group_cap_on", 16 * 1024 * 1024, 16 * 1024 * 1024),
- ("row_group_cap_off", 0, None),
- ("negative_cap_is_off", -1, None),
+ ("row_group_cap_on", 16 * 1024 * 1024, 16 * 1024 * 1024, True),
+ ("row_group_cap_off", 0, None, True),
+ ("negative_cap_is_off", -1, None, True),
+ ("probe_cache_off", 16 * 1024 * 1024, 16 * 1024 * 1024, False),
]
)
@pytest.mark.asyncio
async def test_writes_via_deltalite_when_enabled(
- self, _name: str, row_group_cap: int, expected_max_row_group_bytes: int | None
+ self, _name: str, row_group_cap: int, expected_max_row_group_bytes: int | None, probe_cache: bool
):
logger = make_logger() # captured so we can inspect the structured log without hitting the typed attr
helper = DeltaTableRef(resource_name="t", job=MagicMock(team_id=2, schema_id="sch-1"), logger=logger)
@@ -1318,7 +1319,10 @@ class TestDeltaliteWritePath:
patch.dict("sys.modules", {"deltalite": fake_deltalite}),
patch.object(helper, "_get_delta_table_uri", AsyncMock(return_value="s3://b/t")),
patch.object(helper, "_get_credentials", return_value={"AWS_REGION": "us-east-1"}),
- override_settings(DATA_WAREHOUSE_DELTALITE_MAX_ROW_GROUP_BYTES=row_group_cap),
+ override_settings(
+ DATA_WAREHOUSE_DELTALITE_MAX_ROW_GROUP_BYTES=row_group_cap,
+ DATA_WAREHOUSE_DELTALITE_PROBE_CACHE=probe_cache,
+ ),
):
wrote = await DeltaWriter(helper)._write_via_deltalite(
existing_delta_table=existing,
@@ -1332,6 +1336,9 @@ class TestDeltaliteWritePath:
fake_table.upsert.assert_called_once()
assert fake_table.upsert.call_args.kwargs["max_row_group_bytes"] == expected_max_row_group_bytes
assert "isolate_source_rows" not in fake_table.upsert.call_args.kwargs
+ # The default leaves the argument out, so a deltalite build without it still works.
+ assert fake_table.upsert.call_args.kwargs.get("use_probe_cache", True) is probe_cache
+ assert ("use_probe_cache" in fake_table.upsert.call_args.kwargs) is not probe_cache
# PARTITION_KEY is passed as the partition arg when the table is partitioned.
assert fake_table.upsert.call_args.args[2] == PARTITION_KEY
# The delta-rs handle is not refreshed on the write path (a log listing per batch); the ref
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/writer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/writer.py
index d484bd0092c3..845a87a325ad 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/writer.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/writer.py
@@ -353,6 +353,8 @@ class DeltaWriter:
else None
)
max_row_group_bytes = settings.DATA_WAREHOUSE_DELTALITE_MAX_ROW_GROUP_BYTES
+ # Passed only to turn the cache off, so a deltalite build without the argument keeps working.
+ probe_cache_kwargs = {} if settings.DATA_WAREHOUSE_DELTALITE_PROBE_CACHE else {"use_probe_cache": False}
async with governor.admit(source_bytes=data.nbytes, n_partitions=n_partitions, rewrite=rewrite) as adm:
def _run_upsert(table: Any, upsert_kwargs: dict[str, int]) -> Any:
@@ -362,6 +364,7 @@ class DeltaWriter:
partition_key,
commit_metadata=commit_metadata,
max_row_group_bytes=max_row_group_bytes if max_row_group_bytes > 0 else None,
+ **probe_cache_kwargs,
**upsert_kwargs,
)
diff --git a/pyproject.toml b/pyproject.toml
index 600a244a4404..cc2a3ce4c229 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -214,7 +214,7 @@ dependencies = [
"braintrust-core>=0.0.59,<1",
"wrapt==1.17.3",
"limits==5.8.0",
- "deltalite==0.1.12",
+ "deltalite==0.1.14",
"posthog-hogland==0.4.1",
"requests-oauthlib~=2.0.0",
"adlfs~=2026.8.0",
diff --git a/uv.lock b/uv.lock
index fe0cb4da908d..14fa4fd2482b 100644
--- a/uv.lock
+++ b/uv.lock
@@ -1871,16 +1871,16 @@ pyarrow = [
[[package]]
name = "deltalite"
-version = "0.1.12"
+version = "0.1.14"
source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/21/ad/bcf1451d9513de77d187e1a6d11c9528cf539675bb421bc68061da57c112/deltalite-0.1.12.tar.gz", hash = "sha256:ac62d8186e4db265d5ffca20df66dc2fc1d624e05492148599e868499cc6a3cc", size = 300564, upload-time = "2026-10-20T13:57:57.531Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/65/c7/c445c263ceb6be3ba4e3f2c0f4e57ddae5bb78c26cd95cb8cd75e76570c1/deltalite-0.1.14.tar.gz", hash = "sha256:0910042c7a82ef03a43ebd664ac9ff8bcbee05add605e18f95bf61194482bab0", size = 316998, upload-time = "2026-10-21T14:32:33.215Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/38/d6/21308bf1d8c8c775373eaa94ac481df68e8a66a0985d232b522070988d0a/deltalite-0.1.12-cp312-abi3-macosx_10_12_x86_64.whl", hash = "sha256:9c74b350479b83239351d837023c4e81bc6b31d4e59e5da02b7d2dd7d25a5210", size = 18459027, upload-time = "2026-10-20T13:57:36.784Z" },
- { url = "https://files.pythonhosted.org/packages/59/a9/e049492f0d63c22b0ef402a162fed670674f4d343af4f4c0992b8d036132/deltalite-0.1.12-cp312-abi3-macosx_11_0_arm64.whl", hash = "sha256:27b704274b8cac4975320cabba25667e16cd562faf45d683051eafeaa16a5c52", size = 17924216, upload-time = "2026-10-20T13:57:39.66Z" },
- { url = "https://files.pythonhosted.org/packages/79/35/6b3fc36a43d08da33a9a9f906fbbca87ef03ce3a271d3e5914746a9a1eef/deltalite-0.1.12-cp312-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:143f15bb98641b8fc174f569e9bbd929f517dc02c4b0c6f3191d30fdb7767ae8", size = 64522984, upload-time = "2026-10-20T13:57:42.734Z" },
- { url = "https://files.pythonhosted.org/packages/a5/7b/3209dfec1c9b7d8dc9eac26e84c4e65fa7b259c98e3b4bb12734d753dfff/deltalite-0.1.12-cp312-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:5e37ae14d860256645ef6048e4684a1baf2c8605a267414a9514efa8c1f22b9e", size = 67427768, upload-time = "2026-10-20T13:57:46.922Z" },
- { url = "https://files.pythonhosted.org/packages/2d/80/f0e19f43ad1f0a0ac44deee50b77c6bfad7e5b9d32be485c398f5f0462e6/deltalite-0.1.12-cp312-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:d5edebf257ba63329cc19b098f9943f1c7c1b22188d5088ac020be26b255215b", size = 64668300, upload-time = "2026-10-20T13:57:50.886Z" },
- { url = "https://files.pythonhosted.org/packages/9d/31/afc82195f4c41e7f30218e8d591fbcdd70d95a2ab93e02c7534b88c5889b/deltalite-0.1.12-cp312-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:5e9ca5091aa30b822b06aca13e75b0c21ba495fca2060ed09337c13065c10416", size = 65700242, upload-time = "2026-10-20T13:57:54.816Z" },
+ { url = "https://files.pythonhosted.org/packages/dd/be/446ddb7fe4a4b4df434b6b6a881b175aed38a0f77d6724dda33903851459/deltalite-0.1.14-cp312-abi3-macosx_10_12_x86_64.whl", hash = "sha256:c00677dbffd65d329cfaa33614525f95322e341fc5e4309bdbf8e4683cb04c22", size = 18578473, upload-time = "2026-10-21T14:32:10.148Z" },
+ { url = "https://files.pythonhosted.org/packages/8b/00/f21fcfd7a9344678aa005ea79d73a2390f92c73d0929334a849b6570849c/deltalite-0.1.14-cp312-abi3-macosx_11_0_arm64.whl", hash = "sha256:b64ad3caa08eae01b0b565ffb28d0491e9135774622bd7b56681d49b71b2d4ac", size = 18010303, upload-time = "2026-10-21T14:32:13.442Z" },
+ { url = "https://files.pythonhosted.org/packages/09/17/7c70f6914aec4bdc56a5afbf7d793f3c4a6edff15607066a0e62de6d52cb/deltalite-0.1.14-cp312-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:7babd51fb980c26112809d508be6ad55a1a401969b03008d37d220f9caee41a6", size = 64852729, upload-time = "2026-10-21T14:32:17.198Z" },
+ { url = "https://files.pythonhosted.org/packages/3c/62/b84fb7456fb4c0279527c86e0067d4464c086314a29369207d4d80b5a0c4/deltalite-0.1.14-cp312-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:f18cef257e7d3283061bf6de255a87ee39d125377a5c59149286ccc53d71664e", size = 67727166, upload-time = "2026-10-21T14:32:21.462Z" },
+ { url = "https://files.pythonhosted.org/packages/85/b9/0d15b83487aa81a68d68302bc8be8584980b015bdf8b024706c5a0c79b30/deltalite-0.1.14-cp312-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:336d62143a0e54a2cc5ca866c0eff06c9fe60f555fa5e2dd0d74345829c3f2c2", size = 65013325, upload-time = "2026-10-21T14:32:25.769Z" },
+ { url = "https://files.pythonhosted.org/packages/25/8d/81138d8aa256b1a75cbd4acff63acdbb94fb4f853d10c17a58210b35c3d3/deltalite-0.1.14-cp312-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:07daeeaf9f93a91f7f8fc2b20d2fa757fca1e332bd68c1232814e9ab53760153", size = 66017332, upload-time = "2026-10-21T14:32:30.002Z" },
]
[[package]]
@@ -6224,7 +6224,7 @@ requires-dist = [
{ name = "databricks-sql-connector", specifier = "~=4.3.0" },
{ name = "defusedxml", specifier = "~=0.7.1" },
{ name = "deltalake", extras = ["pyarrow"], specifier = "==1.6.1" },
- { name = "deltalite", specifier = "==0.1.12" },
+ { name = "deltalite", specifier = "==0.1.14" },
{ name = "disposable-email-domains", specifier = "~=0.0.140" },
{ name = "dj-database-url", specifier = "~=3.0.0" },
{ name = "django", specifier = "~=5.2.0" },