:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): use deltalite 0.1.14 probe key cache (#121246)
Tom Owersb416ed0883d9
diff --git a/posthog/settings/data_warehouse.py b/posthog/settings/data_warehouse.pyindex 967f1b05b025..a2e823b6724f 100644--- a/posthog/settings/data_warehouse.py+++ b/posthog/settings/data_warehouse.py@@ -191,6 +191,11 @@ DATA_WAREHOUSE_DELTALITE_MAX_ROW_GROUP_BYTES = get_from_env(DATA_WAREHOUSE_DELTALITE_GOVERNOR_ROW_GROUP_SIZING = get_from_env("DATA_WAREHOUSE_DELTALITE_GOVERNOR_ROW_GROUP_SIZING", True, type_cast=str_to_bool)+# deltalite keeps the primary keys of the data files that a merge reads or writes in a per-process cache,+# so a later merge into the same table decides most files without a read from object storage. The cache+# size comes from the DELTALITE_PROBE_CACHE_BYTES environment variable (default 128 MiB per process). False makes every merge read each+# candidate file again, and needs a deltalite build that has the probe cache.+DATA_WAREHOUSE_DELTALITE_PROBE_CACHE = get_from_env("DATA_WAREHOUSE_DELTALITE_PROBE_CACHE", True, type_cast=str_to_bool)# A rebuild run (first sync, reset, full refresh) compacts its own files once a compaction would remove# this many, so a long rebuild does not leave one post-load pass that outlasts the stuck-batch timeout.DATA_WAREHOUSE_REBUILD_COMPACT_MIN_REMOVABLE_FILES = get_from_env(diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_writer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_writer.pyindex 9429f7a6bd9b..7bbe24d1278a 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_writer.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_writer.py@@ -1293,14 +1293,15 @@ class TestDeltaliteWritePath:@parameterized.expand([- ("row_group_cap_on", 16 * 1024 * 1024, 16 * 1024 * 1024),- ("row_group_cap_off", 0, None),- ("negative_cap_is_off", -1, None),+ ("row_group_cap_on", 16 * 1024 * 1024, 16 * 1024 * 1024, True),+ ("row_group_cap_off", 0, None, True),+ ("negative_cap_is_off", -1, None, True),+ ("probe_cache_off", 16 * 1024 * 1024, 16 * 1024 * 1024, False),])@pytest.mark.asyncioasync def test_writes_via_deltalite_when_enabled(- self, _name: str, row_group_cap: int, expected_max_row_group_bytes: int | None+ self, _name: str, row_group_cap: int, expected_max_row_group_bytes: int | None, probe_cache: bool):logger = make_logger() # captured so we can inspect the structured log without hitting the typed attrhelper = DeltaTableRef(resource_name="t", job=MagicMock(team_id=2, schema_id="sch-1"), logger=logger)@@ -1318,7 +1319,10 @@ class TestDeltaliteWritePath:patch.dict("sys.modules", {"deltalite": fake_deltalite}),patch.object(helper, "_get_delta_table_uri", AsyncMock(return_value="s3://b/t")),patch.object(helper, "_get_credentials", return_value={"AWS_REGION": "us-east-1"}),- override_settings(DATA_WAREHOUSE_DELTALITE_MAX_ROW_GROUP_BYTES=row_group_cap),+ override_settings(+ DATA_WAREHOUSE_DELTALITE_MAX_ROW_GROUP_BYTES=row_group_cap,+ DATA_WAREHOUSE_DELTALITE_PROBE_CACHE=probe_cache,+ ),):wrote = await DeltaWriter(helper)._write_via_deltalite(existing_delta_table=existing,@@ -1332,6 +1336,9 @@ class TestDeltaliteWritePath:fake_table.upsert.assert_called_once()assert fake_table.upsert.call_args.kwargs["max_row_group_bytes"] == expected_max_row_group_bytesassert "isolate_source_rows" not in fake_table.upsert.call_args.kwargs+ # The default leaves the argument out, so a deltalite build without it still works.+ assert fake_table.upsert.call_args.kwargs.get("use_probe_cache", True) is probe_cache+ assert ("use_probe_cache" in fake_table.upsert.call_args.kwargs) is not probe_cache# PARTITION_KEY is passed as the partition arg when the table is partitioned.assert fake_table.upsert.call_args.args[2] == PARTITION_KEY# The delta-rs handle is not refreshed on the write path (a log listing per batch); the refdiff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/writer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/writer.pyindex d484bd0092c3..845a87a325ad 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/writer.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/writer.py@@ -353,6 +353,8 @@ class DeltaWriter:else None)max_row_group_bytes = settings.DATA_WAREHOUSE_DELTALITE_MAX_ROW_GROUP_BYTES+ # Passed only to turn the cache off, so a deltalite build without the argument keeps working.+ probe_cache_kwargs = {} if settings.DATA_WAREHOUSE_DELTALITE_PROBE_CACHE else {"use_probe_cache": False}async with governor.admit(source_bytes=data.nbytes, n_partitions=n_partitions, rewrite=rewrite) as adm:def _run_upsert(table: Any, upsert_kwargs: dict[str, int]) -> Any:@@ -362,6 +364,7 @@ class DeltaWriter:partition_key,commit_metadata=commit_metadata,max_row_group_bytes=max_row_group_bytes if max_row_group_bytes > 0 else None,+ **probe_cache_kwargs,**upsert_kwargs,)diff --git a/pyproject.toml b/pyproject.tomlindex 600a244a4404..cc2a3ce4c229 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -214,7 +214,7 @@ dependencies = ["braintrust-core>=0.0.59,<1","wrapt==1.17.3","limits==5.8.0",- "deltalite==0.1.12",+ "deltalite==0.1.14","posthog-hogland==0.4.1","requests-oauthlib~=2.0.0","adlfs~=2026.8.0",diff --git a/uv.lock b/uv.lockindex fe0cb4da908d..14fa4fd2482b 100644--- a/uv.lock+++ b/uv.lock@@ -1871,16 +1871,16 @@ pyarrow = [[[package]]name = "deltalite"-version = "0.1.12"+version = "0.1.14"source = { registry = "https://pypi.org/simple" }-sdist = { url = "https://files.pythonhosted.org/packages/21/ad/bcf1451d9513de77d187e1a6d11c9528cf539675bb421bc68061da57c112/deltalite-0.1.12.tar.gz", hash = "sha256:ac62d8186e4db265d5ffca20df66dc2fc1d624e05492148599e868499cc6a3cc", size = 300564, upload-time = "2026-10-20T13:57:57.531Z" }+sdist = { url = "https://files.pythonhosted.org/packages/65/c7/c445c263ceb6be3ba4e3f2c0f4e57ddae5bb78c26cd95cb8cd75e76570c1/deltalite-0.1.14.tar.gz", hash = "sha256:0910042c7a82ef03a43ebd664ac9ff8bcbee05add605e18f95bf61194482bab0", size = 316998, upload-time = "2026-10-21T14:32:33.215Z" }wheels = [- { url = "https://files.pythonhosted.org/packages/38/d6/21308bf1d8c8c775373eaa94ac481df68e8a66a0985d232b522070988d0a/deltalite-0.1.12-cp312-abi3-macosx_10_12_x86_64.whl", hash = "sha256:9c74b350479b83239351d837023c4e81bc6b31d4e59e5da02b7d2dd7d25a5210", size = 18459027, upload-time = "2026-10-20T13:57:36.784Z" },- { url = "https://files.pythonhosted.org/packages/59/a9/e049492f0d63c22b0ef402a162fed670674f4d343af4f4c0992b8d036132/deltalite-0.1.12-cp312-abi3-macosx_11_0_arm64.whl", hash = "sha256:27b704274b8cac4975320cabba25667e16cd562faf45d683051eafeaa16a5c52", size = 17924216, upload-time = "2026-10-20T13:57:39.66Z" },- { url = "https://files.pythonhosted.org/packages/79/35/6b3fc36a43d08da33a9a9f906fbbca87ef03ce3a271d3e5914746a9a1eef/deltalite-0.1.12-cp312-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:143f15bb98641b8fc174f569e9bbd929f517dc02c4b0c6f3191d30fdb7767ae8", size = 64522984, upload-time = "2026-10-20T13:57:42.734Z" },- { url = "https://files.pythonhosted.org/packages/a5/7b/3209dfec1c9b7d8dc9eac26e84c4e65fa7b259c98e3b4bb12734d753dfff/deltalite-0.1.12-cp312-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:5e37ae14d860256645ef6048e4684a1baf2c8605a267414a9514efa8c1f22b9e", size = 67427768, upload-time = "2026-10-20T13:57:46.922Z" },- { url = "https://files.pythonhosted.org/packages/2d/80/f0e19f43ad1f0a0ac44deee50b77c6bfad7e5b9d32be485c398f5f0462e6/deltalite-0.1.12-cp312-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:d5edebf257ba63329cc19b098f9943f1c7c1b22188d5088ac020be26b255215b", size = 64668300, upload-time = "2026-10-20T13:57:50.886Z" },- { url = "https://files.pythonhosted.org/packages/9d/31/afc82195f4c41e7f30218e8d591fbcdd70d95a2ab93e02c7534b88c5889b/deltalite-0.1.12-cp312-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:5e9ca5091aa30b822b06aca13e75b0c21ba495fca2060ed09337c13065c10416", size = 65700242, upload-time = "2026-10-20T13:57:54.816Z" },+ { url = "https://files.pythonhosted.org/packages/dd/be/446ddb7fe4a4b4df434b6b6a881b175aed38a0f77d6724dda33903851459/deltalite-0.1.14-cp312-abi3-macosx_10_12_x86_64.whl", hash = "sha256:c00677dbffd65d329cfaa33614525f95322e341fc5e4309bdbf8e4683cb04c22", size = 18578473, upload-time = "2026-10-21T14:32:10.148Z" },+ { url = "https://files.pythonhosted.org/packages/8b/00/f21fcfd7a9344678aa005ea79d73a2390f92c73d0929334a849b6570849c/deltalite-0.1.14-cp312-abi3-macosx_11_0_arm64.whl", hash = "sha256:b64ad3caa08eae01b0b565ffb28d0491e9135774622bd7b56681d49b71b2d4ac", size = 18010303, upload-time = "2026-10-21T14:32:13.442Z" },+ { url = "https://files.pythonhosted.org/packages/09/17/7c70f6914aec4bdc56a5afbf7d793f3c4a6edff15607066a0e62de6d52cb/deltalite-0.1.14-cp312-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:7babd51fb980c26112809d508be6ad55a1a401969b03008d37d220f9caee41a6", size = 64852729, upload-time = "2026-10-21T14:32:17.198Z" },+ { url = "https://files.pythonhosted.org/packages/3c/62/b84fb7456fb4c0279527c86e0067d4464c086314a29369207d4d80b5a0c4/deltalite-0.1.14-cp312-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:f18cef257e7d3283061bf6de255a87ee39d125377a5c59149286ccc53d71664e", size = 67727166, upload-time = "2026-10-21T14:32:21.462Z" },+ { url = "https://files.pythonhosted.org/packages/85/b9/0d15b83487aa81a68d68302bc8be8584980b015bdf8b024706c5a0c79b30/deltalite-0.1.14-cp312-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:336d62143a0e54a2cc5ca866c0eff06c9fe60f555fa5e2dd0d74345829c3f2c2", size = 65013325, upload-time = "2026-10-21T14:32:25.769Z" },+ { url = "https://files.pythonhosted.org/packages/25/8d/81138d8aa256b1a75cbd4acff63acdbb94fb4f853d10c17a58210b35c3d3/deltalite-0.1.14-cp312-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:07daeeaf9f93a91f7f8fc2b20d2fa757fca1e332bd68c1232814e9ab53760153", size = 66017332, upload-time = "2026-10-21T14:32:30.002Z" },][[package]]@@ -6224,7 +6224,7 @@ requires-dist = [{ name = "databricks-sql-connector", specifier = "~=4.3.0" },{ name = "defusedxml", specifier = "~=0.7.1" },{ name = "deltalake", extras = ["pyarrow"], specifier = "==1.6.1" },- { name = "deltalite", specifier = "==0.1.12" },+ { name = "deltalite", specifier = "==0.1.14" },{ name = "disposable-email-domains", specifier = "~=0.0.140" },{ name = "dj-database-url", specifier = "~=3.0.0" },{ name = "django", specifier = "~=5.2.0" },