Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-imports): retry a delta merge that races a concurrent vacuum (#121301)

Tom Owers0547f19107f5
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/errors.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/errors.py
index 298eaaaa1260..12a5a1d059fd 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/errors.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/errors.py
@@ -243,6 +243,29 @@ def is_invalid_version_race(error: BaseException) -> bool:
return type(error) is deltalake.exceptions.DeltaError and DELTA_INVALID_VERSION_RACE_NEEDLE in str(error)
+# `merge()` plans its scan against the target's file list from the `DeltaTable` handle it was given,
+# then fetches each planned file's Parquet footer when the scan actually runs. A concurrent vacuum
+# (most commonly a heartbeat-timed-out zombie attempt running the post-load maintenance pass — see
+# this package's README on the equivalent race for repartition) can commit a REMOVE for one of those
+# files and delete it from the object store in between: the handle's in-memory plan still names it,
+# so the footer fetch 404s. delta-rs has no typed error for this — it surfaces as the same generic
+# DeltaError every Parquet-level failure does, with the object store's 404 body folded into the
+# message. Refreshing the handle (exactly what `execute_with_conflict_retry` already does for
+# `CommitFailedError`) drops the vacuumed file from the next plan, so retrying succeeds once the
+# handle reflects the version vacuum committed against. A file this names that is genuinely missing
+# from a fresh snapshot is real corruption, not this race: the refreshed plan still names it, the
+# retry fails the same way, and it still reaches error tracking once the retry budget is spent.
+DELTA_MISSING_SCAN_FILE_NEEDLE = "Failed to fetch metadata for file"
+
+
+def is_missing_scan_file_race(error: BaseException) -> bool:
+ return (
+ type(error) is deltalake.exceptions.DeltaError
+ and DELTA_MISSING_SCAN_FILE_NEEDLE in str(error)
+ and "not found" in str(error)
+ )
+
+
def _is_too_many_open_files_error(error: BaseException) -> bool:
"""True if opening the app-DB connection failed because this worker is out of file descriptors.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/ops.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/ops.py
index ef48093ef05d..be157a57ec90 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/ops.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/ops.py
@@ -16,6 +16,7 @@ from posthog.exceptions_capture import capture_exception
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.errors import (
TransientObjectStoreError,
is_invalid_version_race,
+ is_missing_scan_file_race,
is_transient_maintenance_error,
is_transient_object_store_error,
)
@@ -140,7 +141,9 @@ async def execute_with_conflict_retry(
See DELTA_MERGE_CONFLICT_RETRIES for why this can't rely on delta-rs's own retry budget. Also
retries `is_invalid_version_race` — the same race surfacing as a plain `DeltaError` rather than
- `CommitFailedError` because delta-rs's Python binding doesn't give that variant its own class.
+ `CommitFailedError` because delta-rs's Python binding doesn't give that variant its own class —
+ and `is_missing_scan_file_race`, where a concurrent vacuum deletes a file the operation's stale
+ handle still plans to scan.
An object-store refusal (see is_object_store_permission_denied) is raised as
`ObjectStorePermissionDeniedError` on the first attempt, so a refused vacuum or compaction can't
@@ -168,7 +171,11 @@ async def execute_with_conflict_retry(
raise TransientObjectStoreError(OBJECT_STORE_TRANSIENT_MESSAGE) from e
if not isinstance(e, deltalake.exceptions.DeltaError):
raise
- if not isinstance(e, deltalake.exceptions.CommitFailedError) and not is_invalid_version_race(e):
+ if (
+ not isinstance(e, deltalake.exceptions.CommitFailedError)
+ and not is_invalid_version_race(e)
+ and not is_missing_scan_file_race(e)
+ ):
raise
if attempt >= conflict_retries:
raise
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_delta_errors.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_delta_errors.py
index 9dc379e06bc8..e3297f26b28a 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_delta_errors.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_delta_errors.py
@@ -10,6 +10,7 @@ from parameterized import parameterized
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.errors import (
TransientObjectStoreError,
is_invalid_version_race,
+ is_missing_scan_file_race,
is_offset_overflow_compaction_error,
is_transient_delta_maintenance_error,
is_transient_maintenance_error,
@@ -363,6 +364,51 @@ class TestIsInvalidVersionRace:
assert is_invalid_version_race(error) is expected
+class TestIsMissingScanFileRace:
+ @parameterized.expand(
+ [
+ (
+ "data_file_404_during_scan",
+ deltalake.exceptions.DeltaError(
+ "Generic DeltaTable error: Parquet error: Parquet error: Failed to fetch metadata "
+ "for file dlt/team_1_source_1/orders/part-00000-abc.snappy.parquet: Object Store "
+ "error: Object at location dlt/team_1_source_1/orders/part-00000-abc.snappy.parquet "
+ "not found: Error performing GET in 20ms - Server returned non-2xx status code: "
+ "404 Not Found"
+ ),
+ True,
+ ),
+ # Same needle, but the object store refused the read rather than reporting it missing —
+ # a real, non-racy refusal that must stay out of this classifier.
+ (
+ "metadata_fetch_permission_denied_not_matched",
+ deltalake.exceptions.DeltaError(
+ "Generic DeltaTable error: Parquet error: Failed to fetch metadata for file "
+ "dlt/team_1_source_1/orders/part-00000-abc.snappy.parquet: Object Store error: "
+ "The operation lacked the necessary privileges to complete"
+ ),
+ False,
+ ),
+ ("unrelated_delta_error", deltalake.exceptions.DeltaError("no protocol found in delta log"), False),
+ # `CommitFailedError` is a `DeltaError` subclass, but this classifier is deliberately exact-type
+ # only, mirroring `is_invalid_version_race` — a missing-file 404 during a commit itself isn't
+ # this scan-planning race.
+ (
+ "commit_failed_error_not_matched",
+ deltalake.exceptions.CommitFailedError("Failed to fetch metadata for file ... not found"),
+ False,
+ ),
+ (
+ "wrong_exception_type",
+ RuntimeError("Failed to fetch metadata for file ... not found"),
+ False,
+ ),
+ ]
+ )
+ def test_classifies_missing_scan_file_race(self, _name: str, error: Exception, expected: bool):
+ assert is_missing_scan_file_race(error) is expected
+
+
class TestIsOffsetOverflowCompactionError:
@parameterized.expand(
[
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_ops.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_ops.py
index c05ff2489c64..4ecc16dec2bc 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_ops.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_ops.py
@@ -172,6 +172,33 @@ class TestExecuteWithConflictRetry:
assert operation_fn.call_count == 2
table.update_incremental.assert_called_once()
+ @pytest.mark.asyncio
+ async def test_retries_on_missing_scan_file_race_then_succeeds(self):
+ # A merge plans its scan against the stale file list on the handle it was given; if a
+ # concurrent vacuum (e.g. a zombie attempt's post-load pass) deletes one of those files
+ # before the scan reads it, delta-rs surfaces a 404 as a generic DeltaError with no typed
+ # class to catch (see errors.py's is_missing_scan_file_race). Regression coverage for this
+ # race killing the sync instead of refreshing the table and retrying like a commit conflict.
+ table = MagicMock()
+ operation_fn = MagicMock(
+ side_effect=[
+ deltalake.exceptions.DeltaError(
+ "Generic DeltaTable error: Parquet error: Parquet error: Failed to fetch metadata "
+ "for file dlt/team_1_source_1/orders/part-00000-abc.snappy.parquet: Object Store "
+ "error: Object at location dlt/team_1_source_1/orders/part-00000-abc.snappy.parquet "
+ "not found: Error performing GET in 20ms - Server returned non-2xx status code: "
+ "404 Not Found"
+ ),
+ {"num_output_rows": 1},
+ ]
+ )
+
+ result = await execute_with_conflict_retry(table, operation_fn, "op", make_logger())
+
+ assert result == {"num_output_rows": 1}
+ assert operation_fn.call_count == 2
+ table.update_incremental.assert_called_once()
+
@pytest.mark.asyncio
async def test_unrelated_delta_error_propagates_without_retry(self):
table = MagicMock()