:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(data-imports): retry a delta merge that races a concurrent vacuum (#121301)
Tom Owers0547f19107f5
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/errors.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/errors.pyindex 298eaaaa1260..12a5a1d059fd 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/errors.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/errors.py@@ -243,6 +243,29 @@ def is_invalid_version_race(error: BaseException) -> bool:return type(error) is deltalake.exceptions.DeltaError and DELTA_INVALID_VERSION_RACE_NEEDLE in str(error)+# `merge()` plans its scan against the target's file list from the `DeltaTable` handle it was given,+# then fetches each planned file's Parquet footer when the scan actually runs. A concurrent vacuum+# (most commonly a heartbeat-timed-out zombie attempt running the post-load maintenance pass — see+# this package's README on the equivalent race for repartition) can commit a REMOVE for one of those+# files and delete it from the object store in between: the handle's in-memory plan still names it,+# so the footer fetch 404s. delta-rs has no typed error for this — it surfaces as the same generic+# DeltaError every Parquet-level failure does, with the object store's 404 body folded into the+# message. Refreshing the handle (exactly what `execute_with_conflict_retry` already does for+# `CommitFailedError`) drops the vacuumed file from the next plan, so retrying succeeds once the+# handle reflects the version vacuum committed against. A file this names that is genuinely missing+# from a fresh snapshot is real corruption, not this race: the refreshed plan still names it, the+# retry fails the same way, and it still reaches error tracking once the retry budget is spent.+DELTA_MISSING_SCAN_FILE_NEEDLE = "Failed to fetch metadata for file"+++def is_missing_scan_file_race(error: BaseException) -> bool:+ return (+ type(error) is deltalake.exceptions.DeltaError+ and DELTA_MISSING_SCAN_FILE_NEEDLE in str(error)+ and "not found" in str(error)+ )++def _is_too_many_open_files_error(error: BaseException) -> bool:"""True if opening the app-DB connection failed because this worker is out of file descriptors.diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/ops.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/ops.pyindex ef48093ef05d..be157a57ec90 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/ops.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/ops.py@@ -16,6 +16,7 @@ from posthog.exceptions_capture import capture_exceptionfrom products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.errors import (TransientObjectStoreError,is_invalid_version_race,+ is_missing_scan_file_race,is_transient_maintenance_error,is_transient_object_store_error,)@@ -140,7 +141,9 @@ async def execute_with_conflict_retry(See DELTA_MERGE_CONFLICT_RETRIES for why this can't rely on delta-rs's own retry budget. Alsoretries `is_invalid_version_race` — the same race surfacing as a plain `DeltaError` rather than- `CommitFailedError` because delta-rs's Python binding doesn't give that variant its own class.+ `CommitFailedError` because delta-rs's Python binding doesn't give that variant its own class —+ and `is_missing_scan_file_race`, where a concurrent vacuum deletes a file the operation's stale+ handle still plans to scan.An object-store refusal (see is_object_store_permission_denied) is raised as`ObjectStorePermissionDeniedError` on the first attempt, so a refused vacuum or compaction can't@@ -168,7 +171,11 @@ async def execute_with_conflict_retry(raise TransientObjectStoreError(OBJECT_STORE_TRANSIENT_MESSAGE) from eif not isinstance(e, deltalake.exceptions.DeltaError):raise- if not isinstance(e, deltalake.exceptions.CommitFailedError) and not is_invalid_version_race(e):+ if (+ not isinstance(e, deltalake.exceptions.CommitFailedError)+ and not is_invalid_version_race(e)+ and not is_missing_scan_file_race(e)+ ):raiseif attempt >= conflict_retries:raisediff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_delta_errors.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_delta_errors.pyindex 9dc379e06bc8..e3297f26b28a 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_delta_errors.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_delta_errors.py@@ -10,6 +10,7 @@ from parameterized import parameterizedfrom products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.errors import (TransientObjectStoreError,is_invalid_version_race,+ is_missing_scan_file_race,is_offset_overflow_compaction_error,is_transient_delta_maintenance_error,is_transient_maintenance_error,@@ -363,6 +364,51 @@ class TestIsInvalidVersionRace:assert is_invalid_version_race(error) is expected+class TestIsMissingScanFileRace:+ @parameterized.expand(+ [+ (+ "data_file_404_during_scan",+ deltalake.exceptions.DeltaError(+ "Generic DeltaTable error: Parquet error: Parquet error: Failed to fetch metadata "+ "for file dlt/team_1_source_1/orders/part-00000-abc.snappy.parquet: Object Store "+ "error: Object at location dlt/team_1_source_1/orders/part-00000-abc.snappy.parquet "+ "not found: Error performing GET in 20ms - Server returned non-2xx status code: "+ "404 Not Found"+ ),+ True,+ ),+ # Same needle, but the object store refused the read rather than reporting it missing —+ # a real, non-racy refusal that must stay out of this classifier.+ (+ "metadata_fetch_permission_denied_not_matched",+ deltalake.exceptions.DeltaError(+ "Generic DeltaTable error: Parquet error: Failed to fetch metadata for file "+ "dlt/team_1_source_1/orders/part-00000-abc.snappy.parquet: Object Store error: "+ "The operation lacked the necessary privileges to complete"+ ),+ False,+ ),+ ("unrelated_delta_error", deltalake.exceptions.DeltaError("no protocol found in delta log"), False),+ # `CommitFailedError` is a `DeltaError` subclass, but this classifier is deliberately exact-type+ # only, mirroring `is_invalid_version_race` — a missing-file 404 during a commit itself isn't+ # this scan-planning race.+ (+ "commit_failed_error_not_matched",+ deltalake.exceptions.CommitFailedError("Failed to fetch metadata for file ... not found"),+ False,+ ),+ (+ "wrong_exception_type",+ RuntimeError("Failed to fetch metadata for file ... not found"),+ False,+ ),+ ]+ )+ def test_classifies_missing_scan_file_race(self, _name: str, error: Exception, expected: bool):+ assert is_missing_scan_file_race(error) is expected++class TestIsOffsetOverflowCompactionError:@parameterized.expand([diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_ops.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_ops.pyindex c05ff2489c64..4ecc16dec2bc 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_ops.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_ops.py@@ -172,6 +172,33 @@ class TestExecuteWithConflictRetry:assert operation_fn.call_count == 2table.update_incremental.assert_called_once()+ @pytest.mark.asyncio+ async def test_retries_on_missing_scan_file_race_then_succeeds(self):+ # A merge plans its scan against the stale file list on the handle it was given; if a+ # concurrent vacuum (e.g. a zombie attempt's post-load pass) deletes one of those files+ # before the scan reads it, delta-rs surfaces a 404 as a generic DeltaError with no typed+ # class to catch (see errors.py's is_missing_scan_file_race). Regression coverage for this+ # race killing the sync instead of refreshing the table and retrying like a commit conflict.+ table = MagicMock()+ operation_fn = MagicMock(+ side_effect=[+ deltalake.exceptions.DeltaError(+ "Generic DeltaTable error: Parquet error: Parquet error: Failed to fetch metadata "+ "for file dlt/team_1_source_1/orders/part-00000-abc.snappy.parquet: Object Store "+ "error: Object at location dlt/team_1_source_1/orders/part-00000-abc.snappy.parquet "+ "not found: Error performing GET in 20ms - Server returned non-2xx status code: "+ "404 Not Found"+ ),+ {"num_output_rows": 1},+ ]+ )++ result = await execute_with_conflict_retry(table, operation_fn, "op", make_logger())++ assert result == {"num_output_rows": 1}+ assert operation_fn.call_count == 2+ table.update_incremental.assert_called_once()+@pytest.mark.asyncioasync def test_unrelated_delta_error_propagates_without_retry(self):table = MagicMock()