Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-imports): treat delta log gap/checkpoint races as transient (#122154)

Tom Owers31663e40fa1e
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/errors.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/errors.py
index 12a5a1d059fd..8aa270c9afe2 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/errors.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/errors.py
@@ -212,6 +212,38 @@ def is_transient_delta_maintenance_error(error: BaseException) -> bool:
return "is older than the existing snapshot version" in text
+# A fourth shape of the same reset-vs-concurrent-read race, caught by the kernel's log-segment
+# listing instead of a single file lookup: `_purge_s3_prefix_once` lists the table's live
+# `_delta_log` files once and bulk-deletes them in one `_rm` call, which S3 does not guarantee to
+# apply in any particular order across keys. A concurrent open that lists the directory (or reads
+# `_last_checkpoint` and fetches the checkpoint it names) mid-delete can see an arbitrary subset of
+# that batch already gone — a run of consecutive commit files missing from the middle of the
+# sequence ("Expected contiguous commit files, but found gap"), or a `_last_checkpoint` hint whose
+# checkpoint file the same batch already removed ("Had a _last_checkpoint hint but didn't find any
+# checkpoints"). Moments later the purge finishes and the next reader opens a fresh, consistent log.
+#
+# Deliberately NOT folded into `is_transient_delta_maintenance_error` above: that function also backs
+# `DeltaTableRef.is_table_corrupted`, whose whole purpose is catching a log left genuinely broken by an
+# interrupted repartition swap or an OOM-crashed merge (see its docstring) so `handle_corrupted_delta_log`
+# can revive it — and unlike the single-file "not found" case, a missing *run* of commit files or a
+# dangling checkpoint hint is exactly the shape that kind of interrupted write leaves behind too. Folding
+# these in would make `is_table_corrupted` read a genuinely and permanently broken log as transient,
+# silently skip the revive, and loop the sync forever with nothing reported. Scoped instead to the one
+# call site (`DeltaTableRef._capture_unless_transient`) that only suppresses error-tracking noise on the
+# common race, while `is_table_corrupted`'s own independent open keeps classifying a log that is still in
+# this state on a later attempt as corrupt and reviving it.
+DELTA_LOG_LISTING_RACE_ERRORS = (
+ "Expected contiguous commit files",
+ "Had a _last_checkpoint hint but didn't find any checkpoints",
+)
+
+
+def is_transient_delta_log_listing_race(error: BaseException) -> bool:
+ return isinstance(error, deltalake.exceptions.DeltaError) and any(
+ needle in str(error) for needle in DELTA_LOG_LISTING_RACE_ERRORS
+ )
+
+
# `optimize.compact` bins files to rewrite by their on-disk (compressed) size, targeting
# `target_size` bytes per bin, then reads every file in a bin into memory as Arrow batches before
# writing them back out as one file. `Batcher` (pipelines/core/batcher.py) already keeps each
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/table.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/table.py
index 725e2d81fba5..ccbeaf0948b5 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/table.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/table.py
@@ -20,6 +20,7 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.del
)
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.errors import (
TransientObjectStoreError,
+ is_transient_delta_log_listing_race,
is_transient_delta_maintenance_error,
is_transient_object_store_error,
)
@@ -275,13 +276,22 @@ class DeltaTableRef:
async def _capture_unless_transient(self, e: Exception) -> None:
"""capture_exception unless `e` is already classified as something other than a defect.
- A known-transient object-store blip (see is_transient_object_store_error) or a
- concurrent-purge race on `_delta_log` (see is_transient_delta_maintenance_error, because the
- open below can lose the same race a maintenance pass can) recovers on retry, so reporting it
- to error tracking is just noise. Both are re-raised as TransientObjectStoreError instead of
- letting the original propagate, because the activity interceptor reports any uncaught
- activity exception unless it is a NonReportableError, which TransientObjectStoreError is, so
- a bare re-raise here would still mint a fresh issue at that boundary.
+ A known-transient object-store blip (see is_transient_object_store_error), a concurrent-purge
+ race on `_delta_log` (see is_transient_delta_maintenance_error, because the open below can
+ lose the same race a maintenance pass can), or the same race caught by the kernel's log-segment
+ listing instead of a single file lookup (see is_transient_delta_log_listing_race) recovers on
+ retry, so reporting it to error tracking is just noise. All three are re-raised as
+ TransientObjectStoreError instead of letting the original propagate, because the activity
+ interceptor reports any uncaught activity exception unless it is a NonReportableError, which
+ TransientObjectStoreError is, so a bare re-raise here would still mint a fresh issue at that
+ boundary.
+
+ is_transient_delta_log_listing_race is deliberately not folded into
+ is_transient_delta_maintenance_error, and so does not affect `is_table_corrupted`'s own
+ classification of the same message shapes: unlike the other races here, a missing run of commit
+ files or a dangling checkpoint hint is also the signature an interrupted repartition swap leaves
+ behind, and `is_table_corrupted` must keep reading a log still in that state on a later attempt
+ as corrupt so it gets revived instead of retried forever.
An object-store refusal (see is_object_store_permission_denied) is re-raised as
ObjectStorePermissionDeniedError, which is not a NonReportableError: a refusal on our own
@@ -303,7 +313,11 @@ class DeltaTableRef:
# hide it behind Temporal's retry policy until the activity gave up.
await self._logger.aerror(f"get_delta_table: the object store denied the read ({type(e).__name__})")
raise ObjectStorePermissionDeniedError(OBJECT_STORE_PERMISSION_DENIED_MESSAGE) from e
- if is_transient_object_store_error(e) or is_transient_delta_maintenance_error(e):
+ if (
+ is_transient_object_store_error(e)
+ or is_transient_delta_maintenance_error(e)
+ or is_transient_delta_log_listing_race(e)
+ ):
await self._logger.awarning(f"get_delta_table: transient object-store error, not reporting: {e}")
raise TransientObjectStoreError(str(e)) from e
capture_exception(e)
@@ -511,6 +525,14 @@ class DeltaTableRef:
is_transient_delta_maintenance_error) is excluded the same way — otherwise a concurrent purge
racing this open would misread as corruption and trigger a needless destructive revive.
+ Deliberately does not exclude is_transient_delta_log_listing_race (a missing run of commit
+ files, or a dangling `_last_checkpoint` hint): that message shape is also what an interrupted
+ repartition swap leaves behind, so a log still raising it on this independent open must read as
+ corrupt and get revived, not retried forever as if it were the brief race the other classifiers
+ cover. `get_delta_table`'s own open suppresses reporting on it regardless (see
+ `_capture_unless_transient`), since a race that has already cleared by the time this runs never
+ reaches here in the first place.
+
A table that opens stays on this ref as the cached handle, so the next `get_delta_table`
call in the same run does not read the log again. A handle that this ref already holds is
proof that the table opened, so it answers the question with no request.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_delta_errors.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_delta_errors.py
index e3297f26b28a..d7ec688fa662 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_delta_errors.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_delta_errors.py
@@ -12,6 +12,7 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.del
is_invalid_version_race,
is_missing_scan_file_race,
is_offset_overflow_compaction_error,
+ is_transient_delta_log_listing_race,
is_transient_delta_maintenance_error,
is_transient_maintenance_error,
is_transient_object_store_error,
@@ -216,6 +217,28 @@ class TestIsTransientDeltaMaintenanceError:
# "not found" outside the log directory must not match, because a data file missing for some
# other reason is a real failure to capture, not this specific log-commit race.
("file_not_found_outside_delta_log", deltalake.exceptions.DeltaError("File not found: some/file"), False),
+ # A log-segment contiguity gap or a dangling checkpoint hint must NOT be folded into this
+ # classifier even though they are also a purge-vs-concurrent-read race (see
+ # TestIsTransientDeltaLogListingRace): this function also backs `is_table_corrupted`, and
+ # unlike the single-file "not found" case above, these two shapes are also the signature of a
+ # log genuinely left broken by an interrupted repartition swap — which must still be read as
+ # corrupt here so it gets revived instead of retried forever.
+ (
+ "delta_log_gap_not_matched_here",
+ deltalake.exceptions.DeltaError(
+ "Kernel error: Generic delta kernel error: Expected contiguous commit files, but found "
+ "gap: ParsedLogPath { ... version: 10, file_type: Commit } -> ParsedLogPath "
+ "{ ... version: 15, file_type: Commit }"
+ ),
+ False,
+ ),
+ (
+ "delta_log_dangling_checkpoint_hint_not_matched_here",
+ deltalake.exceptions.DeltaError(
+ "Kernel error: Invalid Checkpoint: Had a _last_checkpoint hint but didn't find any checkpoints"
+ ),
+ False,
+ ),
# A reset or repartition swap replaced `_delta_log` under a handle that a prior write (or
# note_deltalite_commit) had already advanced further: `update_incremental` refuses to apply
# the now-lower-numbered log as an increment. Same reset-vs-stale-handle race as the missing
@@ -238,6 +261,38 @@ class TestIsTransientDeltaMaintenanceError:
assert is_transient_delta_maintenance_error(error) is expected
+class TestIsTransientDeltaLogListingRace:
+ @parameterized.expand(
+ [
+ # `_purge_s3_prefix_once`'s bulk delete removes a batch of `_delta_log` commit files in no
+ # guaranteed order, so a concurrent listing can see a run of them already gone from the
+ # middle of the sequence.
+ (
+ "delta_log_gap_from_concurrent_purge",
+ deltalake.exceptions.DeltaError(
+ "Kernel error: Generic delta kernel error: Expected contiguous commit files, but found "
+ "gap: ParsedLogPath { ... version: 10, file_type: Commit } -> ParsedLogPath "
+ "{ ... version: 15, file_type: Commit }"
+ ),
+ True,
+ ),
+ # Same race, but the batch happened to take the checkpoint file a `_last_checkpoint` hint
+ # still points to.
+ (
+ "delta_log_dangling_checkpoint_hint_from_concurrent_purge",
+ deltalake.exceptions.DeltaError(
+ "Kernel error: Invalid Checkpoint: Had a _last_checkpoint hint but didn't find any checkpoints"
+ ),
+ True,
+ ),
+ ("unrelated_delta_error", deltalake.exceptions.DeltaError("no protocol found in delta log"), False),
+ ("wrong_exception_type", RuntimeError("Expected contiguous commit files, but found gap"), False),
+ ]
+ )
+ def test_classifies_delta_log_listing_race(self, _name: str, error: Exception, expected: bool):
+ assert is_transient_delta_log_listing_race(error) is expected
+
+
class TestIsTransientMaintenanceError:
@parameterized.expand(
[
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_table.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_table.py
index 555414797660..eeee595a48f3 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_table.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_table.py
@@ -708,6 +708,37 @@ class TestGetDeltaTableUnrecoverableErrors:
mock_purge.assert_not_awaited()
assert table_ref.is_first_sync is False
+ @pytest.mark.asyncio
+ async def test_open_delta_log_listing_race_is_not_captured_but_still_reraised(self):
+ """The same concurrent-purge race as the checkpoint-object case above, but caught by the
+ kernel's log-segment contiguity check over a batch of files instead of a single file lookup
+ (see is_transient_delta_log_listing_race). Must behave identically: not captured, still
+ reraised as TransientObjectStoreError so Temporal retries."""
+ table_ref = DeltaTableRef(resource_name="t", job=MagicMock(), logger=make_logger())
+ delta_uri = "s3://bucket/team_id/job_id/t"
+
+ gap_error = deltalake.exceptions.DeltaError(
+ "Kernel error: Generic delta kernel error: Expected contiguous commit files, but found "
+ "gap: ParsedLogPath { ... version: 10, file_type: Commit } -> ParsedLogPath "
+ "{ ... version: 15, file_type: Commit }"
+ )
+ module = "products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.table"
+ with (
+ patch.object(table_ref, "_get_delta_table_uri", AsyncMock(return_value=delta_uri)),
+ patch(f"{module}.deltalake.DeltaTable") as mock_delta_table,
+ patch(f"{module}._purge_s3_prefix", AsyncMock()) as mock_purge,
+ patch(f"{module}.capture_exception") as mock_capture,
+ ):
+ mock_delta_table.is_deltatable.return_value = True
+ mock_delta_table.side_effect = gap_error
+
+ with pytest.raises(TransientObjectStoreError):
+ await table_ref.get_delta_table()
+
+ mock_capture.assert_not_called()
+ mock_purge.assert_not_awaited()
+ assert table_ref.is_first_sync is False
+
class TestIsTableCorrupted:
_MODULE = "products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.table"
@@ -739,6 +770,29 @@ class TestIsTableCorrupted:
),
False,
),
+ # Unlike the race above, a log-segment contiguity gap or a dangling checkpoint hint is also
+ # what an interrupted repartition swap leaves behind, so these must read as corrupt here even
+ # though get_delta_table's own open suppresses reporting on them too (see
+ # is_transient_delta_log_listing_race) — a log still raising this on a later attempt gets
+ # revived instead of retried forever.
+ (
+ "delta_log_gap_is_corrupt",
+ True,
+ deltalake.exceptions.DeltaError(
+ "Kernel error: Generic delta kernel error: Expected contiguous commit files, but found "
+ "gap: ParsedLogPath { ... version: 10, file_type: Commit } -> ParsedLogPath "
+ "{ ... version: 15, file_type: Commit }"
+ ),
+ True,
+ ),
+ (
+ "dangling_checkpoint_hint_is_corrupt",
+ True,
+ deltalake.exceptions.DeltaError(
+ "Kernel error: Invalid Checkpoint: Had a _last_checkpoint hint but didn't find any checkpoints"
+ ),
+ True,
+ ),
]
)
@pytest.mark.asyncio