Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-imports): classify transient errors in delta history scans (#125144)

Tom Owerse8b36c4882bd
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/errors.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/errors.py
index b261e4d6227e..b1abcd825491 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/errors.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/errors.py
@@ -32,6 +32,11 @@ _EREMOTEIO = getattr(errno, "EREMOTEIO", errno.EIO)
# reporting a connection the remote end dropped mid-response - it already prefixes this with its own
# "Unexpected (temporary)" classification before this classifier even runs, so a retry against a fresh
# connection is exactly what self-heals it, the same as any other blip in this list.
+# - "Connection timed out" is the bare OS-level message a dropped TCP handshake/response leaves behind
+# when our own bucket is a non-AWS S3-compatible store reached through delta-rs's generic S3 client
+# with a custom `endpoint_url` (e.g. Alibaba Cloud OSS's S3-compatible API, addressed virtual-hosted
+# per products/data_warehouse/backend/s3_proxy.py) rather than AWS's own endpoints - that path never
+# goes through s3fs/aiobotocore, so it can't surface as one of the typed botocore exceptions below.
# A retry (of the same idempotent operation) clears these, so they shouldn't be treated the same as a
# bug in our logic.
TRANSIENT_OBJECT_STORE_ERRORS = (
@@ -42,6 +47,7 @@ TRANSIENT_OBJECT_STORE_ERRORS = (
"We encountered an internal error. Please try again.",
"The difference between the request time and the current time is too large.",
"connection closed before message completed",
+ "Connection timed out",
)
# pyarrow's S3FileSystem (the AWS SDK for C++) reports a response it can't classify as the fixed
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_delta_errors.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_delta_errors.py
index 0795b47c4b08..2c2ab5245b00 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_delta_errors.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_delta_errors.py
@@ -190,6 +190,19 @@ class TestIsTransientObjectStoreError:
),
True,
),
+ (
+ # delta-rs's generic S3 client reaching a non-AWS S3-compatible bucket (e.g. Alibaba
+ # OSS, addressed virtual-hosted per s3_proxy.py) never goes through s3fs/botocore, so a
+ # dropped connection surfaces as a bare OSError with this fixed OS-level message instead
+ # of one of the typed botocore exceptions above.
+ "non_aws_endpoint_connection_timed_out",
+ OSError(
+ "Generic error: Unexpected (temporary) at stat, context: ... => send http request, "
+ "source: error sending request: client error (SendRequest): connection error: "
+ "Connection timed out (os error 110)"
+ ),
+ True,
+ ),
# `get_delta_table` re-raises a recognized transient blip as this wrapper (see
# `_capture_unless_transient`) instead of the original OSError/DeltaError. A caller
# further up the stack that catches broadly and re-runs this classifier on the caught
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_writer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_writer.py
index c844849864c9..ed5b2113d316 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_writer.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_writer.py
@@ -31,6 +31,9 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.con
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.deltalite_handles import (
DeltaLiteHandleCache,
)
+from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.errors import (
+ TransientObjectStoreError,
+)
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.maintenance import DeltaMaintenance
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.memory_governor import (
GovernorConfig,
@@ -39,6 +42,7 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.del
)
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.ops import (
DELTA_TABLE_PROPERTIES,
+ ObjectStorePermissionDeniedError,
ensure_table_properties,
)
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.rss_sampler import RssPeakSampler
@@ -188,6 +192,39 @@ class TestHasCommitWithMetadata:
mock_delta.history.assert_called_once_with(limit=123)
+ @parameterized.expand(
+ [
+ (
+ # A bare transient OSError/DeltaError from `history()` must not reach the caller
+ # unclassified: idempotency.py's `is_batch_already_processed` only skips
+ # `capture_exception` for a `NonReportableError`, so an unwrapped blip here mints a
+ # fresh error-tracking issue per retry.
+ "transient_connection_timeout",
+ OSError("Connection timed out (os error 110)"),
+ TransientObjectStoreError,
+ ),
+ (
+ # The transient matcher's "Generic S3 error" needle also prefixes a permanent
+ # privilege refusal - checking it before the permission-denied check would wrap a
+ # reportable bug as a silently-retried blip instead.
+ "permission_denied_prefixed_like_a_generic_s3_error",
+ OSError("Generic S3 error: The operation lacked the necessary privileges to complete"),
+ ObjectStorePermissionDeniedError,
+ ),
+ ]
+ )
+ @pytest.mark.asyncio
+ async def test_history_read_error_is_classified_not_raised_raw(
+ self, _name: str, error: OSError, expected_type: type[Exception]
+ ):
+ writer = _make_writer()
+ mock_delta = MagicMock()
+ mock_delta.history = MagicMock(side_effect=error)
+
+ with patch.object(writer._table, "get_delta_table", AsyncMock(return_value=mock_delta)):
+ with pytest.raises(expected_type):
+ await writer.has_commit_with_metadata({"k": "v"})
+
class TestHasBatchBeenCommitted:
@parameterized.expand(
@@ -234,6 +271,34 @@ class TestHasBatchBeenCommitted:
assert await writer.has_batch_been_committed("run-1", 5) is expected
assert table.history.call_count <= 3
+ @pytest.mark.asyncio
+ async def test_transient_history_read_error_is_wrapped_not_raised_raw(self):
+ # Same guard as has_commit_with_metadata's equivalent test: this is the exact call path the
+ # idempotency crash-recovery check (`is_batch_already_processed`) drives on a Redis miss, so a
+ # raw transient blip here reaches error tracking as a bug instead of being retried silently.
+ writer = _make_writer()
+ table = MagicMock()
+ table.history = MagicMock(side_effect=OSError("Connection timed out (os error 110)"))
+
+ with patch.object(writer._table, "get_delta_table", AsyncMock(return_value=table)):
+ with pytest.raises(TransientObjectStoreError):
+ await writer.has_batch_been_committed("run-1", 5)
+
+ @pytest.mark.asyncio
+ async def test_permission_denied_history_read_error_is_not_classified_as_transient(self):
+ # Same ordering guard as has_commit_with_metadata's equivalent case: a permanent privilege
+ # refusal happens to share the "Generic S3 error" prefix the transient matcher accepts, so the
+ # permission check must run first or this becomes a silently-retried blip.
+ writer = _make_writer()
+ table = MagicMock()
+ table.history = MagicMock(
+ side_effect=OSError("Generic S3 error: The operation lacked the necessary privileges to complete")
+ )
+
+ with patch.object(writer._table, "get_delta_table", AsyncMock(return_value=table)):
+ with pytest.raises(ObjectStorePermissionDeniedError):
+ await writer.has_batch_been_committed("run-1", 5)
+
class TestWriteToDeltalakeCommitMetadataPassThrough:
"""Covers that commit_metadata is forwarded to deltalake.write_deltalake as CommitProperties."""
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/writer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/writer.py
index 39a2e9ad8cb7..6f8fda98dde2 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/writer.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/writer.py
@@ -26,12 +26,20 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.arr
relax_batch_nullability,
)
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.consts import PARTITION_KEY
+from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.errors import (
+ TransientObjectStoreError,
+ is_transient_object_store_error,
+)
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.evolution import evolve_delta_schema
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.ops import (
DELTA_TABLE_PROPERTIES,
+ OBJECT_STORE_PERMISSION_DENIED_MESSAGE,
+ OBJECT_STORE_TRANSIENT_MESSAGE,
+ ObjectStorePermissionDeniedError,
delta_merge_spill_kwargs,
ensure_table_properties,
execute_with_conflict_retry,
+ is_object_store_permission_denied,
)
from products.warehouse_sources.backend.temporal.data_imports.workload_report import report_buffer_bytes, report_phase
@@ -903,6 +911,26 @@ class DeltaWriter:
deleted += int(metrics.get("num_deleted_rows") or 0)
return deleted
+ async def _read_history(self, delta_table: deltalake.DeltaTable, limit: int) -> list[dict[str, Any]]:
+ """Read delta commit history, classifying object-store errors the same way every other
+ read/write here already does (see `execute_with_conflict_retry`), instead of letting one
+ reach the caller as a bare, unclassified OSError/DeltaError.
+
+ Checked in the same order as `execute_with_conflict_retry`: a permission denial first, since
+ the broad transient-error matcher's "Generic S3 error" needle also prefixes a permanent
+ privilege refusal, and classifying that as transient would silently drop a reportable bug.
+ """
+ try:
+ return await asyncio.to_thread(delta_table.history, limit=limit)
+ except (OSError, deltalake.exceptions.DeltaError) as e:
+ if is_object_store_permission_denied(e):
+ await self._logger.awarning(f"history: the object store denied the read ({type(e).__name__})")
+ raise ObjectStorePermissionDeniedError(OBJECT_STORE_PERMISSION_DENIED_MESSAGE) from e
+ if is_transient_object_store_error(e):
+ await self._logger.awarning(f"history: transient object-store error, not reporting: {e}")
+ raise TransientObjectStoreError(OBJECT_STORE_TRANSIENT_MESSAGE) from e
+ raise
+
async def has_commit_with_metadata(self, match: dict[str, str], *, scan_limit: int = 50) -> bool:
"""Check whether any recent delta commit has custom metadata matching all entries in `match`.
@@ -920,7 +948,7 @@ class DeltaWriter:
if delta_table is None:
return False
- history = await asyncio.to_thread(delta_table.history, limit=scan_limit)
+ history = await self._read_history(delta_table, scan_limit)
for commit in history:
if commit_matches(commit, match):
@@ -941,7 +969,7 @@ class DeltaWriter:
limit = scan_limit
while True:
- history = await asyncio.to_thread(delta_table.history, limit=limit)
+ history = await self._read_history(delta_table, limit)
if any(commit_covers_batch(commit, run_uuid, batch_index) for commit in history):
return True
data_commits = sum(1 for commit in history if commit.get("operation") not in _MAINTENANCE_OPERATIONS)