:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(data-imports): classify transient errors in delta history scans (#125144)
Tom Owerse8b36c4882bd
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/errors.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/errors.pyindex b261e4d6227e..b1abcd825491 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/errors.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/errors.py@@ -32,6 +32,11 @@ _EREMOTEIO = getattr(errno, "EREMOTEIO", errno.EIO)# reporting a connection the remote end dropped mid-response - it already prefixes this with its own# "Unexpected (temporary)" classification before this classifier even runs, so a retry against a fresh# connection is exactly what self-heals it, the same as any other blip in this list.+# - "Connection timed out" is the bare OS-level message a dropped TCP handshake/response leaves behind+# when our own bucket is a non-AWS S3-compatible store reached through delta-rs's generic S3 client+# with a custom `endpoint_url` (e.g. Alibaba Cloud OSS's S3-compatible API, addressed virtual-hosted+# per products/data_warehouse/backend/s3_proxy.py) rather than AWS's own endpoints - that path never+# goes through s3fs/aiobotocore, so it can't surface as one of the typed botocore exceptions below.# A retry (of the same idempotent operation) clears these, so they shouldn't be treated the same as a# bug in our logic.TRANSIENT_OBJECT_STORE_ERRORS = (@@ -42,6 +47,7 @@ TRANSIENT_OBJECT_STORE_ERRORS = ("We encountered an internal error. Please try again.","The difference between the request time and the current time is too large.","connection closed before message completed",+ "Connection timed out",)# pyarrow's S3FileSystem (the AWS SDK for C++) reports a response it can't classify as the fixeddiff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_delta_errors.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_delta_errors.pyindex 0795b47c4b08..2c2ab5245b00 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_delta_errors.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_delta_errors.py@@ -190,6 +190,19 @@ class TestIsTransientObjectStoreError:),True,),+ (+ # delta-rs's generic S3 client reaching a non-AWS S3-compatible bucket (e.g. Alibaba+ # OSS, addressed virtual-hosted per s3_proxy.py) never goes through s3fs/botocore, so a+ # dropped connection surfaces as a bare OSError with this fixed OS-level message instead+ # of one of the typed botocore exceptions above.+ "non_aws_endpoint_connection_timed_out",+ OSError(+ "Generic error: Unexpected (temporary) at stat, context: ... => send http request, "+ "source: error sending request: client error (SendRequest): connection error: "+ "Connection timed out (os error 110)"+ ),+ True,+ ),# `get_delta_table` re-raises a recognized transient blip as this wrapper (see# `_capture_unless_transient`) instead of the original OSError/DeltaError. A caller# further up the stack that catches broadly and re-runs this classifier on the caughtdiff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_writer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_writer.pyindex c844849864c9..ed5b2113d316 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_writer.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_writer.py@@ -31,6 +31,9 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.confrom products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.deltalite_handles import (DeltaLiteHandleCache,)+from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.errors import (+ TransientObjectStoreError,+)from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.maintenance import DeltaMaintenancefrom products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.memory_governor import (GovernorConfig,@@ -39,6 +42,7 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.del)from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.ops import (DELTA_TABLE_PROPERTIES,+ ObjectStorePermissionDeniedError,ensure_table_properties,)from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.rss_sampler import RssPeakSampler@@ -188,6 +192,39 @@ class TestHasCommitWithMetadata:mock_delta.history.assert_called_once_with(limit=123)+ @parameterized.expand(+ [+ (+ # A bare transient OSError/DeltaError from `history()` must not reach the caller+ # unclassified: idempotency.py's `is_batch_already_processed` only skips+ # `capture_exception` for a `NonReportableError`, so an unwrapped blip here mints a+ # fresh error-tracking issue per retry.+ "transient_connection_timeout",+ OSError("Connection timed out (os error 110)"),+ TransientObjectStoreError,+ ),+ (+ # The transient matcher's "Generic S3 error" needle also prefixes a permanent+ # privilege refusal - checking it before the permission-denied check would wrap a+ # reportable bug as a silently-retried blip instead.+ "permission_denied_prefixed_like_a_generic_s3_error",+ OSError("Generic S3 error: The operation lacked the necessary privileges to complete"),+ ObjectStorePermissionDeniedError,+ ),+ ]+ )+ @pytest.mark.asyncio+ async def test_history_read_error_is_classified_not_raised_raw(+ self, _name: str, error: OSError, expected_type: type[Exception]+ ):+ writer = _make_writer()+ mock_delta = MagicMock()+ mock_delta.history = MagicMock(side_effect=error)++ with patch.object(writer._table, "get_delta_table", AsyncMock(return_value=mock_delta)):+ with pytest.raises(expected_type):+ await writer.has_commit_with_metadata({"k": "v"})+class TestHasBatchBeenCommitted:@parameterized.expand(@@ -234,6 +271,34 @@ class TestHasBatchBeenCommitted:assert await writer.has_batch_been_committed("run-1", 5) is expectedassert table.history.call_count <= 3+ @pytest.mark.asyncio+ async def test_transient_history_read_error_is_wrapped_not_raised_raw(self):+ # Same guard as has_commit_with_metadata's equivalent test: this is the exact call path the+ # idempotency crash-recovery check (`is_batch_already_processed`) drives on a Redis miss, so a+ # raw transient blip here reaches error tracking as a bug instead of being retried silently.+ writer = _make_writer()+ table = MagicMock()+ table.history = MagicMock(side_effect=OSError("Connection timed out (os error 110)"))++ with patch.object(writer._table, "get_delta_table", AsyncMock(return_value=table)):+ with pytest.raises(TransientObjectStoreError):+ await writer.has_batch_been_committed("run-1", 5)++ @pytest.mark.asyncio+ async def test_permission_denied_history_read_error_is_not_classified_as_transient(self):+ # Same ordering guard as has_commit_with_metadata's equivalent case: a permanent privilege+ # refusal happens to share the "Generic S3 error" prefix the transient matcher accepts, so the+ # permission check must run first or this becomes a silently-retried blip.+ writer = _make_writer()+ table = MagicMock()+ table.history = MagicMock(+ side_effect=OSError("Generic S3 error: The operation lacked the necessary privileges to complete")+ )++ with patch.object(writer._table, "get_delta_table", AsyncMock(return_value=table)):+ with pytest.raises(ObjectStorePermissionDeniedError):+ await writer.has_batch_been_committed("run-1", 5)+class TestWriteToDeltalakeCommitMetadataPassThrough:"""Covers that commit_metadata is forwarded to deltalake.write_deltalake as CommitProperties."""diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/writer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/writer.pyindex 39a2e9ad8cb7..6f8fda98dde2 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/writer.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/writer.py@@ -26,12 +26,20 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.arrrelax_batch_nullability,)from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.consts import PARTITION_KEY+from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.errors import (+ TransientObjectStoreError,+ is_transient_object_store_error,+)from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.evolution import evolve_delta_schemafrom products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.ops import (DELTA_TABLE_PROPERTIES,+ OBJECT_STORE_PERMISSION_DENIED_MESSAGE,+ OBJECT_STORE_TRANSIENT_MESSAGE,+ ObjectStorePermissionDeniedError,delta_merge_spill_kwargs,ensure_table_properties,execute_with_conflict_retry,+ is_object_store_permission_denied,)from products.warehouse_sources.backend.temporal.data_imports.workload_report import report_buffer_bytes, report_phase@@ -903,6 +911,26 @@ class DeltaWriter:deleted += int(metrics.get("num_deleted_rows") or 0)return deleted+ async def _read_history(self, delta_table: deltalake.DeltaTable, limit: int) -> list[dict[str, Any]]:+ """Read delta commit history, classifying object-store errors the same way every other+ read/write here already does (see `execute_with_conflict_retry`), instead of letting one+ reach the caller as a bare, unclassified OSError/DeltaError.++ Checked in the same order as `execute_with_conflict_retry`: a permission denial first, since+ the broad transient-error matcher's "Generic S3 error" needle also prefixes a permanent+ privilege refusal, and classifying that as transient would silently drop a reportable bug.+ """+ try:+ return await asyncio.to_thread(delta_table.history, limit=limit)+ except (OSError, deltalake.exceptions.DeltaError) as e:+ if is_object_store_permission_denied(e):+ await self._logger.awarning(f"history: the object store denied the read ({type(e).__name__})")+ raise ObjectStorePermissionDeniedError(OBJECT_STORE_PERMISSION_DENIED_MESSAGE) from e+ if is_transient_object_store_error(e):+ await self._logger.awarning(f"history: transient object-store error, not reporting: {e}")+ raise TransientObjectStoreError(OBJECT_STORE_TRANSIENT_MESSAGE) from e+ raise+async def has_commit_with_metadata(self, match: dict[str, str], *, scan_limit: int = 50) -> bool:"""Check whether any recent delta commit has custom metadata matching all entries in `match`.@@ -920,7 +948,7 @@ class DeltaWriter:if delta_table is None:return False- history = await asyncio.to_thread(delta_table.history, limit=scan_limit)+ history = await self._read_history(delta_table, scan_limit)for commit in history:if commit_matches(commit, match):@@ -941,7 +969,7 @@ class DeltaWriter:limit = scan_limitwhile True:- history = await asyncio.to_thread(delta_table.history, limit=limit)+ history = await self._read_history(delta_table, limit)if any(commit_covers_batch(commit, run_uuid, batch_index) for commit in history):return Truedata_commits = sum(1 for commit in history if commit.get("operation") not in _MAINTENANCE_OPERATIONS)