:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(data-imports): classify s3fs InternalError errno as transient (#125131)
Tom Owers911e0201f03e
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/errors.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/errors.pyindex b53708d50145..b261e4d6227e 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/errors.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/errors.py@@ -9,6 +9,12 @@ import deltalake.exceptionsfrom posthog.temporal.common.errors import NonReportableError+# `errno` only defines EREMOTEIO on platforms that have it (Linux); referencing it directly breaks+# import on a Mac dev machine. s3fs's own translation table (s3fs/errors.py) falls back to EIO for+# the same reason, so matching with the identical fallback stays consistent with what s3fs actually+# sets there.+_EREMOTEIO = getattr(errno, "EREMOTEIO", errno.EIO)+# Substrings of the object-store errors raised talking to our own S3-backed data-warehouse bucket# that are transient and self-recovering, not a bug in our code or a customer credential problem:# - the first three come from delta-rs's Rust `object_store` crate inside `DeltaTable.is_deltatable()`@@ -119,6 +125,12 @@ def is_transient_object_store_error(error: BaseException) -> bool:covers the 503 "Service Unavailable" case the message needles below miss, without matching onthe vendor's free text.+ A bare `OSError` with errno `EREMOTEIO` is the same `translate_boto_error` table's entry for+ `InternalError`/bare "500" - the exact blip the "We encountered an internal error" needle below+ names, but recognized by errno instead so an S3-compatible provider that returns its own wording+ for that response code (self-hosted setups routing the data-warehouse bucket through a non-AWS+ endpoint) still matches without a needle per vendor.+A bodyless 5xx that pyarrow's S3FileSystem reports as "AWS Error UNKNOWN" (see`_is_bodyless_5xx_unknown_error`) is also transient - but only once its status is confirmed to be5xx, because AWS omits the error body (and so reports the same UNKNOWN code) for a permanent@@ -150,7 +162,7 @@ def is_transient_object_store_error(error: BaseException) -> bool:# re-runs this classifier on the wrapper, rather than the original OSError/DeltaError it# wraps, must still treat it as transient.return True- if isinstance(error, OSError) and error.errno in (errno.EMFILE, errno.ENFILE, errno.EBUSY):+ if isinstance(error, OSError) and error.errno in (errno.EMFILE, errno.ENFILE, errno.EBUSY, _EREMOTEIO):return Trueif isinstance(error, OSError) and _is_bodyless_5xx_unknown_error(error):return Truediff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_delta_errors.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_delta_errors.pyindex bb96f898deaa..0795b47c4b08 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_delta_errors.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_delta_errors.py@@ -8,6 +8,7 @@ import botocore.exceptionsfrom parameterized import parameterizedfrom products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.errors import (+ _EREMOTEIO,TransientObjectStoreError,is_invalid_version_race,is_missing_scan_file_race,@@ -105,6 +106,14 @@ class TestIsTransientObjectStoreError:OSError(errno.EBUSY, "Service Unavailable"),True,),+ (+ # A non-AWS S3-compatible provider's own wording for the same InternalError/500+ # response s3fs always maps to this errno (see s3fs/errors.py::translate_boto_error) -+ # the message needle above only catches AWS's fixed text, so this must match by errno.+ "s3_compatible_internal_error_non_aws_wording_eremoteio",+ OSError(_EREMOTEIO, "Please contact the server administrator, [email protected]"),+ True,+ ),(# A different errno must not be swept up by the fd-exhaustion check."unrelated_os_error_with_errno",