Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-imports): classify s3fs InternalError errno as transient (#125131)

Tom Owers911e0201f03e
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/errors.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/errors.py
index b53708d50145..b261e4d6227e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/errors.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/errors.py
@@ -9,6 +9,12 @@ import deltalake.exceptions
from posthog.temporal.common.errors import NonReportableError
+# `errno` only defines EREMOTEIO on platforms that have it (Linux); referencing it directly breaks
+# import on a Mac dev machine. s3fs's own translation table (s3fs/errors.py) falls back to EIO for
+# the same reason, so matching with the identical fallback stays consistent with what s3fs actually
+# sets there.
+_EREMOTEIO = getattr(errno, "EREMOTEIO", errno.EIO)
+
# Substrings of the object-store errors raised talking to our own S3-backed data-warehouse bucket
# that are transient and self-recovering, not a bug in our code or a customer credential problem:
# - the first three come from delta-rs's Rust `object_store` crate inside `DeltaTable.is_deltatable()`
@@ -119,6 +125,12 @@ def is_transient_object_store_error(error: BaseException) -> bool:
covers the 503 "Service Unavailable" case the message needles below miss, without matching on
the vendor's free text.
+ A bare `OSError` with errno `EREMOTEIO` is the same `translate_boto_error` table's entry for
+ `InternalError`/bare "500" - the exact blip the "We encountered an internal error" needle below
+ names, but recognized by errno instead so an S3-compatible provider that returns its own wording
+ for that response code (self-hosted setups routing the data-warehouse bucket through a non-AWS
+ endpoint) still matches without a needle per vendor.
+
A bodyless 5xx that pyarrow's S3FileSystem reports as "AWS Error UNKNOWN" (see
`_is_bodyless_5xx_unknown_error`) is also transient - but only once its status is confirmed to be
5xx, because AWS omits the error body (and so reports the same UNKNOWN code) for a permanent
@@ -150,7 +162,7 @@ def is_transient_object_store_error(error: BaseException) -> bool:
# re-runs this classifier on the wrapper, rather than the original OSError/DeltaError it
# wraps, must still treat it as transient.
return True
- if isinstance(error, OSError) and error.errno in (errno.EMFILE, errno.ENFILE, errno.EBUSY):
+ if isinstance(error, OSError) and error.errno in (errno.EMFILE, errno.ENFILE, errno.EBUSY, _EREMOTEIO):
return True
if isinstance(error, OSError) and _is_bodyless_5xx_unknown_error(error):
return True
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_delta_errors.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_delta_errors.py
index bb96f898deaa..0795b47c4b08 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_delta_errors.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_delta_errors.py
@@ -8,6 +8,7 @@ import botocore.exceptions
from parameterized import parameterized
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.errors import (
+ _EREMOTEIO,
TransientObjectStoreError,
is_invalid_version_race,
is_missing_scan_file_race,
@@ -105,6 +106,14 @@ class TestIsTransientObjectStoreError:
OSError(errno.EBUSY, "Service Unavailable"),
True,
),
+ (
+ # A non-AWS S3-compatible provider's own wording for the same InternalError/500
+ # response s3fs always maps to this errno (see s3fs/errors.py::translate_boto_error) -
+ # the message needle above only catches AWS's fixed text, so this must match by errno.
+ "s3_compatible_internal_error_non_aws_wording_eremoteio",
+ OSError(_EREMOTEIO, "Please contact the server administrator, [email protected]"),
+ True,
+ ),
(
# A different errno must not be swept up by the fd-exhaustion check.
"unrelated_os_error_with_errno",