Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-imports): stop retrying object-store builder errors forever (#129969)

Tom Owersbf448d065cc6
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/errors.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/errors.py
index b1abcd825491..aa9ef33caf68 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/errors.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/errors.py
@@ -80,6 +80,20 @@ def _is_oss_internal_error(error: BaseException) -> bool:
return isinstance(error, OSError | deltalake.exceptions.DeltaError) and _OSS_INTERNAL_ERROR_NEEDLE in str(error)
+# delta-rs's Rust `object_store` crate raises this for a failure to even build the HTTP client —
+# e.g. "URL scheme is not allowed" when the endpoint is `http://` and `AWS_ALLOW_HTTP` isn't set,
+# or any other invalid builder config. Unlike every blip above, the client is rebuilt fresh from
+# the same storage options on every call, so a builder error is deterministic and can never clear
+# on retry.
+_OBJECT_STORE_BUILDER_ERROR_NEEDLE = "builder error"
+
+
+def _is_object_store_builder_error(error: BaseException) -> bool:
+ return isinstance(error, OSError | deltalake.exceptions.DeltaError) and _OBJECT_STORE_BUILDER_ERROR_NEEDLE in str(
+ error
+ )
+
+
class TransientObjectStoreError(NonReportableError):
"""A known-transient object-store blip (see is_transient_object_store_error), re-raised as this
type instead of the original OSError/DeltaError. Skipping the inline capture_exception call
@@ -146,6 +160,14 @@ def is_transient_object_store_error(error: BaseException) -> bool:
OSS-specific error shape (see `_is_oss_internal_error`), already marked "(temporary)" by delta-rs's
object-store layer before this classifier even runs.
+ Checked and excluded before the broad needle match: a failed client *construction* (see
+ `_is_object_store_builder_error`) is also prefixed "Generic S3 error" by delta-rs, but it is a
+ static config problem (e.g. a plain `http://` endpoint without `AWS_ALLOW_HTTP`), not a blip —
+ the same client is rebuilt from the same options on every retry, so it fails the identical way
+ forever. Same reasoning as the permission-denied carve-out `_capture_unless_transient` checks
+ ahead of this function: a definitive failure arriving under the shared "Generic S3 error" prefix
+ must not be hidden behind Temporal's retry policy.
+
Deliberately does not cover `ensure_bucket_exists`'s own exhausted `HeadBucket` 403 retry (see
`_is_exhausted_head_bucket_forbidden` below) — that check only runs under `USE_LOCAL_SETUP`,
where the bucket credentials are operator-configured rather than our own IAM instance role, so an
@@ -174,6 +196,15 @@ def is_transient_object_store_error(error: BaseException) -> bool:
return True
if _is_oss_internal_error(error):
return True
+ if _is_object_store_builder_error(error):
+ # Checked before the broad needle match below: delta-rs prefixes a failed *client
+ # construction* with the same "Generic S3 error" text the blips above use, so the needle
+ # would otherwise call this transient too. A builder error (e.g. "URL scheme is not
+ # allowed" for an http:// endpoint without AWS_ALLOW_HTTP) means the client itself is
+ # misconfigured — every retry builds the identical client and fails the identical way, so
+ # folding it into the blip list would retry it forever without ever telling anyone, same as
+ # the permission-denied refusal carved out above it in `_capture_unless_transient`.
+ return False
return isinstance(error, OSError | deltalake.exceptions.DeltaError) and any(
needle in str(error) for needle in TRANSIENT_OBJECT_STORE_ERRORS
)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_delta_errors.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_delta_errors.py
index 2c2ab5245b00..7734e8973dbe 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_delta_errors.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_delta_errors.py
@@ -208,6 +208,19 @@ class TestIsTransientObjectStoreError:
# further up the stack that catches broadly and re-runs this classifier on the caught
# exception sees the wrapper, not the original — it must still read as transient.
("already_wrapped_transient_error", TransientObjectStoreError("Please reduce your request rate"), True),
+ (
+ # delta-rs prefixes a failed client *construction* with the same "Generic S3 error"
+ # text the blips above use, so without the builder-error carve-out this would also
+ # match the broad needle and read as transient — but an http:// endpoint without
+ # AWS_ALLOW_HTTP fails the identical way on every retry, so it must not.
+ "object_store_builder_error_url_scheme_not_allowed",
+ OSError(
+ "Generic S3 error: Error performing list request: Error performing GET "
+ "http://objectstorage:19000/data-warehouse?list-type=2&prefix=_delta_log/ "
+ "in 8.225µs - HTTP error: builder error: URL scheme is not allowed"
+ ),
+ False,
+ ),
(
# An exhausted `ensure_bucket_exists` HeadBucket 403 retry is deliberately NOT folded
# in here: under `USE_LOCAL_SETUP` the credentials are operator-configured, so this