Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-imports): classify Aliyun OSS internal errors as transient (#125129)

Tom Owers0840a5107ef6
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/errors.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/errors.py
index 8aa270c9afe2..5ddcc8a13e48 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/errors.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/errors.py
@@ -48,6 +48,21 @@ def _is_bodyless_5xx_unknown_error(error: BaseException) -> bool:
return match is not None and match.group(1).startswith("5")
+# A warehouse bucket backed by Aliyun OSS instead of S3 (see s3_proxy.py's module docstring on OSS as
+# a supported backend) surfaces a 500 as this OSS-specific error shape instead of the S3 messages
+# above - delta-rs's own object-store layer already marks it "(temporary)" before wrapping OSS's
+# Debug-formatted source struct. Matched on the `code` field rather than the free-text `message`,
+# which, unlike S3's fixed internal-error string, OSS's own InternalError responses don't keep fixed.
+_OSS_INTERNAL_ERROR_NEEDLE = 'OssError { code: "InternalError"'
+
+
+def _is_oss_internal_error(error: BaseException) -> bool:
+ # `is_deltatable()` exposes the raw `OSError`, but table-level operations (vacuum, optimize,
+ # commit) wrap the identical underlying object-store error in `deltalake.exceptions.DeltaError`
+ # instead - same split the S3 needles above already account for.
+ return isinstance(error, OSError | deltalake.exceptions.DeltaError) and _OSS_INTERNAL_ERROR_NEEDLE in str(error)
+
+
class TransientObjectStoreError(NonReportableError):
"""A known-transient object-store blip (see is_transient_object_store_error), re-raised as this
type instead of the original OSError/DeltaError. Skipping the inline capture_exception call
@@ -104,6 +119,10 @@ def is_transient_object_store_error(error: BaseException) -> bool:
5xx, because AWS omits the error body (and so reports the same UNKNOWN code) for a permanent
HeadObject 403 or 404 too.
+ A warehouse bucket backed by Aliyun OSS instead of S3 reports its own 500 "InternalError" as this
+ OSS-specific error shape (see `_is_oss_internal_error`), already marked "(temporary)" by delta-rs's
+ object-store layer before this classifier even runs.
+
Deliberately does not cover `ensure_bucket_exists`'s own exhausted `HeadBucket` 403 retry (see
`_is_exhausted_head_bucket_forbidden` below) — that check only runs under `USE_LOCAL_SETUP`,
where the bucket credentials are operator-configured rather than our own IAM instance role, so an
@@ -130,6 +149,8 @@ def is_transient_object_store_error(error: BaseException) -> bool:
return True
if isinstance(error, OSError) and _is_bodyless_5xx_unknown_error(error):
return True
+ if _is_oss_internal_error(error):
+ return True
return isinstance(error, OSError | deltalake.exceptions.DeltaError) and any(
needle in str(error) for needle in TRANSIENT_OBJECT_STORE_ERRORS
)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_delta_errors.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_delta_errors.py
index d7ec688fa662..aaa18bc830f4 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_delta_errors.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_delta_errors.py
@@ -127,6 +127,44 @@ class TestIsTransientObjectStoreError:
),
False,
),
+ (
+ # A warehouse bucket backed by Aliyun OSS reports its own 500 as this OSS-specific
+ # error shape instead of one of the S3 messages above - matched on the fixed `code`
+ # field since OSS's own InternalError `message` text isn't fixed the way S3's is.
+ "oss_internal_error",
+ OSError(
+ "Generic Unexpected error: Unexpected (temporary) at list, context: { uri: "
+ "https://example-bucket.oss-us-west-1.aliyuncs.com/?list-type=2&prefix=_delta_log/, "
+ "response: Parts { status: 500, service: oss } } => "
+ 'OssError { code: "InternalError", message: "Please contact the server administrator, '
+ '[email protected]", request_id: "6AE261CD76EC4B363925E506" }'
+ ),
+ True,
+ ),
+ (
+ # A different OSS error code sharing the same wrapper shape must not be swept up by
+ # the code-field match - only "InternalError" is a transient 5xx.
+ "oss_access_denied_not_transient",
+ OSError(
+ "Generic Unexpected error: Unexpected at list, context: { uri: "
+ "https://example-bucket.oss-us-west-1.aliyuncs.com/?list-type=2&prefix=_delta_log/ } => "
+ 'OssError { code: "AccessDenied", message: "Access denied by bucket policy" }'
+ ),
+ False,
+ ),
+ (
+ # Table-level operations (vacuum, optimize, commit) wrap the identical underlying
+ # object-store error in `DeltaError` instead of a bare `OSError` - the same OSS 500
+ # must still classify as transient through that wrapper.
+ "oss_internal_error_wrapped_in_delta_error",
+ deltalake.exceptions.DeltaError(
+ "Generic error: Unexpected (temporary) at commit, context: { uri: "
+ "https://example-bucket.oss-us-west-1.aliyuncs.com/_delta_log/00000000000000000001.json } => "
+ 'OssError { code: "InternalError", message: "Please contact the server administrator, '
+ '[email protected]", request_id: "6AE261CD76EC4B363925E507" }'
+ ),
+ True,
+ ),
# `get_delta_table` re-raises a recognized transient blip as this wrapper (see
# `_capture_unless_transient`) instead of the original OSError/DeltaError. A caller
# further up the stack that catches broadly and re-runs this classifier on the caught