:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(data-imports): classify Aliyun OSS internal errors as transient (#125129)
Tom Owers0840a5107ef6
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/errors.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/errors.pyindex 8aa270c9afe2..5ddcc8a13e48 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/errors.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/errors.py@@ -48,6 +48,21 @@ def _is_bodyless_5xx_unknown_error(error: BaseException) -> bool:return match is not None and match.group(1).startswith("5")+# A warehouse bucket backed by Aliyun OSS instead of S3 (see s3_proxy.py's module docstring on OSS as+# a supported backend) surfaces a 500 as this OSS-specific error shape instead of the S3 messages+# above - delta-rs's own object-store layer already marks it "(temporary)" before wrapping OSS's+# Debug-formatted source struct. Matched on the `code` field rather than the free-text `message`,+# which, unlike S3's fixed internal-error string, OSS's own InternalError responses don't keep fixed.+_OSS_INTERNAL_ERROR_NEEDLE = 'OssError { code: "InternalError"'+++def _is_oss_internal_error(error: BaseException) -> bool:+ # `is_deltatable()` exposes the raw `OSError`, but table-level operations (vacuum, optimize,+ # commit) wrap the identical underlying object-store error in `deltalake.exceptions.DeltaError`+ # instead - same split the S3 needles above already account for.+ return isinstance(error, OSError | deltalake.exceptions.DeltaError) and _OSS_INTERNAL_ERROR_NEEDLE in str(error)++class TransientObjectStoreError(NonReportableError):"""A known-transient object-store blip (see is_transient_object_store_error), re-raised as thistype instead of the original OSError/DeltaError. Skipping the inline capture_exception call@@ -104,6 +119,10 @@ def is_transient_object_store_error(error: BaseException) -> bool:5xx, because AWS omits the error body (and so reports the same UNKNOWN code) for a permanentHeadObject 403 or 404 too.+ A warehouse bucket backed by Aliyun OSS instead of S3 reports its own 500 "InternalError" as this+ OSS-specific error shape (see `_is_oss_internal_error`), already marked "(temporary)" by delta-rs's+ object-store layer before this classifier even runs.+Deliberately does not cover `ensure_bucket_exists`'s own exhausted `HeadBucket` 403 retry (see`_is_exhausted_head_bucket_forbidden` below) — that check only runs under `USE_LOCAL_SETUP`,where the bucket credentials are operator-configured rather than our own IAM instance role, so an@@ -130,6 +149,8 @@ def is_transient_object_store_error(error: BaseException) -> bool:return Trueif isinstance(error, OSError) and _is_bodyless_5xx_unknown_error(error):return True+ if _is_oss_internal_error(error):+ return Truereturn isinstance(error, OSError | deltalake.exceptions.DeltaError) and any(needle in str(error) for needle in TRANSIENT_OBJECT_STORE_ERRORS)diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_delta_errors.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_delta_errors.pyindex d7ec688fa662..aaa18bc830f4 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_delta_errors.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_delta_errors.py@@ -127,6 +127,44 @@ class TestIsTransientObjectStoreError:),False,),+ (+ # A warehouse bucket backed by Aliyun OSS reports its own 500 as this OSS-specific+ # error shape instead of one of the S3 messages above - matched on the fixed `code`+ # field since OSS's own InternalError `message` text isn't fixed the way S3's is.+ "oss_internal_error",+ OSError(+ "Generic Unexpected error: Unexpected (temporary) at list, context: { uri: "+ "https://example-bucket.oss-us-west-1.aliyuncs.com/?list-type=2&prefix=_delta_log/, "+ "response: Parts { status: 500, service: oss } } => "+ 'OssError { code: "InternalError", message: "Please contact the server administrator, '+ '[email protected]", request_id: "6AE261CD76EC4B363925E506" }'+ ),+ True,+ ),+ (+ # A different OSS error code sharing the same wrapper shape must not be swept up by+ # the code-field match - only "InternalError" is a transient 5xx.+ "oss_access_denied_not_transient",+ OSError(+ "Generic Unexpected error: Unexpected at list, context: { uri: "+ "https://example-bucket.oss-us-west-1.aliyuncs.com/?list-type=2&prefix=_delta_log/ } => "+ 'OssError { code: "AccessDenied", message: "Access denied by bucket policy" }'+ ),+ False,+ ),+ (+ # Table-level operations (vacuum, optimize, commit) wrap the identical underlying+ # object-store error in `DeltaError` instead of a bare `OSError` - the same OSS 500+ # must still classify as transient through that wrapper.+ "oss_internal_error_wrapped_in_delta_error",+ deltalake.exceptions.DeltaError(+ "Generic error: Unexpected (temporary) at commit, context: { uri: "+ "https://example-bucket.oss-us-west-1.aliyuncs.com/_delta_log/00000000000000000001.json } => "+ 'OssError { code: "InternalError", message: "Please contact the server administrator, '+ '[email protected]", request_id: "6AE261CD76EC4B363925E507" }'+ ),+ True,+ ),# `get_delta_table` re-raises a recognized transient blip as this wrapper (see# `_capture_unless_transient`) instead of the original OSError/DeltaError. A caller# further up the stack that catches broadly and re-runs this classifier on the caught