Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-imports): classify a dropped object-store connection as transient (#125134)

Tom Owers6ca916660667
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/errors.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/errors.py
index 5ddcc8a13e48..b53708d50145 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/errors.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/errors.py
@@ -22,6 +22,10 @@ from posthog.temporal.common.errors import NonReportableError
# onto the same generic PermissionError (s3fs/errors.py::translate_boto_error), so this message - not
# a permission denial - is the only way to tell the two apart. The worker's own clock resyncs and the
# identical request succeeds moments later.
+# - "connection closed before message completed" is delta-rs's own object_store crate (reqwest/hyper)
+# reporting a connection the remote end dropped mid-response - it already prefixes this with its own
+# "Unexpected (temporary)" classification before this classifier even runs, so a retry against a fresh
+# connection is exactly what self-heals it, the same as any other blip in this list.
# A retry (of the same idempotent operation) clears these, so they shouldn't be treated the same as a
# bug in our logic.
TRANSIENT_OBJECT_STORE_ERRORS = (
@@ -31,6 +35,7 @@ TRANSIENT_OBJECT_STORE_ERRORS = (
"Please reduce your request rate",
"We encountered an internal error. Please try again.",
"The difference between the request time and the current time is too large.",
+ "connection closed before message completed",
)
# pyarrow's S3FileSystem (the AWS SDK for C++) reports a response it can't classify as the fixed
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_delta_errors.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_delta_errors.py
index aaa18bc830f4..bb96f898deaa 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_delta_errors.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_delta_errors.py
@@ -45,6 +45,22 @@ class TestIsTransientObjectStoreError:
OSError("[Errno 121] We encountered an internal error. Please try again."),
True,
),
+ (
+ # delta-rs's own object_store crate (reqwest/hyper) reports a connection the remote
+ # end dropped mid-response this way, already marked "(temporary)" before this
+ # classifier runs - a retry against a fresh connection self-heals it.
+ "connection_closed_before_message_completed",
+ OSError(
+ "Generic IoError error: Unexpected (temporary) at read, context: { url: "
+ "https://example-bucket.oss-us-west-1.aliyuncs.com/_delta_log/00000000000000000002.json, "
+ "called: http_util::Client::send, service: oss, path: "
+ "_delta_log/00000000000000000002.json, range: 0-3460 } => send http request, source: "
+ "error sending request for url (https://example-bucket.oss-us-west-1.aliyuncs.com/"
+ "_delta_log/00000000000000000002.json) => client error (SendRequest) => connection "
+ "closed before message completed"
+ ),
+ True,
+ ),
(
# s3fs/aiobotocore's own credential resolution (distinct from delta-rs's Rust
# object_store crate) can raise this bare, unwrapped — same IMDS/STS blip