Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(redshift): classify mid-stream connection drops as retryable (#121145)

Tom Owers203e84ffe465
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/redshift/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/redshift/source.py
index 7e80941dc83b..e7ca085f84d7 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/redshift/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/redshift/source.py
@@ -33,6 +33,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.generated_
RedshiftSourceConfig,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.redshift.redshift import (
+ _TRANSIENT_CONNECTION_DROP_SUBSTRINGS,
REDSHIFT_READ_TIMEOUT_ERROR,
RedshiftImplementation,
get_connection_metadata as get_connection_metadata_redshift,
@@ -71,6 +72,19 @@ _HOST_UNREACHABLE_VALIDATION_ERROR = (
# blocked network path, so a later attempt can succeed.
_CONNECT_TIMEOUT_ERROR = "connection timeout expired"
+# `_TRANSIENT_CONNECTION_DROP_SUBSTRINGS` (redshift.py) already retries a connection drop in-process
+# during sync setup, but once `_stream_arrow_batches` has yielded a batch it can no longer safely
+# replay the query, so the same drop mid-stream propagates all the way up to this activity. Temporal
+# then retries the whole activity and the failure is transient and self-recovering, so classify it
+# here too — otherwise `_handle_import_error` logs it at `exception` on every occurrence, flooding
+# error tracking with a self-recovering network blip. Mirrors the equivalent Postgres source.
+_CONNECTION_DROPPED_EXHAUSTED_MESSAGE = (
+ "PostHog's connection to your Redshift cluster kept closing before the sync could finish, and "
+ "reconnecting didn't help. Check that the cluster is not paused, resizing, or dropping idle "
+ "connections (a firewall or load balancer in front of it can do this too). This sync is still "
+ "enabled and will run again on its next schedule."
+)
+
RedshiftErrors = {
"password authentication failed for user": "Invalid user or password",
"could not translate host name": _DNS_RESOLUTION_VALIDATION_ERROR,
@@ -196,6 +210,7 @@ class RedshiftSource(SQLSource[RedshiftSourceConfig], SSHTunnelMixin, ValidateDa
TEMPORARY_HOST_RESOLUTION_ERROR,
REDSHIFT_READ_TIMEOUT_ERROR,
_CONNECT_TIMEOUT_ERROR,
+ *_TRANSIENT_CONNECTION_DROP_SUBSTRINGS,
}
def get_retry_exhausted_errors(self) -> dict[str, str]:
@@ -214,6 +229,7 @@ class RedshiftSource(SQLSource[RedshiftSourceConfig], SSHTunnelMixin, ValidateDa
"SSH tunnel, check that the bastion host can reach the cluster. This sync is still enabled "
"and will run again on its next schedule."
),
+ **dict.fromkeys(_TRANSIENT_CONNECTION_DROP_SUBSTRINGS, _CONNECTION_DROPPED_EXHAUSTED_MESSAGE),
}
def get_non_retryable_errors(self) -> dict[str, str | None]:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/redshift/tests/test_redshift.py b/products/warehouse_sources/backend/temporal/data_imports/sources/redshift/tests/test_redshift.py
index c130704f593a..377d12a940f5 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/redshift/tests/test_redshift.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/redshift/tests/test_redshift.py
@@ -1019,6 +1019,25 @@ class TestRedshiftSourceNonRetryableErrors:
assert not error_message_matches(error_msg, {**Any_Source_Errors, **source.get_non_retryable_errors()})
assert source.get_retryable_errors() == set(source.get_retry_exhausted_errors().keys())
+ @pytest.mark.parametrize(
+ "error_msg",
+ [
+ # Reported by error tracking: a mid-stream socket drop raised once `cursor.stream()`
+ # has already yielded rows, so `_stream_arrow_batches` can no longer retry in-process
+ # (re-running the query would re-emit consumed rows) and the raw psycopg error
+ # propagates all the way up to the activity.
+ "consuming input failed: SSL SYSCALL error: EOF detected",
+ "server closed the connection unexpectedly",
+ "the connection is lost",
+ ],
+ )
+ def test_mid_stream_connection_drops_are_classified_retryable(self, error_msg):
+ # The retryable/exhausted key-set invariant is covered once, for any error message, by
+ # `test_connect_time_failures_are_classified_retryable` above.
+ source = RedshiftSource()
+ assert error_message_matches(error_msg, source.get_retryable_errors())
+ assert not error_message_matches(error_msg, {**Any_Source_Errors, **source.get_non_retryable_errors()})
+
class TestRedshiftValidateCredentials:
def test_server_without_ssl_returns_friendly_error_without_capturing(self, mocker):