:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(redshift): classify mid-stream connection drops as retryable (#121145)
Tom Owers203e84ffe465
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/redshift/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/redshift/source.pyindex 7e80941dc83b..e7ca085f84d7 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/redshift/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/redshift/source.py@@ -33,6 +33,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.generated_RedshiftSourceConfig,)from products.warehouse_sources.backend.temporal.data_imports.sources.redshift.redshift import (+ _TRANSIENT_CONNECTION_DROP_SUBSTRINGS,REDSHIFT_READ_TIMEOUT_ERROR,RedshiftImplementation,get_connection_metadata as get_connection_metadata_redshift,@@ -71,6 +72,19 @@ _HOST_UNREACHABLE_VALIDATION_ERROR = (# blocked network path, so a later attempt can succeed._CONNECT_TIMEOUT_ERROR = "connection timeout expired"+# `_TRANSIENT_CONNECTION_DROP_SUBSTRINGS` (redshift.py) already retries a connection drop in-process+# during sync setup, but once `_stream_arrow_batches` has yielded a batch it can no longer safely+# replay the query, so the same drop mid-stream propagates all the way up to this activity. Temporal+# then retries the whole activity and the failure is transient and self-recovering, so classify it+# here too — otherwise `_handle_import_error` logs it at `exception` on every occurrence, flooding+# error tracking with a self-recovering network blip. Mirrors the equivalent Postgres source.+_CONNECTION_DROPPED_EXHAUSTED_MESSAGE = (+ "PostHog's connection to your Redshift cluster kept closing before the sync could finish, and "+ "reconnecting didn't help. Check that the cluster is not paused, resizing, or dropping idle "+ "connections (a firewall or load balancer in front of it can do this too). This sync is still "+ "enabled and will run again on its next schedule."+)+RedshiftErrors = {"password authentication failed for user": "Invalid user or password","could not translate host name": _DNS_RESOLUTION_VALIDATION_ERROR,@@ -196,6 +210,7 @@ class RedshiftSource(SQLSource[RedshiftSourceConfig], SSHTunnelMixin, ValidateDaTEMPORARY_HOST_RESOLUTION_ERROR,REDSHIFT_READ_TIMEOUT_ERROR,_CONNECT_TIMEOUT_ERROR,+ *_TRANSIENT_CONNECTION_DROP_SUBSTRINGS,}def get_retry_exhausted_errors(self) -> dict[str, str]:@@ -214,6 +229,7 @@ class RedshiftSource(SQLSource[RedshiftSourceConfig], SSHTunnelMixin, ValidateDa"SSH tunnel, check that the bastion host can reach the cluster. This sync is still enabled ""and will run again on its next schedule."),+ **dict.fromkeys(_TRANSIENT_CONNECTION_DROP_SUBSTRINGS, _CONNECTION_DROPPED_EXHAUSTED_MESSAGE),}def get_non_retryable_errors(self) -> dict[str, str | None]:diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/redshift/tests/test_redshift.py b/products/warehouse_sources/backend/temporal/data_imports/sources/redshift/tests/test_redshift.pyindex c130704f593a..377d12a940f5 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/redshift/tests/test_redshift.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/redshift/tests/test_redshift.py@@ -1019,6 +1019,25 @@ class TestRedshiftSourceNonRetryableErrors:assert not error_message_matches(error_msg, {**Any_Source_Errors, **source.get_non_retryable_errors()})assert source.get_retryable_errors() == set(source.get_retry_exhausted_errors().keys())+ @pytest.mark.parametrize(+ "error_msg",+ [+ # Reported by error tracking: a mid-stream socket drop raised once `cursor.stream()`+ # has already yielded rows, so `_stream_arrow_batches` can no longer retry in-process+ # (re-running the query would re-emit consumed rows) and the raw psycopg error+ # propagates all the way up to the activity.+ "consuming input failed: SSL SYSCALL error: EOF detected",+ "server closed the connection unexpectedly",+ "the connection is lost",+ ],+ )+ def test_mid_stream_connection_drops_are_classified_retryable(self, error_msg):+ # The retryable/exhausted key-set invariant is covered once, for any error message, by+ # `test_connect_time_failures_are_classified_retryable` above.+ source = RedshiftSource()+ assert error_message_matches(error_msg, source.get_retryable_errors())+ assert not error_message_matches(error_msg, {**Any_Source_Errors, **source.get_non_retryable_errors()})+class TestRedshiftValidateCredentials:def test_server_without_ssl_returns_friendly_error_without_capturing(self, mocker):