:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(warehouse-sources): retry redshift connect timeouts (#119262)
posthog[bot]c1fbc4a8916a
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/redshift/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/redshift/source.pyindex 777653bba90a..7e80941dc83b 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/redshift/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/redshift/source.py@@ -66,6 +66,11 @@ _HOST_UNREACHABLE_VALIDATION_ERROR = ("correct, and that PostHog's IP addresses are allowed through your firewall.")+# psycopg raises `ConnectionTimeout` with this text when `connect_timeout` runs out before the+# cluster answers. A paused, resizing or briefly overloaded cluster gives the same result as a+# blocked network path, so a later attempt can succeed.+_CONNECT_TIMEOUT_ERROR = "connection timeout expired"+RedshiftErrors = {"password authentication failed for user": "Invalid user or password","could not translate host name": _DNS_RESOLUTION_VALIDATION_ERROR,@@ -186,7 +191,12 @@ class RedshiftSource(SQLSource[RedshiftSourceConfig], SSHTunnelMixin, ValidateDa## `REDSHIFT_READ_TIMEOUT_ERROR` is the client-side limit on the wait for one batch of rows.# The cluster was reachable and then sent nothing, so a later attempt can succeed.- return {HOST_RESOLUTION_TIMEOUT_ERROR, TEMPORARY_HOST_RESOLUTION_ERROR, REDSHIFT_READ_TIMEOUT_ERROR}+ return {+ HOST_RESOLUTION_TIMEOUT_ERROR,+ TEMPORARY_HOST_RESOLUTION_ERROR,+ REDSHIFT_READ_TIMEOUT_ERROR,+ _CONNECT_TIMEOUT_ERROR,+ }def get_retry_exhausted_errors(self) -> dict[str, str]:return {@@ -197,6 +207,13 @@ class RedshiftSource(SQLSource[RedshiftSourceConfig], SSHTunnelMixin, ValidateDa"on every retry. Check that the cluster is not paused or resizing and that it has capacity ""for the read. This sync is still enabled and will run again on its next schedule."),+ _CONNECT_TIMEOUT_ERROR: (+ "PostHog couldn't connect to your Redshift cluster before the connection timed out, and it "+ "did so again on every retry. Check that the cluster is running and not paused, and that "+ "PostHog's IP addresses are allowed through its security group. If you connect through an "+ "SSH tunnel, check that the bastion host can reach the cluster. This sync is still enabled "+ "and will run again on its next schedule."+ ),}def get_non_retryable_errors(self) -> dict[str, str | None]:@@ -244,7 +261,6 @@ class RedshiftSource(SQLSource[RedshiftSourceConfig], SSHTunnelMixin, ValidateDa),"No route to host": None,"password authentication failed connection": None,- "connection timeout expired": None,"Connection refused": None,}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/redshift/tests/test_redshift.py b/products/warehouse_sources/backend/temporal/data_imports/sources/redshift/tests/test_redshift.pyindex 9e34132272c3..c130704f593a 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/redshift/tests/test_redshift.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/redshift/tests/test_redshift.py@@ -1007,12 +1007,16 @@ class TestRedshiftSourceNonRetryableErrors:@pytest.mark.parametrize("error_msg",- [f"{HOST_RESOLUTION_TIMEOUT_ERROR} after 15.0s", TEMPORARY_HOST_RESOLUTION_ERROR],+ [+ f"{HOST_RESOLUTION_TIMEOUT_ERROR} after 15.0s",+ TEMPORARY_HOST_RESOLUTION_ERROR,+ "connection timeout expired",+ ],)- def test_resolver_failures_before_the_connect_are_classified_retryable(self, error_msg):+ def test_connect_time_failures_are_classified_retryable(self, error_msg):source = RedshiftSource()assert any(pattern in error_msg for pattern in source.get_retryable_errors())- assert not any(pattern in error_msg for pattern in source.get_non_retryable_errors())+ assert not error_message_matches(error_msg, {**Any_Source_Errors, **source.get_non_retryable_errors()})assert source.get_retryable_errors() == set(source.get_retry_exhausted_errors().keys())diff --git a/products/warehouse_sources/backend/temporal/data_imports/tests/test_external_data_job_activities.py b/products/warehouse_sources/backend/temporal/data_imports/tests/test_external_data_job_activities.pyindex 5846c9dcb3cf..731caf11a3af 100644--- a/products/warehouse_sources/backend/temporal/data_imports/tests/test_external_data_job_activities.py+++ b/products/warehouse_sources/backend/temporal/data_imports/tests/test_external_data_job_activities.py@@ -38,6 +38,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.matomo.soufrom products.warehouse_sources.backend.temporal.data_imports.sources.postgres.source import (_CONNECTION_LIMIT_EXHAUSTED_MESSAGE,)+from products.warehouse_sources.backend.temporal.data_imports.sources.redshift.source import RedshiftSourcefrom products.warehouse_sources.backend.types import ExternalDataSourceType@@ -451,6 +452,13 @@ def test_read_only_transaction_disables_the_schema_only_when_the_source_raised_i"Matomo API error (retryable): status=503, url=https://matomo.example.com/index.php",MatomoSource().get_retry_exhausted_errors()["Matomo API error (retryable)"],),+ # A Redshift cluster that never answered the connect keeps its schedule.+ (+ "redshift_connect_timeout",+ ExternalDataSourceType.REDSHIFT,+ "connection timeout expired",+ RedshiftSource().get_retry_exhausted_errors()["connection timeout expired"],+ ),])@pytest.mark.django_db(transaction=True)