Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(warehouse-sources): retry redshift connect timeouts (#119262)

posthog[bot]c1fbc4a8916a
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/redshift/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/redshift/source.py
index 777653bba90a..7e80941dc83b 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/redshift/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/redshift/source.py
@@ -66,6 +66,11 @@ _HOST_UNREACHABLE_VALIDATION_ERROR = (
"correct, and that PostHog's IP addresses are allowed through your firewall."
)
+# psycopg raises `ConnectionTimeout` with this text when `connect_timeout` runs out before the
+# cluster answers. A paused, resizing or briefly overloaded cluster gives the same result as a
+# blocked network path, so a later attempt can succeed.
+_CONNECT_TIMEOUT_ERROR = "connection timeout expired"
+
RedshiftErrors = {
"password authentication failed for user": "Invalid user or password",
"could not translate host name": _DNS_RESOLUTION_VALIDATION_ERROR,
@@ -186,7 +191,12 @@ class RedshiftSource(SQLSource[RedshiftSourceConfig], SSHTunnelMixin, ValidateDa
#
# `REDSHIFT_READ_TIMEOUT_ERROR` is the client-side limit on the wait for one batch of rows.
# The cluster was reachable and then sent nothing, so a later attempt can succeed.
- return {HOST_RESOLUTION_TIMEOUT_ERROR, TEMPORARY_HOST_RESOLUTION_ERROR, REDSHIFT_READ_TIMEOUT_ERROR}
+ return {
+ HOST_RESOLUTION_TIMEOUT_ERROR,
+ TEMPORARY_HOST_RESOLUTION_ERROR,
+ REDSHIFT_READ_TIMEOUT_ERROR,
+ _CONNECT_TIMEOUT_ERROR,
+ }
def get_retry_exhausted_errors(self) -> dict[str, str]:
return {
@@ -197,6 +207,13 @@ class RedshiftSource(SQLSource[RedshiftSourceConfig], SSHTunnelMixin, ValidateDa
"on every retry. Check that the cluster is not paused or resizing and that it has capacity "
"for the read. This sync is still enabled and will run again on its next schedule."
),
+ _CONNECT_TIMEOUT_ERROR: (
+ "PostHog couldn't connect to your Redshift cluster before the connection timed out, and it "
+ "did so again on every retry. Check that the cluster is running and not paused, and that "
+ "PostHog's IP addresses are allowed through its security group. If you connect through an "
+ "SSH tunnel, check that the bastion host can reach the cluster. This sync is still enabled "
+ "and will run again on its next schedule."
+ ),
}
def get_non_retryable_errors(self) -> dict[str, str | None]:
@@ -244,7 +261,6 @@ class RedshiftSource(SQLSource[RedshiftSourceConfig], SSHTunnelMixin, ValidateDa
),
"No route to host": None,
"password authentication failed connection": None,
- "connection timeout expired": None,
"Connection refused": None,
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/redshift/tests/test_redshift.py b/products/warehouse_sources/backend/temporal/data_imports/sources/redshift/tests/test_redshift.py
index 9e34132272c3..c130704f593a 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/redshift/tests/test_redshift.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/redshift/tests/test_redshift.py
@@ -1007,12 +1007,16 @@ class TestRedshiftSourceNonRetryableErrors:
@pytest.mark.parametrize(
"error_msg",
- [f"{HOST_RESOLUTION_TIMEOUT_ERROR} after 15.0s", TEMPORARY_HOST_RESOLUTION_ERROR],
+ [
+ f"{HOST_RESOLUTION_TIMEOUT_ERROR} after 15.0s",
+ TEMPORARY_HOST_RESOLUTION_ERROR,
+ "connection timeout expired",
+ ],
)
- def test_resolver_failures_before_the_connect_are_classified_retryable(self, error_msg):
+ def test_connect_time_failures_are_classified_retryable(self, error_msg):
source = RedshiftSource()
assert any(pattern in error_msg for pattern in source.get_retryable_errors())
- assert not any(pattern in error_msg for pattern in source.get_non_retryable_errors())
+ assert not error_message_matches(error_msg, {**Any_Source_Errors, **source.get_non_retryable_errors()})
assert source.get_retryable_errors() == set(source.get_retry_exhausted_errors().keys())
diff --git a/products/warehouse_sources/backend/temporal/data_imports/tests/test_external_data_job_activities.py b/products/warehouse_sources/backend/temporal/data_imports/tests/test_external_data_job_activities.py
index 5846c9dcb3cf..731caf11a3af 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/tests/test_external_data_job_activities.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/tests/test_external_data_job_activities.py
@@ -38,6 +38,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.matomo.sou
from products.warehouse_sources.backend.temporal.data_imports.sources.postgres.source import (
_CONNECTION_LIMIT_EXHAUSTED_MESSAGE,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.redshift.source import RedshiftSource
from products.warehouse_sources.backend.types import ExternalDataSourceType
@@ -451,6 +452,13 @@ def test_read_only_transaction_disables_the_schema_only_when_the_source_raised_i
"Matomo API error (retryable): status=503, url=https://matomo.example.com/index.php",
MatomoSource().get_retry_exhausted_errors()["Matomo API error (retryable)"],
),
+ # A Redshift cluster that never answered the connect keeps its schedule.
+ (
+ "redshift_connect_timeout",
+ ExternalDataSourceType.REDSHIFT,
+ "connection timeout expired",
+ RedshiftSource().get_retry_exhausted_errors()["connection timeout expired"],
+ ),
]
)
@pytest.mark.django_db(transaction=True)