:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(postgres): stop flooding error tracking on fdw foreign-server outages (#125279)
Tom Owers684260ad7049
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.pyindex b9cd818cf4f2..bc5616ffe8d3 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py@@ -411,6 +411,14 @@ _FOREIGN_SERVER_UNREACHABLE_ERROR = ("persists, check that the foreign server is running and reachable from your source database.")+# Terminal twin of `_FOREIGN_SERVER_UNREACHABLE_ERROR` once the activity's own retry budget is spent+# on a sustained outage. The schedule picks the sync back up on its own, so it must not read like+# the sync needs re-enabling — mirrors `_SSH_FORWARD_FAILED_EXHAUSTED_MESSAGE` below.+_FOREIGN_SERVER_UNREACHABLE_EXHAUSTED_MESSAGE = (+ f"{_FOREIGN_SERVER_UNREACHABLE_ERROR} This kept happening on every retry. This sync is still "+ "enabled and will run again on its next schedule."+)+# sshtunnel raises BaseSSHTunnelForwarderError("Could not establish session to SSH gateway") when it# can't open a session to the bastion — the SSH host/port is wrong or unreachable, the bastion is# down, or its firewall blocks PostHog's IPs. The raw message tells the user nothing actionable, so@@ -1779,6 +1787,12 @@ class PostgresSource(# never appears in CockroachDB's message. CockroachDB expects the whole transaction to be# retried on conflict, which is exactly what letting this propagate to Temporal's# whole-activity retry does, so it's the same self-recovering class as the other entries here.+ #+ # `_FOREIGN_SERVER_UNREACHABLE_ERROR` (see `ForeignServerUnreachableError`) is the same+ # self-recovering shape again, for a postgres_fdw foreign table's own downstream server+ # rather than the direct connection. Without it here, every occurrence of an otherwise+ # transient foreign-server blip would still be logged at `exception` and flood error+ # tracking, the exact failure mode this set exists to avoid.return {CLIENT_DEADLINE_ERROR,"canceling statement due to user request",@@ -1794,6 +1808,7 @@ class PostgresSource(_SSH_FORWARD_FAILED_ERROR,_CONNECT_TIMEOUT_ERROR,_POOLER_AUTH_CIRCUIT_BREAKER_ERROR,+ _FOREIGN_SERVER_UNREACHABLE_ERROR,}def get_retry_exhausted_errors(self) -> dict[str, str]:@@ -1821,6 +1836,7 @@ class PostgresSource(_SSH_FORWARD_FAILED_ERROR: _SSH_FORWARD_FAILED_EXHAUSTED_MESSAGE,_CONNECT_TIMEOUT_ERROR: _CONNECT_TIMEOUT_EXHAUSTED_MESSAGE,_POOLER_AUTH_CIRCUIT_BREAKER_ERROR: _POOLER_AUTH_CIRCUIT_BREAKER_EXHAUSTED_MESSAGE,+ _FOREIGN_SERVER_UNREACHABLE_ERROR: _FOREIGN_SERVER_UNREACHABLE_EXHAUSTED_MESSAGE,}def reconcile_schema_metadata(diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.pyindex d8d95b225823..e7b0f38031db 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py@@ -408,6 +408,21 @@ class TestPostgresSourceForeignServerConnectionError:assert not is_non_retryable, f"A foreign-server connection failure must stay retryable: {error_msg}"assert "Connection refused" not in error_msg+ # Also classified in get_retryable_errors, so _handle_import_error logs this self-recovering+ # failure as a warning instead of flooding error tracking with an exception on every retry.+ assert error_message_matches(error_msg, source.get_retryable_errors())++ message = next(+ (+ friendly+ for pattern, friendly in source.get_retry_exhausted_errors().items()+ if error_message_matches(error_msg, [pattern])+ ),+ None,+ )+ assert message is not None, "an unreachable foreign server must surface an actionable message once exhausted"+ assert "still enabled" in message.lower()+class TestPostgresSourceNonRetryableErrors:@pytest.fixture