Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(postgres): stop flooding error tracking on fdw foreign-server outages (#125279)

Tom Owers684260ad7049
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py
index b9cd818cf4f2..bc5616ffe8d3 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py
@@ -411,6 +411,14 @@ _FOREIGN_SERVER_UNREACHABLE_ERROR = (
"persists, check that the foreign server is running and reachable from your source database."
)
+# Terminal twin of `_FOREIGN_SERVER_UNREACHABLE_ERROR` once the activity's own retry budget is spent
+# on a sustained outage. The schedule picks the sync back up on its own, so it must not read like
+# the sync needs re-enabling — mirrors `_SSH_FORWARD_FAILED_EXHAUSTED_MESSAGE` below.
+_FOREIGN_SERVER_UNREACHABLE_EXHAUSTED_MESSAGE = (
+ f"{_FOREIGN_SERVER_UNREACHABLE_ERROR} This kept happening on every retry. This sync is still "
+ "enabled and will run again on its next schedule."
+)
+
# sshtunnel raises BaseSSHTunnelForwarderError("Could not establish session to SSH gateway") when it
# can't open a session to the bastion — the SSH host/port is wrong or unreachable, the bastion is
# down, or its firewall blocks PostHog's IPs. The raw message tells the user nothing actionable, so
@@ -1779,6 +1787,12 @@ class PostgresSource(
# never appears in CockroachDB's message. CockroachDB expects the whole transaction to be
# retried on conflict, which is exactly what letting this propagate to Temporal's
# whole-activity retry does, so it's the same self-recovering class as the other entries here.
+ #
+ # `_FOREIGN_SERVER_UNREACHABLE_ERROR` (see `ForeignServerUnreachableError`) is the same
+ # self-recovering shape again, for a postgres_fdw foreign table's own downstream server
+ # rather than the direct connection. Without it here, every occurrence of an otherwise
+ # transient foreign-server blip would still be logged at `exception` and flood error
+ # tracking, the exact failure mode this set exists to avoid.
return {
CLIENT_DEADLINE_ERROR,
"canceling statement due to user request",
@@ -1794,6 +1808,7 @@ class PostgresSource(
_SSH_FORWARD_FAILED_ERROR,
_CONNECT_TIMEOUT_ERROR,
_POOLER_AUTH_CIRCUIT_BREAKER_ERROR,
+ _FOREIGN_SERVER_UNREACHABLE_ERROR,
}
def get_retry_exhausted_errors(self) -> dict[str, str]:
@@ -1821,6 +1836,7 @@ class PostgresSource(
_SSH_FORWARD_FAILED_ERROR: _SSH_FORWARD_FAILED_EXHAUSTED_MESSAGE,
_CONNECT_TIMEOUT_ERROR: _CONNECT_TIMEOUT_EXHAUSTED_MESSAGE,
_POOLER_AUTH_CIRCUIT_BREAKER_ERROR: _POOLER_AUTH_CIRCUIT_BREAKER_EXHAUSTED_MESSAGE,
+ _FOREIGN_SERVER_UNREACHABLE_ERROR: _FOREIGN_SERVER_UNREACHABLE_EXHAUSTED_MESSAGE,
}
def reconcile_schema_metadata(
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py
index d8d95b225823..e7b0f38031db 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py
@@ -408,6 +408,21 @@ class TestPostgresSourceForeignServerConnectionError:
assert not is_non_retryable, f"A foreign-server connection failure must stay retryable: {error_msg}"
assert "Connection refused" not in error_msg
+ # Also classified in get_retryable_errors, so _handle_import_error logs this self-recovering
+ # failure as a warning instead of flooding error tracking with an exception on every retry.
+ assert error_message_matches(error_msg, source.get_retryable_errors())
+
+ message = next(
+ (
+ friendly
+ for pattern, friendly in source.get_retry_exhausted_errors().items()
+ if error_message_matches(error_msg, [pattern])
+ ),
+ None,
+ )
+ assert message is not None, "an unreachable foreign server must surface an actionable message once exhausted"
+ assert "still enabled" in message.lower()
+
class TestPostgresSourceNonRetryableErrors:
@pytest.fixture