Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(postgres): treat proxy "service temporarily unavailable" refusals as retryable (#125095)

Tom Owers96fec670ada7
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/postgres.py b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/postgres.py
index 9bb0a4ebf780..a3876ef32f4b 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/postgres.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/postgres.py
@@ -596,32 +596,50 @@ def _is_server_starting_up_error(error: BaseException) -> bool:
return any(substring in message for substring in _SERVER_STARTING_UP_ERROR_SUBSTRINGS)
+# A connect-time refusal some proxy/pooler layers in front of Postgres raise during their own brief
+# internal hiccup: a bare "FATAL: service temporarily unavailable" with no SQLSTATE-mapped
+# subclass and none of the lifecycle wording above (it names no startup/shutdown/recovery state, and
+# no capacity limit like `_CONNECTION_LIMIT_ERROR_SUBSTRINGS`). Deterministic for the moment it's
+# raised but self-recovering on a fresh attempt once the outage clears, so it stays retryable rather
+# than disabling the sync.
+_SERVICE_UNAVAILABLE_ERROR_SUBSTRING = "service temporarily unavailable"
+
+
+def _is_service_unavailable_error(error: BaseException) -> bool:
+ if not isinstance(error, psycopg.OperationalError):
+ return False
+ return _SERVICE_UNAVAILABLE_ERROR_SUBSTRING in " ".join(str(arg) for arg in error.args).lower()
+
+
def _is_dropped_or_connect_timeout(error: BaseException) -> bool:
"""Transient connect-path failures the import/read reconnect recovers from in process.
A mid-stream drop (`_is_connection_dropped_error`), a connect-time timeout, a connect-time
connection-limit refusal (`_is_connection_limit_error`), a connect-time "server not ready"
- refusal while the source is still starting up / recovering (`_is_server_starting_up_error`), or
- this worker process running out of file descriptors while opening the socket
+ refusal while the source is still starting up / recovering (`_is_server_starting_up_error`), a
+ proxy/pooler reporting a brief service outage (`_is_service_unavailable_error`), or this worker
+ process running out of file descriptors while opening the socket
(`_is_too_many_open_files_error`). psycopg raises `ConnectionTimeout` ("connection timeout
expired") only while *establishing* a connection, never mid-query, so on the import/read path
it's transient: the source was reachable moments earlier in the same sync, and the reconnect
- just needs retrying. Connection-limit refusals ("sorry, too many clients already", etc.) and
- server-still-starting-up refusals ("the database system is not yet accepting connections", etc.)
- are likewise transient — a slot frees the moment another connection closes, and the server
- begins accepting connections once startup/recovery finishes. A file-descriptor exhaustion is the
- same shape on our side of the wire: a descriptor frees the moment another connection/cursor in
- this worker closes. Used by the read/sync connect retry (`_connect_with_dropped_retry`) and the
- `offset_chunking` reconnect. The schema-discovery path retries drops, connection-limit refusals,
- server-startup refusals, and recovery conflicts too (via `_is_dropped_or_connection_limit`) but
- deliberately keeps failing fast on connect-time *timeouts*, where a timeout usually means an
- unreachable host / unconfigured firewall (see `PostgresErrors`). A fast failure there is still
- retried by the activity, because `get_retryable_errors` lists the timeout.
+ just needs retrying. Connection-limit refusals ("sorry, too many clients already", etc.),
+ server-still-starting-up refusals ("the database system is not yet accepting connections", etc.),
+ and bare service-unavailable refusals are likewise transient — a slot frees the moment another
+ connection closes, the server begins accepting connections once startup/recovery finishes, and a
+ proxy/pooler outage clears on its own. A file-descriptor exhaustion is the same shape on our side
+ of the wire: a descriptor frees the moment another connection/cursor in this worker closes. Used
+ by the read/sync connect retry (`_connect_with_dropped_retry`) and the `offset_chunking`
+ reconnect. The schema-discovery path retries drops, connection-limit refusals, server-startup
+ refusals, and recovery conflicts too (via `_is_dropped_or_connection_limit`) but deliberately
+ keeps failing fast on connect-time *timeouts*, where a timeout usually means an unreachable host
+ / unconfigured firewall (see `PostgresErrors`). A fast failure there is still retried by the
+ activity, because `get_retryable_errors` lists the timeout.
"""
return (
_is_connection_dropped_error(error)
or _is_connection_limit_error(error)
or _is_server_starting_up_error(error)
+ or _is_service_unavailable_error(error)
or _is_too_many_open_files_error(error)
or isinstance(error, psycopg.errors.ConnectionTimeout)
)
@@ -632,22 +650,24 @@ def _is_dropped_or_connection_limit(error: BaseException) -> bool:
A mid-stream drop (`_is_connection_dropped_error`), a connection-limit refusal
(`_is_connection_limit_error`), a "server not ready" refusal while the source is still starting
- up / recovering (`_is_server_starting_up_error`), or a hot-standby recovery conflict
+ up / recovering (`_is_server_starting_up_error`), a proxy/pooler reporting a brief service outage
+ (`_is_service_unavailable_error`), or a hot-standby recovery conflict
(`_is_recovery_conflict_error`). All are transient — a slot frees as connections close, a
pooler-cached login failure clears once the upstream has capacity, the server begins accepting
- connections once startup/recovery finishes, and a recovery conflict clears once the replica's WAL
- replay moves past the conflicting row versions — so discovery retries them on a fresh connection
- instead of failing the activity and surfacing captured error-tracking noise. The import read path
- already retries the same recovery-conflict condition mid-stream (see `handle_recovery_conflict`);
- discovery just needed the same treatment. Unlike the read/sync connect path
- (`_is_dropped_or_connect_timeout`), a connect-time *timeout* is deliberately excluded: during
- discovery a timeout usually means a now-unreachable host, which should fail fast rather than burn
- the retry budget.
+ connections once startup/recovery finishes, a proxy/pooler outage clears on its own, and a
+ recovery conflict clears once the replica's WAL replay moves past the conflicting row versions —
+ so discovery retries them on a fresh connection instead of failing the activity and surfacing
+ captured error-tracking noise. The import read path already retries the same recovery-conflict
+ condition mid-stream (see `handle_recovery_conflict`); discovery just needed the same treatment.
+ Unlike the read/sync connect path (`_is_dropped_or_connect_timeout`), a connect-time *timeout* is
+ deliberately excluded: during discovery a timeout usually means a now-unreachable host, which
+ should fail fast rather than burn the retry budget.
"""
return (
_is_connection_dropped_error(error)
or _is_connection_limit_error(error)
or _is_server_starting_up_error(error)
+ or _is_service_unavailable_error(error)
or _is_recovery_conflict_error(error)
)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py
index 50af15a7b8a9..f6b6eda9edcd 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py
@@ -71,6 +71,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.postgres.p
_CONNECTION_LIMIT_ERROR_SUBSTRINGS,
_POOLER_CONNECTION_DROPPED_ERROR_SUBSTRINGS,
_SERVER_STARTING_UP_ERROR_SUBSTRINGS,
+ _SERVICE_UNAVAILABLE_ERROR_SUBSTRING,
_SSH_HANDSHAKE_EOF_ERROR,
XMIN_AS_INCREMENTAL_FIELD_ERROR,
PostgresImplementation,
@@ -482,6 +483,16 @@ _SERVER_UNAVAILABLE_EXHAUSTED_MESSAGE = (
"running and healthy. This sync is still enabled and will run again on its next schedule."
)
+# Twin of `_SERVER_UNAVAILABLE_EXHAUSTED_MESSAGE` for a bare proxy/pooler "service temporarily
+# unavailable" refusal (see `_SERVICE_UNAVAILABLE_ERROR_SUBSTRING` in postgres.py), which names no
+# lifecycle state, so the message doesn't claim one either.
+_SERVICE_UNAVAILABLE_EXHAUSTED_MESSAGE = (
+ "Your database refused PostHog's connection, reporting that the service was temporarily "
+ "unavailable, and it was still refusing connections after every retry. Check your database and "
+ "any connection pooler or proxy in front of it for an ongoing outage, maintenance, or failover. "
+ "This sync is still enabled and will run again on its next schedule."
+)
+
_CONNECTION_LIMIT_EXHAUSTED_MESSAGE = (
"Your database had no free connection slots for PostHog, and none freed up before the retries "
"ran out. Raise the connection limit on the database or its pooler, or reduce how many other "
@@ -1702,6 +1713,11 @@ class PostgresSource(
# new connections while the source is going down — all retried in-process by the
# offset-chunking reconnect, same exhausted-budget fallback as above.
#
+ # `_SERVICE_UNAVAILABLE_ERROR_SUBSTRING` is a bare proxy/pooler "service temporarily
+ # unavailable" refusal at connect time — no lifecycle state, no capacity limit, just a brief
+ # outage on the layer in front of Postgres. Same self-recovering shape, same in-process retry
+ # via `_is_dropped_or_connect_timeout` / `_is_dropped_or_connection_limit`.
+ #
# `_CONNECTION_LIMIT_ERROR_SUBSTRINGS` (e.g. "remaining connection slots are reserved") is a
# connect-time capacity refusal already retried in-process by `_connect_with_dropped_retry` /
# `_is_dropped_or_connection_limit`. A slot frees the moment another connection closes, so a
@@ -1760,6 +1776,7 @@ class PostgresSource(
*_CONNECTION_DROPPED_ERROR_SUBSTRINGS,
*_POOLER_CONNECTION_DROPPED_ERROR_SUBSTRINGS,
*_SERVER_STARTING_UP_ERROR_SUBSTRINGS,
+ _SERVICE_UNAVAILABLE_ERROR_SUBSTRING,
*_CONNECTION_LIMIT_ERROR_SUBSTRINGS,
"conflict with recovery",
"TransactionRetryWithProtoRefreshError",
@@ -1784,6 +1801,7 @@ class PostgresSource(
**dict.fromkeys(_CONNECTION_DROPPED_ERROR_SUBSTRINGS, _CONNECTION_DROPPED_EXHAUSTED_MESSAGE),
**dict.fromkeys(_POOLER_CONNECTION_DROPPED_ERROR_SUBSTRINGS, _CONNECTION_DROPPED_EXHAUSTED_MESSAGE),
**dict.fromkeys(_SERVER_STARTING_UP_ERROR_SUBSTRINGS, _SERVER_UNAVAILABLE_EXHAUSTED_MESSAGE),
+ _SERVICE_UNAVAILABLE_ERROR_SUBSTRING: _SERVICE_UNAVAILABLE_EXHAUSTED_MESSAGE,
**dict.fromkeys(_CONNECTION_LIMIT_ERROR_SUBSTRINGS, _CONNECTION_LIMIT_EXHAUSTED_MESSAGE),
"conflict with recovery": _RECOVERY_CONFLICT_EXHAUSTED_MESSAGE,
"TransactionRetryWithProtoRefreshError": _SERIALIZATION_CONFLICT_EXHAUSTED_MESSAGE,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py
index 71bc474966dd..ab8f8fdaff15 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py
@@ -737,6 +737,11 @@ class TestPostgresSourceNonRetryableErrors:
"restart transaction: TransactionRetryWithProtoRefreshError: ReadWithinUncertaintyIntervalError: "
"read at time 100.000000000,0 encountered previous write with future timestamp 100.000000000,1: "
'"sql txn" meta={id=00000000}',
+ # A proxy/pooler's own brief service outage at connect time, with no lifecycle or
+ # capacity wording. Self-recovering once the outage clears, so it must stay classified
+ # retryable like the other connect-time refusals above. IP is invented, not a real
+ # customer value.
+ 'OperationalError: connection failed: connection to server at "203.0.113.7", port 5432 failed: FATAL: service temporarily unavailable',
],
)
def test_exhausted_connection_drops_are_classified_retryable(self, source, error_msg):
@@ -780,6 +785,10 @@ class TestPostgresSourceNonRetryableErrors:
"read at time 100.000000000,0",
"serialization conflict",
),
+ (
+ 'OperationalError: connection failed: connection to server at "203.0.113.7", port 5432 failed: FATAL: service temporarily unavailable',
+ "temporarily unavailable",
+ ),
],
)
def test_exhausted_retries_replace_raw_driver_text(self, source, error_msg, expected_phrase):
@@ -796,7 +805,7 @@ class TestPostgresSourceNonRetryableErrors:
)
assert message is not None, f"Exhausted retryable error must surface a message: {error_msg}"
assert expected_phrase in message.lower()
- assert "db.example.com" not in message and "10.0.0.1" not in message
+ assert "db.example.com" not in message and "10.0.0.1" not in message and "203.0.113.7" not in message
def test_every_retryable_error_has_an_exhaustion_message(self, source):
# A transient substring added to postgres.py flows into get_retryable_errors automatically,
@@ -2843,6 +2852,13 @@ class TestDroppedOrConnectTimeout:
'connection failed: connection to server at "10.0.0.1", port 5432 failed: '
"FATAL: the database system is shutting down"
),
+ # A proxy/pooler in front of the source refusing a connect attempt during its own brief
+ # internal hiccup — no SQLSTATE-mapped subclass, no lifecycle wording. Transient — a
+ # fresh connect after a short backoff succeeds once the outage clears — so it must be
+ # recognised here too instead of burning the whole activity's retry budget.
+ psycopg.OperationalError(
+ 'connection failed: connection to server at "10.0.0.1", port 5432 failed: FATAL: service temporarily unavailable'
+ ),
# This worker's process-wide (EMFILE) or system-wide (ENFILE) file-descriptor table is
# full while opening the socket for a fresh connect. Transient on our side of the wire —
# a descriptor frees the moment another connection/cursor in this worker closes — so the