Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-imports): stop reporting a spurious postgres cancel as an exception (#117593)

Tom Owers43507c993d74
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py
index 3523abb93793..c9cd9eefb858 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py
@@ -453,6 +453,14 @@ _RECOVERY_CONFLICT_EXHAUSTED_MESSAGE = (
"schedule."
)
+_SPURIOUS_CANCEL_EXHAUSTED_MESSAGE = (
+ "Your database canceled one of PostHog's statements, and the sync kept hitting the same "
+ "cancellation on every retry. This can happen when a connection pooler recycles connections "
+ "mid-query, or someone or something on your database canceled it. Check your database and "
+ "pooler logs around the time of the failure. This sync is still enabled and will run again on "
+ "its next schedule."
+)
+
@SourceRegistry.register
class PostgresSource(
@@ -1561,8 +1569,19 @@ class PostgresSource(
# `CLIENT_DEADLINE_ERROR` is the client-side statement deadline. It acts only when the
# server's own statement timeout did not, so it reports a pooler or a server that stopped
# answering, not a query that is too slow for its index.
+ #
+ # "canceling statement due to user request" (psycopg's `QueryCanceled`) reaching here is
+ # *not* our own client-side deadline — that path always converts an expired deadline's
+ # cancel into `ClientDeadlineExceededError` first (see `client_side_deadline`). This is the
+ # raw message, so either our own watchdog's cancel request landed on a later statement on
+ # the same connection after its own statement already finished (the race
+ # `_StatementWatchdog.finish` documents), or something external — a pooler recycling the
+ # connection mid-query, or an admin cancel — canceled it. Neither says anything about this
+ # query or schema, and a retry runs on a fresh connection, so it's self-recovering like the
+ # other entries here rather than a verdict to stop retrying.
return {
CLIENT_DEADLINE_ERROR,
+ "canceling statement due to user request",
*_CONNECTION_DROPPED_ERROR_SUBSTRINGS,
*_POOLER_CONNECTION_DROPPED_ERROR_SUBSTRINGS,
*_SERVER_STARTING_UP_ERROR_SUBSTRINGS,
@@ -1589,6 +1608,7 @@ class PostgresSource(
**dict.fromkeys(_CONNECTION_LIMIT_ERROR_SUBSTRINGS, _CONNECTION_LIMIT_EXHAUSTED_MESSAGE),
"conflict with recovery": _RECOVERY_CONFLICT_EXHAUSTED_MESSAGE,
CLIENT_DEADLINE_ERROR: _CLIENT_DEADLINE_EXHAUSTED_MESSAGE,
+ "canceling statement due to user request": _SPURIOUS_CANCEL_EXHAUSTED_MESSAGE,
HOST_RESOLUTION_TIMEOUT_ERROR: HOST_RESOLUTION_EXHAUSTED_MESSAGE,
TEMPORARY_HOST_RESOLUTION_ERROR: HOST_RESOLUTION_EXHAUSTED_MESSAGE,
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py
index a66551aaa8b5..2b08cf6c39ee 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py
@@ -701,6 +701,12 @@ class TestPostgresSourceNonRetryableErrors:
# in-process fallback already retries elsewhere, unlike the "kept canceling reads..."/"no
# key that can resume..." exhausted-retry aborts below, which stay non-retryable.
"canceling statement due to conflict with recovery\nDETAIL: User query might have needed to see row versions that must be removed.",
+ # Not our own client-side deadline (that path always converts to
+ # ClientDeadlineExceededError first) — either our watchdog's cancel request landed on a
+ # later statement after its own one already finished, or an external cancel (a pooler
+ # recycling the connection, an admin cancel) hit the query. Neither is a verdict on this
+ # query, so a fresh connection on retry recovers.
+ "canceling statement due to user request",
],
)
def test_exhausted_connection_drops_are_classified_retryable(self, source, error_msg):
@@ -726,6 +732,10 @@ class TestPostgresSourceNonRetryableErrors:
'OperationalError: connection failed: connection to server at "db.example.com" (10.0.0.1), port 5432 failed: FATAL: sorry, too many clients already',
"connection slots",
),
+ (
+ "canceling statement due to user request",
+ "canceled",
+ ),
],
)
def test_exhausted_retries_replace_raw_driver_text(self, source, error_msg, expected_phrase):