:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(data-imports): stop reporting a spurious postgres cancel as an exception (#117593)
Tom Owers43507c993d74
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.pyindex 3523abb93793..c9cd9eefb858 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py@@ -453,6 +453,14 @@ _RECOVERY_CONFLICT_EXHAUSTED_MESSAGE = ("schedule.")+_SPURIOUS_CANCEL_EXHAUSTED_MESSAGE = (+ "Your database canceled one of PostHog's statements, and the sync kept hitting the same "+ "cancellation on every retry. This can happen when a connection pooler recycles connections "+ "mid-query, or someone or something on your database canceled it. Check your database and "+ "pooler logs around the time of the failure. This sync is still enabled and will run again on "+ "its next schedule."+)+@SourceRegistry.registerclass PostgresSource(@@ -1561,8 +1569,19 @@ class PostgresSource(# `CLIENT_DEADLINE_ERROR` is the client-side statement deadline. It acts only when the# server's own statement timeout did not, so it reports a pooler or a server that stopped# answering, not a query that is too slow for its index.+ #+ # "canceling statement due to user request" (psycopg's `QueryCanceled`) reaching here is+ # *not* our own client-side deadline — that path always converts an expired deadline's+ # cancel into `ClientDeadlineExceededError` first (see `client_side_deadline`). This is the+ # raw message, so either our own watchdog's cancel request landed on a later statement on+ # the same connection after its own statement already finished (the race+ # `_StatementWatchdog.finish` documents), or something external — a pooler recycling the+ # connection mid-query, or an admin cancel — canceled it. Neither says anything about this+ # query or schema, and a retry runs on a fresh connection, so it's self-recovering like the+ # other entries here rather than a verdict to stop retrying.return {CLIENT_DEADLINE_ERROR,+ "canceling statement due to user request",*_CONNECTION_DROPPED_ERROR_SUBSTRINGS,*_POOLER_CONNECTION_DROPPED_ERROR_SUBSTRINGS,*_SERVER_STARTING_UP_ERROR_SUBSTRINGS,@@ -1589,6 +1608,7 @@ class PostgresSource(**dict.fromkeys(_CONNECTION_LIMIT_ERROR_SUBSTRINGS, _CONNECTION_LIMIT_EXHAUSTED_MESSAGE),"conflict with recovery": _RECOVERY_CONFLICT_EXHAUSTED_MESSAGE,CLIENT_DEADLINE_ERROR: _CLIENT_DEADLINE_EXHAUSTED_MESSAGE,+ "canceling statement due to user request": _SPURIOUS_CANCEL_EXHAUSTED_MESSAGE,HOST_RESOLUTION_TIMEOUT_ERROR: HOST_RESOLUTION_EXHAUSTED_MESSAGE,TEMPORARY_HOST_RESOLUTION_ERROR: HOST_RESOLUTION_EXHAUSTED_MESSAGE,}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.pyindex a66551aaa8b5..2b08cf6c39ee 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py@@ -701,6 +701,12 @@ class TestPostgresSourceNonRetryableErrors:# in-process fallback already retries elsewhere, unlike the "kept canceling reads..."/"no# key that can resume..." exhausted-retry aborts below, which stay non-retryable."canceling statement due to conflict with recovery\nDETAIL: User query might have needed to see row versions that must be removed.",+ # Not our own client-side deadline (that path always converts to+ # ClientDeadlineExceededError first) — either our watchdog's cancel request landed on a+ # later statement after its own one already finished, or an external cancel (a pooler+ # recycling the connection, an admin cancel) hit the query. Neither is a verdict on this+ # query, so a fresh connection on retry recovers.+ "canceling statement due to user request",],)def test_exhausted_connection_drops_are_classified_retryable(self, source, error_msg):@@ -726,6 +732,10 @@ class TestPostgresSourceNonRetryableErrors:'OperationalError: connection failed: connection to server at "db.example.com" (10.0.0.1), port 5432 failed: FATAL: sorry, too many clients already',"connection slots",),+ (+ "canceling statement due to user request",+ "canceled",+ ),],)def test_exhausted_retries_replace_raw_driver_text(self, source, error_msg, expected_phrase):