:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(postgres): stop flooding error tracking with CockroachDB serialization conflicts (#123635)
Tom Owers2ddba67e9b0f
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.pyindex 7e653b52a5de..b42ec63db6cc 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py@@ -515,6 +515,14 @@ _SPURIOUS_CANCEL_EXHAUSTED_MESSAGE = ("its next schedule.")+_SERIALIZATION_CONFLICT_EXHAUSTED_MESSAGE = (+ "Your database kept aborting PostHog's read with a transaction serialization conflict, and the "+ "conflict outlasted every retry. This is common on CockroachDB and similar databases under "+ "heavy write load. Check how busy the table is during the sync window, or point the connection "+ "at a replica with less write traffic. This sync is still enabled and will run again on its "+ "next schedule."+)+@SourceRegistry.registerclass PostgresSource(@@ -1723,6 +1731,15 @@ class PostgresSource(# `_SSH_FORWARD_FAILED_ERROR` reaches here once `_connected_ssh_tunnel`'s own ~14 second# retry gives up opening the port forward to the database behind the bastion. See its# definition above for why that stays retryable rather than disabling the schema.+ #+ # "TransactionRetryWithProtoRefreshError" is CockroachDB's serialization-conflict wrapper+ # (its wire protocol is Postgres-compatible, so customers configure it as a plain Postgres+ # source — see the TABLESAMPLE workarounds in postgres.py). It covers every conflict reason+ # CockroachDB nests under it (read-within-uncertainty-interval, write-too-old, ...), unlike+ # "conflict with recovery" above, which is specifically Postgres hot-standby wording and+ # never appears in CockroachDB's message. CockroachDB expects the whole transaction to be+ # retried on conflict, which is exactly what letting this propagate to Temporal's+ # whole-activity retry does, so it's the same self-recovering class as the other entries here.return {CLIENT_DEADLINE_ERROR,"canceling statement due to user request",@@ -1731,6 +1748,7 @@ class PostgresSource(*_SERVER_STARTING_UP_ERROR_SUBSTRINGS,*_CONNECTION_LIMIT_ERROR_SUBSTRINGS,"conflict with recovery",+ "TransactionRetryWithProtoRefreshError",HOST_RESOLUTION_TIMEOUT_ERROR,TEMPORARY_HOST_RESOLUTION_ERROR,_SSH_FORWARD_FAILED_ERROR,@@ -1752,6 +1770,7 @@ class PostgresSource(**dict.fromkeys(_SERVER_STARTING_UP_ERROR_SUBSTRINGS, _SERVER_UNAVAILABLE_EXHAUSTED_MESSAGE),**dict.fromkeys(_CONNECTION_LIMIT_ERROR_SUBSTRINGS, _CONNECTION_LIMIT_EXHAUSTED_MESSAGE),"conflict with recovery": _RECOVERY_CONFLICT_EXHAUSTED_MESSAGE,+ "TransactionRetryWithProtoRefreshError": _SERIALIZATION_CONFLICT_EXHAUSTED_MESSAGE,CLIENT_DEADLINE_ERROR: _CLIENT_DEADLINE_EXHAUSTED_MESSAGE,"canceling statement due to user request": _SPURIOUS_CANCEL_EXHAUSTED_MESSAGE,HOST_RESOLUTION_TIMEOUT_ERROR: HOST_RESOLUTION_EXHAUSTED_MESSAGE,diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.pyindex 6198efa783da..bea8d9f063d5 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py@@ -734,6 +734,14 @@ class TestPostgresSourceNonRetryableErrors:# recycling the connection, an admin cancel) hit the query. Neither is a verdict on this# query, so a fresh connection on retry recovers."canceling statement due to user request",+ # CockroachDB's serialization-conflict wrapper (its Postgres-compatible wire protocol+ # means customers configure it as a plain Postgres source). CockroachDB expects the+ # whole transaction to be retried on conflict, which the whole-activity Temporal retry+ # already does, so it's self-recovering like the other entries here. Timestamps and txn+ # ids are invented; CockroachDB embeds both in every message of this class.+ "restart transaction: TransactionRetryWithProtoRefreshError: ReadWithinUncertaintyIntervalError: "+ "read at time 100.000000000,0 encountered previous write with future timestamp 100.000000000,1: "+ '"sql txn" meta={id=00000000}',],)def test_exhausted_connection_drops_are_classified_retryable(self, source, error_msg):@@ -763,6 +771,11 @@ class TestPostgresSourceNonRetryableErrors:"canceling statement due to user request","canceled",),+ (+ "restart transaction: TransactionRetryWithProtoRefreshError: ReadWithinUncertaintyIntervalError: "+ "read at time 100.000000000,0",+ "serialization conflict",+ ),],)def test_exhausted_retries_replace_raw_driver_text(self, source, error_msg, expected_phrase):