Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(postgres): stop flooding error tracking with CockroachDB serialization conflicts (#123635)

Tom Owers2ddba67e9b0f
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py
index 7e653b52a5de..b42ec63db6cc 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py
@@ -515,6 +515,14 @@ _SPURIOUS_CANCEL_EXHAUSTED_MESSAGE = (
"its next schedule."
)
+_SERIALIZATION_CONFLICT_EXHAUSTED_MESSAGE = (
+ "Your database kept aborting PostHog's read with a transaction serialization conflict, and the "
+ "conflict outlasted every retry. This is common on CockroachDB and similar databases under "
+ "heavy write load. Check how busy the table is during the sync window, or point the connection "
+ "at a replica with less write traffic. This sync is still enabled and will run again on its "
+ "next schedule."
+)
+
@SourceRegistry.register
class PostgresSource(
@@ -1723,6 +1731,15 @@ class PostgresSource(
# `_SSH_FORWARD_FAILED_ERROR` reaches here once `_connected_ssh_tunnel`'s own ~14 second
# retry gives up opening the port forward to the database behind the bastion. See its
# definition above for why that stays retryable rather than disabling the schema.
+ #
+ # "TransactionRetryWithProtoRefreshError" is CockroachDB's serialization-conflict wrapper
+ # (its wire protocol is Postgres-compatible, so customers configure it as a plain Postgres
+ # source — see the TABLESAMPLE workarounds in postgres.py). It covers every conflict reason
+ # CockroachDB nests under it (read-within-uncertainty-interval, write-too-old, ...), unlike
+ # "conflict with recovery" above, which is specifically Postgres hot-standby wording and
+ # never appears in CockroachDB's message. CockroachDB expects the whole transaction to be
+ # retried on conflict, which is exactly what letting this propagate to Temporal's
+ # whole-activity retry does, so it's the same self-recovering class as the other entries here.
return {
CLIENT_DEADLINE_ERROR,
"canceling statement due to user request",
@@ -1731,6 +1748,7 @@ class PostgresSource(
*_SERVER_STARTING_UP_ERROR_SUBSTRINGS,
*_CONNECTION_LIMIT_ERROR_SUBSTRINGS,
"conflict with recovery",
+ "TransactionRetryWithProtoRefreshError",
HOST_RESOLUTION_TIMEOUT_ERROR,
TEMPORARY_HOST_RESOLUTION_ERROR,
_SSH_FORWARD_FAILED_ERROR,
@@ -1752,6 +1770,7 @@ class PostgresSource(
**dict.fromkeys(_SERVER_STARTING_UP_ERROR_SUBSTRINGS, _SERVER_UNAVAILABLE_EXHAUSTED_MESSAGE),
**dict.fromkeys(_CONNECTION_LIMIT_ERROR_SUBSTRINGS, _CONNECTION_LIMIT_EXHAUSTED_MESSAGE),
"conflict with recovery": _RECOVERY_CONFLICT_EXHAUSTED_MESSAGE,
+ "TransactionRetryWithProtoRefreshError": _SERIALIZATION_CONFLICT_EXHAUSTED_MESSAGE,
CLIENT_DEADLINE_ERROR: _CLIENT_DEADLINE_EXHAUSTED_MESSAGE,
"canceling statement due to user request": _SPURIOUS_CANCEL_EXHAUSTED_MESSAGE,
HOST_RESOLUTION_TIMEOUT_ERROR: HOST_RESOLUTION_EXHAUSTED_MESSAGE,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py
index 6198efa783da..bea8d9f063d5 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py
@@ -734,6 +734,14 @@ class TestPostgresSourceNonRetryableErrors:
# recycling the connection, an admin cancel) hit the query. Neither is a verdict on this
# query, so a fresh connection on retry recovers.
"canceling statement due to user request",
+ # CockroachDB's serialization-conflict wrapper (its Postgres-compatible wire protocol
+ # means customers configure it as a plain Postgres source). CockroachDB expects the
+ # whole transaction to be retried on conflict, which the whole-activity Temporal retry
+ # already does, so it's self-recovering like the other entries here. Timestamps and txn
+ # ids are invented; CockroachDB embeds both in every message of this class.
+ "restart transaction: TransactionRetryWithProtoRefreshError: ReadWithinUncertaintyIntervalError: "
+ "read at time 100.000000000,0 encountered previous write with future timestamp 100.000000000,1: "
+ '"sql txn" meta={id=00000000}',
],
)
def test_exhausted_connection_drops_are_classified_retryable(self, source, error_msg):
@@ -763,6 +771,11 @@ class TestPostgresSourceNonRetryableErrors:
"canceling statement due to user request",
"canceled",
),
+ (
+ "restart transaction: TransactionRetryWithProtoRefreshError: ReadWithinUncertaintyIntervalError: "
+ "read at time 100.000000000,0",
+ "serialization conflict",
+ ),
],
)
def test_exhausted_retries_replace_raw_driver_text(self, source, error_msg, expected_phrase):