Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(mongodb): stop retrying when the Atlas cluster is gone (#120871)

Tom Owers32cd69ee99fa
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/mongodb/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/mongodb/source.py
index e23f5d4fbed7..97e534ef2878 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/mongodb/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/mongodb/source.py
@@ -114,6 +114,19 @@ _MONGO_KEY_NOT_FOUND_MESSAGE = (
"and ensure all replica set members are healthy, then try again."
)
+# MongoDB OperationFailure code 125 (CommandFailed) with this errmsg comes from Atlas Data
+# Federation / Online Archive: the federated database's data source still names a cluster that
+# was since renamed or deleted from the Atlas project. The cluster name and project id are
+# variable, so match the fixed phrase between them rather than the whole sentence. A retry reads
+# the same stale data source config and fails the same way every time.
+_MONGO_CLUSTER_NOT_FOUND_MESSAGE = (
+ "MongoDB reported that the cluster this connection refers to no longer exists in your Atlas "
+ "project (MongoDB error code 125: CommandFailed). This happens when connecting through an "
+ "Atlas Data Federation or Online Archive endpoint whose data source still points at a cluster "
+ "that was renamed or deleted. Update the data source in your Federated Database Instance, or "
+ "connect with your cluster's regular connection string instead, then try again."
+)
+
# Connection succeeded but nothing importable came back. This is usually a wrong-database or
# permission problem rather than a genuinely empty database: a connection string ending in /admin
# or /test lands on an empty system database, and a user without read access sees no collections.
@@ -312,6 +325,9 @@ class MongoDBSource(ResumableSource[MongoDBSourceConfig, MongoResumeConfig], Val
# (timestamp, key id) is stable and unambiguous. Retrying the same cursor against the
# same cluster always fails with the same error.
"No keys found for HMAC": _MONGO_KEY_NOT_FOUND_MESSAGE,
+ # MongoDB OperationFailure code 125 (CommandFailed): an Atlas Data Federation / Online
+ # Archive data source still names a cluster that no longer exists in the project.
+ "not found in project": _MONGO_CLUSTER_NOT_FOUND_MESSAGE,
}
def get_retryable_errors(self) -> set[str]:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/mongodb/test_mongo.py b/products/warehouse_sources/backend/temporal/data_imports/sources/mongodb/test_mongo.py
index f7ca9676760c..02e67517fe11 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/mongodb/test_mongo.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/mongodb/test_mongo.py
@@ -435,6 +435,15 @@ class TestMongoDBNonRetryableErrors(SimpleTestCase):
"Server at cluster.abc.mongodb.net:27017 reports wire version 7, but this version of "
"PyMongo requires at least 8 (MongoDB 4.2).",
),
+ # Real pymongo OperationFailure string for an Atlas Data Federation / Online Archive
+ # data source whose cluster was renamed or deleted (code 125, CommandFailed).
+ (
+ "cluster_not_found_in_project",
+ 'cluster "myCluster0" not found in project 000000000000000000000000, correlationID = '
+ "000000000000000000000000, full error: {'ok': 0, 'errmsg': 'cluster \"myCluster0\" not "
+ "found in project 000000000000000000000000, correlationID = 000000000000000000000000', "
+ "'code': 125, 'codeName': 'CommandFailed'}",
+ ),
]
)
def test_known_errors_are_non_retryable(self, _name, error_msg):
@@ -493,6 +502,7 @@ class TestMongoDBNonRetryableErrors(SimpleTestCase):
("server_too_old", "version of PyMongo requires at least", "upgrade the cluster"),
("replica_set_no_members", "No replica set members available for replica set name", "replica set"),
("replica_set_mismatch", "client is configured to connect to a replica set named", "replica set"),
+ ("cluster_not_found_in_project", "not found in project", "data federation"),
]
)
def test_pattern_has_friendly_message(self, _name, pattern, expected_substring):