:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(data-warehouse): resume mongodb first syncs and cap incremental reads (#120430)
Daniel RC2439d65f4786
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/mongodb/mongo.py b/products/warehouse_sources/backend/temporal/data_imports/sources/mongodb/mongo.pyindex dca7efba3908..7218a54f3cf0 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/mongodb/mongo.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/mongodb/mongo.py@@ -20,7 +20,7 @@ from bson.code import Codefrom bson.codec_options import DatetimeConversionfrom bson.dbref import DBReffrom bson.json_util import JSONMode, JSONOptions-from pymongo import ASCENDING, MongoClient+from pymongo import ASCENDING, DESCENDING, MongoClientfrom pymongo.collection import Collectionfrom pymongo.cursor import Cursorfrom pymongo.database import Database@@ -265,6 +265,7 @@ def _build_query(incremental_field: Optional[str],incremental_field_type: Optional[IncrementalFieldType],db_incremental_field_last_value: Optional[Any],+ upper_bound: Optional[Any] = None,) -> dict[str, Any]:query: dict[str, Any] = {}@@ -282,6 +283,9 @@ def _build_query(else:query = {incremental_field: {"$gt": db_incremental_field_last_value, "$exists": True}}+ if upper_bound is not None:+ query[incremental_field]["$lte"] = upper_bound+return query@@ -679,6 +683,59 @@ def _estimate_rows_to_sync(collection: Collection, query: dict[str, Any], loggerreturn 0+def _serves_range_and_sort(spec: dict[str, Any], incremental_field: str) -> bool:+ """Whether the planner can serve `field > x` ordered by `field` from this index.++ A hashed, text or geo index serves neither the range nor the sort, a partial index serves them+ only for a query that implies its filter, which the cap query does not, and the planner ignores+ a hidden index entirely. The field has to come first: a compound index that only holds it later+ cannot order by it on its own.+ """+ field, direction = spec["key"][0]+ return (+ field == incremental_field+ and direction in (ASCENDING, DESCENDING)+ and "partialFilterExpression" not in spec+ and not spec.get("hidden")+ )+++def _read_upper_bound(+ collection: Collection, query: dict[str, Any], incremental_field: str, logger: FilteringBoundLogger+) -> Any | None:+ """The newest incremental value when the read starts, or None to leave the read uncapped.++ The read is in `_id` order, and the watermark it stores is the newest value it read anywhere.+ An update to a document the cursor already passed can get a value below that watermark, and no+ later run reads it. With the read capped here, such an update lands above the cap and the+ watermark, so the next run reads it.+ """+ try:+ indexed = any(+ _serves_range_and_sort(spec, incremental_field) for spec in collection.index_information().values()+ )+ if not indexed:+ # Without a usable index the lookup scans the whole collection a second time.+ logger.debug(f"MongoDB: no usable index on {incremental_field}, so the read is not capped")+ return None+ newest = collection.find_one(+ query,+ {incremental_field: 1},+ sort=[(incremental_field, DESCENDING)],+ max_time_ms=COLLECTION_STATS_TIMEOUT_MS,+ )+ except _UNREACHABLE_CLUSTER_ERRORS:+ raise+ except PyMongoError as e:+ logger.warning(f"MongoDB: could not read the newest {incremental_field} ({e}), so the read is not capped")+ return None+ if newest is None:+ # Nothing matches yet. Capping at the lower bound reads nothing, so a document written during+ # the read waits for the next run, as it does when the read is capped.+ return query[incremental_field]["$gt"]+ return newest.get(incremental_field)++def _get_rows_to_sync(collection: Collection, query: dict[str, Any], logger: FilteringBoundLogger) -> int:try:rows_to_sync = collection.count_documents(query, maxTimeMS=ROW_COUNT_TIMEOUT_MS)@@ -755,15 +812,17 @@ _EXECUTION_TIMEOUT_ERROR_CODE = 50@frozenclass MongoResumeConfig:- """Checkpoint of a full-refresh read: the `_id` of the last document handed to the pipeline.+ """Checkpoint of a read: the `_id` of the last document handed to the pipeline, and the cap of+ an incremental read, so every attempt of the run reads with the same filter.- The value is canonical MongoDB Extended JSON. Plain JSON has no ObjectId, UUID, date or+ The values are canonical MongoDB Extended JSON. Plain JSON has no ObjectId, UUID, date orDecimal128, and the resume manager writes `str()` of a value it cannot encode, which no laterattempt can parse. Extended JSON gives back the same BSON type and value, so `$gt` compares itthe way the server compared the original."""last_id: str+ upper_bound: str | None = None# UNSPECIFIED keeps every binary `_id` a `Binary` of its own subtype on decode. A representation@@ -837,20 +896,28 @@ def _to_exact_bson(value: Any) -> Any:return value-def encode_resume_id(value: Any) -> str | None:- """Extended JSON for a document `_id`, or None when a resumed read cannot seek past it."""- if _id_type_bracket(value) is None:- return None+def _encode_bson_value(value: Any) -> str | None:try:return json_util.dumps(_to_exact_bson(value), json_options=_RESUME_ID_JSON_OPTIONS)except (TypeError, ValueError):return None-def decode_resume_id(encoded: str) -> Any:+def encode_resume_id(value: Any) -> str | None:+ """Extended JSON for a document `_id`, or None when a resumed read cannot seek past it."""+ if _id_type_bracket(value) is None:+ return None+ return _encode_bson_value(value)+++def _decode_bson_value(encoded: str) -> Any:return json_util.loads(encoded, json_options=_RESUME_ID_JSON_OPTIONS)+def decode_resume_id(encoded: str) -> Any:+ return _decode_bson_value(encoded)++def _id_seek_filter(last_id: Any) -> dict[str, Any]:"""Match every document whose `_id` sorts after `last_id` in MongoDB's cross-type order."""bracket = _id_type_bracket(last_id)@@ -874,11 +941,12 @@ def mongo_source() -> SourceResponse:connection_params = _parse_connection_string(connection_string, database_name)- # Only a full refresh resumes. Its read is `find({}).sort(_id)` with no filter, so every document- # that sorts before the checkpoint was already handed on. An incremental or append read restarts- # from its incremental watermark instead, and an `_id` seek would have to agree with a filter on- # another field.- resume_manager = None if should_use_incremental_field else resumable_source_manager+ # The read is sorted by `_id`, so every document before the checkpoint was already handed on as+ # long as the filter is the same on every attempt. A full refresh has no filter. A first+ # incremental or append sync filters on the initial watermark, which no attempt moves. A later+ # incremental read restarts from its stored watermark instead.+ resumes_by_id = not should_use_incremental_field or db_incremental_field_last_value is None+ resume_manager = resumable_source_manager if resumes_by_id else Noneif not connection_params["database"]:raise ValueError(DATABASE_NAME_REQUIRED_ERROR)@@ -921,20 +989,42 @@ def mongo_source(db_incremental_field_last_value,)- if resume_manager is not None and resume_manager.can_resume():- state = resume_manager.load_state()- if state is not None:- try:- resume_after = decode_resume_id(state.last_id)- query = _id_seek_filter(resume_after)- except Exception:- # This attempt already counts as a resume, so it cannot fall back to a- # fresh read. Clearing the state lets the next attempt start one.- resume_manager.clear_state()- raise- logger.debug(- f"MongoDB: resuming collection={collection_name} after _id type={type(resume_after).__name__}"- )+ state = resume_manager.load_state() if resume_manager is not None and resume_manager.can_resume() else None+ seek: dict[str, Any] | None = None+ upper_bound: Any = None+ encoded_bound: str | None = None+ if resume_manager is not None and state is not None:+ try:+ resume_after = decode_resume_id(state.last_id)+ seek = _id_seek_filter(resume_after)+ # A new bound would change the filter, and the documents before the checkpoint+ # were read with the old one.+ encoded_bound = state.upper_bound+ upper_bound = None if encoded_bound is None else _decode_bson_value(encoded_bound)+ except Exception:+ # This attempt already counts as a resume, so it cannot fall back to a+ # fresh read. Clearing the state lets the next attempt start one.+ resume_manager.clear_state()+ raise+ logger.debug(+ f"MongoDB: resuming collection={collection_name} after _id type={type(resume_after).__name__}"+ )+ elif should_use_incremental_field and incremental_field is not None:+ upper_bound = _read_upper_bound(read_collection, query, incremental_field, logger)+ encoded_bound = None if upper_bound is None else _encode_bson_value(upper_bound)+ if encoded_bound is None:+ upper_bound = None++ if upper_bound is not None:+ query = _build_query(+ should_use_incremental_field,+ incremental_field,+ incremental_field_type,+ db_incremental_field_last_value,+ upper_bound,+ )+ if seek is not None:+ query = {"$and": [query, seek]} if query else seek# Set when the resume state was cleared for an `_id` that cannot be a checkpoint, so a# run of such documents costs one Redis delete and not one per document.@@ -999,7 +1089,9 @@ def mongo_source(# after it writes the batch that holds this document.encoded_id = encode_resume_id(last_id)if encoded_id is not None:- resume_manager.save_state(MongoResumeConfig(last_id=encoded_id))+ resume_manager.save_state(+ MongoResumeConfig(last_id=encoded_id, upper_bound=encoded_bound)+ )resume_state_cleared = Falseelif not resume_state_cleared:# An older checkpoint would re-read the rows written since it,diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/mongodb/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/mongodb/source.pyindex 97e534ef2878..19bb38ea0816 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/mongodb/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/mongodb/source.py@@ -554,8 +554,8 @@ class MongoDBSource(ResumableSource[MongoDBSourceConfig, MongoResumeConfig], Valincremental_or_append: bool,schema_name: str | None = None,) -> bool:- # The `_id` checkpoint covers only a full refresh. An incremental or append run restarts from- # its watermark, so it keeps the incremental retry budget.+ # The `_id` checkpoint covers a full refresh. It also covers a first incremental or append+ # sync, but this cannot see the watermark, so those runs keep the incremental retry budget.return not incremental_or_appenddef get_resumable_source_manager(self, inputs: SourceInputs) -> ResumableSourceManager[MongoResumeConfig]:diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/mongodb/test_mongo.py b/products/warehouse_sources/backend/temporal/data_imports/sources/mongodb/test_mongo.pyindex 02e67517fe11..97caa64aefde 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/mongodb/test_mongo.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/mongodb/test_mongo.py@@ -18,6 +18,7 @@ from bson.codec_options import CodecOptionsfrom bson.max_key import MaxKeyfrom bson.min_key import MinKeyfrom parameterized import parameterized+from pymongo import DESCENDINGfrom pymongo.errors import CursorNotFound, ExecutionTimeout, OperationFailure, ServerSelectionTimeoutErrorfrom pymongo.hello import Hellofrom pymongo.server_description import ServerDescription@@ -49,6 +50,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.mongodb.mo_make_safe_server_selector,_process_doc_with_field_logging,_process_nested_value,+ _read_upper_bound,connection_timeouts,decode_resume_id,encode_resume_id,@@ -712,6 +714,28 @@ class TestProbesFailFastOnUnreachableCluster(SimpleTestCase):with pytest.raises(type(error)):_get_avg_document_size(coll, MagicMock())+ @parameterized.expand(+ [+ ("server_selection_timeout", ServerSelectionTimeoutError("No servers found yet, Topology Description: .")),+ ("host_not_allowed", HostNotAllowedError(f"{DATABASE_HOST_NOT_ALLOWED_ERROR}: internal IP")),+ ]+ )+ def test_upper_bound_probe_propagates(self, _name, error):+ coll = MagicMock()+ coll.index_information.side_effect = error++ with pytest.raises(type(error)):+ _read_upper_bound(coll, {}, "updated_at", MagicMock())+++class TestReadUpperBound(SimpleTestCase):+ def test_a_lookup_that_fails_leaves_the_read_uncapped(self):+ coll = MagicMock()+ coll.index_information.return_value = {"updated_at_1": {"key": [("updated_at", 1)]}}+ coll.find_one.side_effect = ExecutionTimeout("operation exceeded time limit")++ assert _read_upper_bound(coll, {}, "updated_at", MagicMock()) is None+class TestListImportableCollectionNames(SimpleTestCase):def test_excludes_reserved_system_collections(self):@@ -801,6 +825,18 @@ class _FakeCollection:self.last_cursor: _FakeCursor | None = Noneself.database = MagicMock()self.database.command.return_value = {"avgObjSize": None}+ self.indexes: dict[str, Any] = {"_id_": {"key": [("_id", 1)]}}+ self.newest: dict[str, Any] | None = None+ self.find_one_calls: list[dict[str, Any]] = []+ self.find_one_sorts: list[Any] = []++ def index_information(self) -> dict[str, Any]:+ return self.indexes++ def find_one(self, query: dict[str, Any], *args: Any, **kwargs: Any) -> dict[str, Any] | None:+ self.find_one_calls.append(query)+ self.find_one_sorts.append(kwargs.get("sort"))+ return self.newestdef find(self, query: dict[str, Any], **kwargs: Any) -> _FakeCursor:self.find_kwargs = kwargs@@ -831,6 +867,7 @@ def _read_rows(*,resumable_source_manager: Any = None,should_use_incremental_field: bool = False,+ db_incremental_field_last_value: Any = None,on_row: Callable[[dict[str, Any]], None] | None = None,) -> tuple[Any, list[dict[str, Any]]]:@contextlib.contextmanager@@ -850,7 +887,7 @@ def _read_rows(logger=MagicMock(),team_id=1,should_use_incremental_field=should_use_incremental_field,- db_incremental_field_last_value=None,+ db_incremental_field_last_value=db_incremental_field_last_value,incremental_field="_id" if should_use_incremental_field else None,incremental_field_type=IncrementalFieldType.ObjectID if should_use_incremental_field else None,resumable_source_manager=resumable_source_manager,@@ -1127,11 +1164,16 @@ class TestMongoSourceResume(SimpleTestCase):assert [row["_id"] for row in rows] == [str(doc["_id"]) for doc in docs]assert collection.find_queries[1] == {"$and": [_id_seek_filter(saved_id), {"_id": {"$gt": docs[1]["_id"]}}]}- def test_incremental_run_does_not_resume(self) -> None:+ def test_incremental_run_with_a_watermark_does_not_resume(self) -> None:collection = _FakeCollection([{"_id": ObjectId("65f1c2a4e4b0a1b2c3d4e5f6")}])manager = _FakeResumeManager(_saved_state(ObjectId("65f1c2a4e4b0a1b2c3d4e5f0")))- response, rows = _read_rows(collection, resumable_source_manager=manager, should_use_incremental_field=True)+ response, rows = _read_rows(+ collection,+ resumable_source_manager=manager,+ should_use_incremental_field=True,+ db_incremental_field_last_value=ObjectId("65f1c2a4e4b0a1b2c3d4e5e0"),+ )assert len(rows) == 1assert response.supports_resume is False@@ -1139,6 +1181,71 @@ class TestMongoSourceResume(SimpleTestCase):assert "$or" not in collection.find_queries[0]assert (manager.load_calls, manager.staged, manager.clear_calls) == (0, [], 0)+ @parameterized.expand([("capped", ObjectId("65f1c2a4e4b0a1b2c3d4e5f9")), ("uncapped", None)])+ def test_first_incremental_sync_resumes_inside_the_filter_of_its_first_attempt(+ self, _name: str, saved_bound: ObjectId | None+ ) -> None:+ saved_id = ObjectId("65f1c2a4e4b0a1b2c3d4e5f0")+ next_id = ObjectId("65f1c2a4e4b0a1b2c3d4e5f6")+ collection = _FakeCollection([{"_id": next_id}])+ collection.newest = {"_id": ObjectId("65f1c2a4e4b0a1b2c3d4e5ff")}+ bound = None if saved_bound is None else encode_resume_id(saved_bound)+ manager = _FakeResumeManager(MongoResumeConfig(last_id=_saved_state(saved_id).last_id, upper_bound=bound))++ response, rows = _read_rows(collection, resumable_source_manager=manager, should_use_incremental_field=True)++ assert [row["_id"] for row in rows] == [str(next_id)]+ assert response.supports_resume is True+ incremental_filter, seek = collection.find_queries[0]["$and"]+ assert incremental_filter == _build_query(True, "_id", IncrementalFieldType.ObjectID, None, saved_bound)+ assert seek == _id_seek_filter(saved_id)+ assert collection.find_one_calls == []+ assert [(decode_resume_id(s.last_id), s.upper_bound) for s in manager.staged] == [(next_id, bound)]++ @parameterized.expand(+ [+ ("ascending_index", {"_id_": {"key": [("_id", 1)]}}, True),+ ("descending_index", {"_id_-1": {"key": [("_id", -1)]}}, True),+ ("hashed_index", {"_id_hashed": {"key": [("_id", "hashed")]}}, False),+ (+ "partial_index",+ {"_id_1": {"key": [("_id", 1)], "partialFilterExpression": {"active": True}}},+ False,+ ),+ ("hidden_index", {"_id_1": {"key": [("_id", 1)], "hidden": True}}, False),+ ("compound_index_on_another_field_first", {"other_1__id_1": {"key": [("other", 1), ("_id", 1)]}}, False),+ ("not_indexed", {}, False),+ ]+ )+ def test_first_incremental_sync_caps_its_read_at_the_newest_value_when_it_starts(+ self, _name: str, indexes: dict[str, Any], capped: bool+ ) -> None:+ newest = ObjectId("65f1c2a4e4b0a1b2c3d4e5f9")+ doc_id = ObjectId("65f1c2a4e4b0a1b2c3d4e5f6")+ collection = _FakeCollection([{"_id": doc_id}])+ collection.newest = {"_id": newest}+ collection.indexes = indexes+ manager = _FakeResumeManager()++ _read_rows(collection, resumable_source_manager=manager, should_use_incremental_field=True)++ bound = newest if capped else None+ assert collection.find_queries[0] == _build_query(True, "_id", IncrementalFieldType.ObjectID, None, bound)+ assert [s.upper_bound for s in manager.staged] == [encode_resume_id(bound) if capped else None]+ probes = [_build_query(True, "_id", IncrementalFieldType.ObjectID, None)] if capped else []+ assert collection.find_one_calls == probes+ assert collection.find_one_sorts == ([[("_id", DESCENDING)]] if capped else [])++ def test_incremental_read_with_nothing_to_read_at_its_start_reads_nothing(self) -> None:+ collection = _FakeCollection([{"_id": ObjectId("65f1c2a4e4b0a1b2c3d4e5f6")}])+ collection.newest = None++ _read_rows(collection, resumable_source_manager=_FakeResumeManager(), should_use_incremental_field=True)++ lower_bound = _build_query(True, "_id", IncrementalFieldType.ObjectID, None)["_id"]["$gt"]+ expected = _build_query(True, "_id", IncrementalFieldType.ObjectID, None, lower_bound)+ assert collection.find_queries[0] == expected+def test_id_without_a_checkpoint_clears_the_state_once_per_run_of_such_ids(self) -> None:collection = _FakeCollection([{"_id": None}, {"_id": float("nan")}, {"_id": 1}, {"_id": MaxKey()}])manager = _FakeResumeManager()