Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-warehouse): resume mongodb first syncs and cap incremental reads (#120430)

Daniel RC2439d65f4786
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/mongodb/mongo.py b/products/warehouse_sources/backend/temporal/data_imports/sources/mongodb/mongo.py
index dca7efba3908..7218a54f3cf0 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/mongodb/mongo.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/mongodb/mongo.py
@@ -20,7 +20,7 @@ from bson.code import Code
from bson.codec_options import DatetimeConversion
from bson.dbref import DBRef
from bson.json_util import JSONMode, JSONOptions
-from pymongo import ASCENDING, MongoClient
+from pymongo import ASCENDING, DESCENDING, MongoClient
from pymongo.collection import Collection
from pymongo.cursor import Cursor
from pymongo.database import Database
@@ -265,6 +265,7 @@ def _build_query(
incremental_field: Optional[str],
incremental_field_type: Optional[IncrementalFieldType],
db_incremental_field_last_value: Optional[Any],
+ upper_bound: Optional[Any] = None,
) -> dict[str, Any]:
query: dict[str, Any] = {}
@@ -282,6 +283,9 @@ def _build_query(
else:
query = {incremental_field: {"$gt": db_incremental_field_last_value, "$exists": True}}
+ if upper_bound is not None:
+ query[incremental_field]["$lte"] = upper_bound
+
return query
@@ -679,6 +683,59 @@ def _estimate_rows_to_sync(collection: Collection, query: dict[str, Any], logger
return 0
+def _serves_range_and_sort(spec: dict[str, Any], incremental_field: str) -> bool:
+ """Whether the planner can serve `field > x` ordered by `field` from this index.
+
+ A hashed, text or geo index serves neither the range nor the sort, a partial index serves them
+ only for a query that implies its filter, which the cap query does not, and the planner ignores
+ a hidden index entirely. The field has to come first: a compound index that only holds it later
+ cannot order by it on its own.
+ """
+ field, direction = spec["key"][0]
+ return (
+ field == incremental_field
+ and direction in (ASCENDING, DESCENDING)
+ and "partialFilterExpression" not in spec
+ and not spec.get("hidden")
+ )
+
+
+def _read_upper_bound(
+ collection: Collection, query: dict[str, Any], incremental_field: str, logger: FilteringBoundLogger
+) -> Any | None:
+ """The newest incremental value when the read starts, or None to leave the read uncapped.
+
+ The read is in `_id` order, and the watermark it stores is the newest value it read anywhere.
+ An update to a document the cursor already passed can get a value below that watermark, and no
+ later run reads it. With the read capped here, such an update lands above the cap and the
+ watermark, so the next run reads it.
+ """
+ try:
+ indexed = any(
+ _serves_range_and_sort(spec, incremental_field) for spec in collection.index_information().values()
+ )
+ if not indexed:
+ # Without a usable index the lookup scans the whole collection a second time.
+ logger.debug(f"MongoDB: no usable index on {incremental_field}, so the read is not capped")
+ return None
+ newest = collection.find_one(
+ query,
+ {incremental_field: 1},
+ sort=[(incremental_field, DESCENDING)],
+ max_time_ms=COLLECTION_STATS_TIMEOUT_MS,
+ )
+ except _UNREACHABLE_CLUSTER_ERRORS:
+ raise
+ except PyMongoError as e:
+ logger.warning(f"MongoDB: could not read the newest {incremental_field} ({e}), so the read is not capped")
+ return None
+ if newest is None:
+ # Nothing matches yet. Capping at the lower bound reads nothing, so a document written during
+ # the read waits for the next run, as it does when the read is capped.
+ return query[incremental_field]["$gt"]
+ return newest.get(incremental_field)
+
+
def _get_rows_to_sync(collection: Collection, query: dict[str, Any], logger: FilteringBoundLogger) -> int:
try:
rows_to_sync = collection.count_documents(query, maxTimeMS=ROW_COUNT_TIMEOUT_MS)
@@ -755,15 +812,17 @@ _EXECUTION_TIMEOUT_ERROR_CODE = 50
@frozen
class MongoResumeConfig:
- """Checkpoint of a full-refresh read: the `_id` of the last document handed to the pipeline.
+ """Checkpoint of a read: the `_id` of the last document handed to the pipeline, and the cap of
+ an incremental read, so every attempt of the run reads with the same filter.
- The value is canonical MongoDB Extended JSON. Plain JSON has no ObjectId, UUID, date or
+ The values are canonical MongoDB Extended JSON. Plain JSON has no ObjectId, UUID, date or
Decimal128, and the resume manager writes `str()` of a value it cannot encode, which no later
attempt can parse. Extended JSON gives back the same BSON type and value, so `$gt` compares it
the way the server compared the original.
"""
last_id: str
+ upper_bound: str | None = None
# UNSPECIFIED keeps every binary `_id` a `Binary` of its own subtype on decode. A representation
@@ -837,20 +896,28 @@ def _to_exact_bson(value: Any) -> Any:
return value
-def encode_resume_id(value: Any) -> str | None:
- """Extended JSON for a document `_id`, or None when a resumed read cannot seek past it."""
- if _id_type_bracket(value) is None:
- return None
+def _encode_bson_value(value: Any) -> str | None:
try:
return json_util.dumps(_to_exact_bson(value), json_options=_RESUME_ID_JSON_OPTIONS)
except (TypeError, ValueError):
return None
-def decode_resume_id(encoded: str) -> Any:
+def encode_resume_id(value: Any) -> str | None:
+ """Extended JSON for a document `_id`, or None when a resumed read cannot seek past it."""
+ if _id_type_bracket(value) is None:
+ return None
+ return _encode_bson_value(value)
+
+
+def _decode_bson_value(encoded: str) -> Any:
return json_util.loads(encoded, json_options=_RESUME_ID_JSON_OPTIONS)
+def decode_resume_id(encoded: str) -> Any:
+ return _decode_bson_value(encoded)
+
+
def _id_seek_filter(last_id: Any) -> dict[str, Any]:
"""Match every document whose `_id` sorts after `last_id` in MongoDB's cross-type order."""
bracket = _id_type_bracket(last_id)
@@ -874,11 +941,12 @@ def mongo_source(
) -> SourceResponse:
connection_params = _parse_connection_string(connection_string, database_name)
- # Only a full refresh resumes. Its read is `find({}).sort(_id)` with no filter, so every document
- # that sorts before the checkpoint was already handed on. An incremental or append read restarts
- # from its incremental watermark instead, and an `_id` seek would have to agree with a filter on
- # another field.
- resume_manager = None if should_use_incremental_field else resumable_source_manager
+ # The read is sorted by `_id`, so every document before the checkpoint was already handed on as
+ # long as the filter is the same on every attempt. A full refresh has no filter. A first
+ # incremental or append sync filters on the initial watermark, which no attempt moves. A later
+ # incremental read restarts from its stored watermark instead.
+ resumes_by_id = not should_use_incremental_field or db_incremental_field_last_value is None
+ resume_manager = resumable_source_manager if resumes_by_id else None
if not connection_params["database"]:
raise ValueError(DATABASE_NAME_REQUIRED_ERROR)
@@ -921,20 +989,42 @@ def mongo_source(
db_incremental_field_last_value,
)
- if resume_manager is not None and resume_manager.can_resume():
- state = resume_manager.load_state()
- if state is not None:
- try:
- resume_after = decode_resume_id(state.last_id)
- query = _id_seek_filter(resume_after)
- except Exception:
- # This attempt already counts as a resume, so it cannot fall back to a
- # fresh read. Clearing the state lets the next attempt start one.
- resume_manager.clear_state()
- raise
- logger.debug(
- f"MongoDB: resuming collection={collection_name} after _id type={type(resume_after).__name__}"
- )
+ state = resume_manager.load_state() if resume_manager is not None and resume_manager.can_resume() else None
+ seek: dict[str, Any] | None = None
+ upper_bound: Any = None
+ encoded_bound: str | None = None
+ if resume_manager is not None and state is not None:
+ try:
+ resume_after = decode_resume_id(state.last_id)
+ seek = _id_seek_filter(resume_after)
+ # A new bound would change the filter, and the documents before the checkpoint
+ # were read with the old one.
+ encoded_bound = state.upper_bound
+ upper_bound = None if encoded_bound is None else _decode_bson_value(encoded_bound)
+ except Exception:
+ # This attempt already counts as a resume, so it cannot fall back to a
+ # fresh read. Clearing the state lets the next attempt start one.
+ resume_manager.clear_state()
+ raise
+ logger.debug(
+ f"MongoDB: resuming collection={collection_name} after _id type={type(resume_after).__name__}"
+ )
+ elif should_use_incremental_field and incremental_field is not None:
+ upper_bound = _read_upper_bound(read_collection, query, incremental_field, logger)
+ encoded_bound = None if upper_bound is None else _encode_bson_value(upper_bound)
+ if encoded_bound is None:
+ upper_bound = None
+
+ if upper_bound is not None:
+ query = _build_query(
+ should_use_incremental_field,
+ incremental_field,
+ incremental_field_type,
+ db_incremental_field_last_value,
+ upper_bound,
+ )
+ if seek is not None:
+ query = {"$and": [query, seek]} if query else seek
# Set when the resume state was cleared for an `_id` that cannot be a checkpoint, so a
# run of such documents costs one Redis delete and not one per document.
@@ -999,7 +1089,9 @@ def mongo_source(
# after it writes the batch that holds this document.
encoded_id = encode_resume_id(last_id)
if encoded_id is not None:
- resume_manager.save_state(MongoResumeConfig(last_id=encoded_id))
+ resume_manager.save_state(
+ MongoResumeConfig(last_id=encoded_id, upper_bound=encoded_bound)
+ )
resume_state_cleared = False
elif not resume_state_cleared:
# An older checkpoint would re-read the rows written since it,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/mongodb/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/mongodb/source.py
index 97e534ef2878..19bb38ea0816 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/mongodb/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/mongodb/source.py
@@ -554,8 +554,8 @@ class MongoDBSource(ResumableSource[MongoDBSourceConfig, MongoResumeConfig], Val
incremental_or_append: bool,
schema_name: str | None = None,
) -> bool:
- # The `_id` checkpoint covers only a full refresh. An incremental or append run restarts from
- # its watermark, so it keeps the incremental retry budget.
+ # The `_id` checkpoint covers a full refresh. It also covers a first incremental or append
+ # sync, but this cannot see the watermark, so those runs keep the incremental retry budget.
return not incremental_or_append
def get_resumable_source_manager(self, inputs: SourceInputs) -> ResumableSourceManager[MongoResumeConfig]:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/mongodb/test_mongo.py b/products/warehouse_sources/backend/temporal/data_imports/sources/mongodb/test_mongo.py
index 02e67517fe11..97caa64aefde 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/mongodb/test_mongo.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/mongodb/test_mongo.py
@@ -18,6 +18,7 @@ from bson.codec_options import CodecOptions
from bson.max_key import MaxKey
from bson.min_key import MinKey
from parameterized import parameterized
+from pymongo import DESCENDING
from pymongo.errors import CursorNotFound, ExecutionTimeout, OperationFailure, ServerSelectionTimeoutError
from pymongo.hello import Hello
from pymongo.server_description import ServerDescription
@@ -49,6 +50,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.mongodb.mo
_make_safe_server_selector,
_process_doc_with_field_logging,
_process_nested_value,
+ _read_upper_bound,
connection_timeouts,
decode_resume_id,
encode_resume_id,
@@ -712,6 +714,28 @@ class TestProbesFailFastOnUnreachableCluster(SimpleTestCase):
with pytest.raises(type(error)):
_get_avg_document_size(coll, MagicMock())
+ @parameterized.expand(
+ [
+ ("server_selection_timeout", ServerSelectionTimeoutError("No servers found yet, Topology Description: .")),
+ ("host_not_allowed", HostNotAllowedError(f"{DATABASE_HOST_NOT_ALLOWED_ERROR}: internal IP")),
+ ]
+ )
+ def test_upper_bound_probe_propagates(self, _name, error):
+ coll = MagicMock()
+ coll.index_information.side_effect = error
+
+ with pytest.raises(type(error)):
+ _read_upper_bound(coll, {}, "updated_at", MagicMock())
+
+
+class TestReadUpperBound(SimpleTestCase):
+ def test_a_lookup_that_fails_leaves_the_read_uncapped(self):
+ coll = MagicMock()
+ coll.index_information.return_value = {"updated_at_1": {"key": [("updated_at", 1)]}}
+ coll.find_one.side_effect = ExecutionTimeout("operation exceeded time limit")
+
+ assert _read_upper_bound(coll, {}, "updated_at", MagicMock()) is None
+
class TestListImportableCollectionNames(SimpleTestCase):
def test_excludes_reserved_system_collections(self):
@@ -801,6 +825,18 @@ class _FakeCollection:
self.last_cursor: _FakeCursor | None = None
self.database = MagicMock()
self.database.command.return_value = {"avgObjSize": None}
+ self.indexes: dict[str, Any] = {"_id_": {"key": [("_id", 1)]}}
+ self.newest: dict[str, Any] | None = None
+ self.find_one_calls: list[dict[str, Any]] = []
+ self.find_one_sorts: list[Any] = []
+
+ def index_information(self) -> dict[str, Any]:
+ return self.indexes
+
+ def find_one(self, query: dict[str, Any], *args: Any, **kwargs: Any) -> dict[str, Any] | None:
+ self.find_one_calls.append(query)
+ self.find_one_sorts.append(kwargs.get("sort"))
+ return self.newest
def find(self, query: dict[str, Any], **kwargs: Any) -> _FakeCursor:
self.find_kwargs = kwargs
@@ -831,6 +867,7 @@ def _read_rows(
*,
resumable_source_manager: Any = None,
should_use_incremental_field: bool = False,
+ db_incremental_field_last_value: Any = None,
on_row: Callable[[dict[str, Any]], None] | None = None,
) -> tuple[Any, list[dict[str, Any]]]:
@contextlib.contextmanager
@@ -850,7 +887,7 @@ def _read_rows(
logger=MagicMock(),
team_id=1,
should_use_incremental_field=should_use_incremental_field,
- db_incremental_field_last_value=None,
+ db_incremental_field_last_value=db_incremental_field_last_value,
incremental_field="_id" if should_use_incremental_field else None,
incremental_field_type=IncrementalFieldType.ObjectID if should_use_incremental_field else None,
resumable_source_manager=resumable_source_manager,
@@ -1127,11 +1164,16 @@ class TestMongoSourceResume(SimpleTestCase):
assert [row["_id"] for row in rows] == [str(doc["_id"]) for doc in docs]
assert collection.find_queries[1] == {"$and": [_id_seek_filter(saved_id), {"_id": {"$gt": docs[1]["_id"]}}]}
- def test_incremental_run_does_not_resume(self) -> None:
+ def test_incremental_run_with_a_watermark_does_not_resume(self) -> None:
collection = _FakeCollection([{"_id": ObjectId("65f1c2a4e4b0a1b2c3d4e5f6")}])
manager = _FakeResumeManager(_saved_state(ObjectId("65f1c2a4e4b0a1b2c3d4e5f0")))
- response, rows = _read_rows(collection, resumable_source_manager=manager, should_use_incremental_field=True)
+ response, rows = _read_rows(
+ collection,
+ resumable_source_manager=manager,
+ should_use_incremental_field=True,
+ db_incremental_field_last_value=ObjectId("65f1c2a4e4b0a1b2c3d4e5e0"),
+ )
assert len(rows) == 1
assert response.supports_resume is False
@@ -1139,6 +1181,71 @@ class TestMongoSourceResume(SimpleTestCase):
assert "$or" not in collection.find_queries[0]
assert (manager.load_calls, manager.staged, manager.clear_calls) == (0, [], 0)
+ @parameterized.expand([("capped", ObjectId("65f1c2a4e4b0a1b2c3d4e5f9")), ("uncapped", None)])
+ def test_first_incremental_sync_resumes_inside_the_filter_of_its_first_attempt(
+ self, _name: str, saved_bound: ObjectId | None
+ ) -> None:
+ saved_id = ObjectId("65f1c2a4e4b0a1b2c3d4e5f0")
+ next_id = ObjectId("65f1c2a4e4b0a1b2c3d4e5f6")
+ collection = _FakeCollection([{"_id": next_id}])
+ collection.newest = {"_id": ObjectId("65f1c2a4e4b0a1b2c3d4e5ff")}
+ bound = None if saved_bound is None else encode_resume_id(saved_bound)
+ manager = _FakeResumeManager(MongoResumeConfig(last_id=_saved_state(saved_id).last_id, upper_bound=bound))
+
+ response, rows = _read_rows(collection, resumable_source_manager=manager, should_use_incremental_field=True)
+
+ assert [row["_id"] for row in rows] == [str(next_id)]
+ assert response.supports_resume is True
+ incremental_filter, seek = collection.find_queries[0]["$and"]
+ assert incremental_filter == _build_query(True, "_id", IncrementalFieldType.ObjectID, None, saved_bound)
+ assert seek == _id_seek_filter(saved_id)
+ assert collection.find_one_calls == []
+ assert [(decode_resume_id(s.last_id), s.upper_bound) for s in manager.staged] == [(next_id, bound)]
+
+ @parameterized.expand(
+ [
+ ("ascending_index", {"_id_": {"key": [("_id", 1)]}}, True),
+ ("descending_index", {"_id_-1": {"key": [("_id", -1)]}}, True),
+ ("hashed_index", {"_id_hashed": {"key": [("_id", "hashed")]}}, False),
+ (
+ "partial_index",
+ {"_id_1": {"key": [("_id", 1)], "partialFilterExpression": {"active": True}}},
+ False,
+ ),
+ ("hidden_index", {"_id_1": {"key": [("_id", 1)], "hidden": True}}, False),
+ ("compound_index_on_another_field_first", {"other_1__id_1": {"key": [("other", 1), ("_id", 1)]}}, False),
+ ("not_indexed", {}, False),
+ ]
+ )
+ def test_first_incremental_sync_caps_its_read_at_the_newest_value_when_it_starts(
+ self, _name: str, indexes: dict[str, Any], capped: bool
+ ) -> None:
+ newest = ObjectId("65f1c2a4e4b0a1b2c3d4e5f9")
+ doc_id = ObjectId("65f1c2a4e4b0a1b2c3d4e5f6")
+ collection = _FakeCollection([{"_id": doc_id}])
+ collection.newest = {"_id": newest}
+ collection.indexes = indexes
+ manager = _FakeResumeManager()
+
+ _read_rows(collection, resumable_source_manager=manager, should_use_incremental_field=True)
+
+ bound = newest if capped else None
+ assert collection.find_queries[0] == _build_query(True, "_id", IncrementalFieldType.ObjectID, None, bound)
+ assert [s.upper_bound for s in manager.staged] == [encode_resume_id(bound) if capped else None]
+ probes = [_build_query(True, "_id", IncrementalFieldType.ObjectID, None)] if capped else []
+ assert collection.find_one_calls == probes
+ assert collection.find_one_sorts == ([[("_id", DESCENDING)]] if capped else [])
+
+ def test_incremental_read_with_nothing_to_read_at_its_start_reads_nothing(self) -> None:
+ collection = _FakeCollection([{"_id": ObjectId("65f1c2a4e4b0a1b2c3d4e5f6")}])
+ collection.newest = None
+
+ _read_rows(collection, resumable_source_manager=_FakeResumeManager(), should_use_incremental_field=True)
+
+ lower_bound = _build_query(True, "_id", IncrementalFieldType.ObjectID, None)["_id"]["$gt"]
+ expected = _build_query(True, "_id", IncrementalFieldType.ObjectID, None, lower_bound)
+ assert collection.find_queries[0] == expected
+
def test_id_without_a_checkpoint_clears_the_state_once_per_run_of_such_ids(self) -> None:
collection = _FakeCollection([{"_id": None}, {"_id": float("nan")}, {"_id": 1}, {"_id": MaxKey()}])
manager = _FakeResumeManager()