The agent engineering platform.
Python agentsaiai-agentsanthropicchatgptdeepagentsenterpriseframeworkgeminigenerative-ailangchainlanggraphllmmultiagentopen-sourceopenaipydanticpythonragtypescript
community: Enhance MongoDBLoader with flexible metadata and optimized field extraction (#23376)
RUO0a177ec2cc39
diff --git a/libs/community/langchain_community/document_loaders/mongodb.py b/libs/community/langchain_community/document_loaders/mongodb.pyindex b1e0622265..7eddb2f7eb 100644--- a/libs/community/langchain_community/document_loaders/mongodb.py+++ b/libs/community/langchain_community/document_loaders/mongodb.py@@ -20,13 +20,37 @@ class MongodbLoader(BaseLoader):*,filter_criteria: Optional[Dict] = None,field_names: Optional[Sequence[str]] = None,+ metadata_names: Optional[Sequence[str]] = None,+ include_db_collection_in_metadata: bool = True,) -> None:+ """+ Initializes the MongoDB loader with necessary database connection+ details and configurations.++ Args:+ connection_string (str): MongoDB connection URI.+ db_name (str):Name of the database to connect to.+ collection_name (str): Name of the collection to fetch documents from.+ filter_criteria (Optional[Dict]): MongoDB filter criteria for querying+ documents.+ field_names (Optional[Sequence[str]]): List of field names to retrieve+ from documents.+ metadata_names (Optional[Sequence[str]]): Additional metadata fields to+ extract from documents.+ include_db_collection_in_metadata (bool): Flag to include database and+ collection names in metadata.++ Raises:+ ImportError: If the motor library is not installed.+ ValueError: If any necessary argument is missing.+ """try:from motor.motor_asyncio import AsyncIOMotorClientexcept ImportError as e:raise ImportError("Cannot import from motor, please install with `pip install motor`.") from e+if not connection_string:raise ValueError("connection_string must be provided.")@@ -39,8 +63,10 @@ class MongodbLoader(BaseLoader):self.client = AsyncIOMotorClient(connection_string)self.db_name = db_nameself.collection_name = collection_name- self.field_names = field_names+ self.field_names = field_names or []self.filter_criteria = filter_criteria or {}+ self.metadata_names = metadata_names or []+ self.include_db_collection_in_metadata = include_db_collection_in_metadataself.db = self.client.get_database(db_name)self.collection = self.db.get_collection(collection_name)@@ -60,36 +86,24 @@ class MongodbLoader(BaseLoader):return asyncio.run(self.aload())async def aload(self) -> List[Document]:- """Load data into Document objects."""+ """Asynchronously loads data into Document objects."""result = []total_docs = await self.collection.count_documents(self.filter_criteria)- # Construct the projection dictionary if field_names are specified- projection = (- {field: 1 for field in self.field_names} if self.field_names else None- )+ projection = self._construct_projection()async for doc in self.collection.find(self.filter_criteria, projection):- metadata = {- "database": self.db_name,- "collection": self.collection_name,- }+ metadata = self._extract_fields(doc, self.metadata_names, default="")++ # Optionally add database and collection names to metadata+ if self.include_db_collection_in_metadata:+ metadata.update(+ {"database": self.db_name, "collection": self.collection_name}+ )# Extract text content from filtered fields or use the entire documentif self.field_names is not None:- fields = {}- for name in self.field_names:- # Split the field names to handle nested fields- keys = name.split(".")- value = doc- for key in keys:- if key in value:- value = value[key]- else:- value = ""- break- fields[name] = value-+ fields = self._extract_fields(doc, self.field_names, default="")texts = [str(value) for value in fields.values()]text = " ".join(texts)else:@@ -104,3 +118,29 @@ class MongodbLoader(BaseLoader):)return result++ def _construct_projection(self) -> Optional[Dict]:+ """Constructs the projection dictionary for MongoDB query based+ on the specified field names and metadata names."""+ field_names = list(self.field_names) or []+ metadata_names = list(self.metadata_names) or []+ all_fields = field_names + metadata_names+ return {field: 1 for field in all_fields} if all_fields else None++ def _extract_fields(+ self,+ document: Dict,+ fields: Sequence[str],+ default: str = "",+ ) -> Dict:+ """Extracts and returns values for specified fields from a document."""+ extracted = {}+ for field in fields or []:+ value = document+ for key in field.split("."):+ value = value.get(key, default)+ if value == default:+ break+ new_field_name = field.replace(".", "_")+ extracted[new_field_name] = value+ return extracteddiff --git a/libs/community/tests/unit_tests/document_loaders/test_mongodb.py b/libs/community/tests/unit_tests/document_loaders/test_mongodb.pyindex 121e8b39fd..72ed08905f 100644--- a/libs/community/tests/unit_tests/document_loaders/test_mongodb.py+++ b/libs/community/tests/unit_tests/document_loaders/test_mongodb.py@@ -12,6 +12,7 @@ def raw_docs() -> List[Dict]:return [{"_id": "1", "address": {"building": "1", "room": "1"}},{"_id": "2", "address": {"building": "2", "room": "2"}},+ {"_id": "3", "address": {"building": "3", "room": "2"}},]@@ -19,18 +20,23 @@ def raw_docs() -> List[Dict]:def expected_documents() -> List[Document]:return [Document(- page_content="{'_id': '1', 'address': {'building': '1', 'room': '1'}}",+ page_content="{'_id': '2', 'address': {'building': '2', 'room': '2'}}",metadata={"database": "sample_restaurants", "collection": "restaurants"},),Document(- page_content="{'_id': '2', 'address': {'building': '2', 'room': '2'}}",+ page_content="{'_id': '3', 'address': {'building': '3', 'room': '2'}}",metadata={"database": "sample_restaurants", "collection": "restaurants"},),]@pytest.mark.requires("motor")-async def test_load_mocked(expected_documents: List[Document]) -> None:+async def test_load_mocked_with_filters(expected_documents: List[Document]) -> None:+ filter_criteria = {"address.room": {"$eq": "2"}}+ field_names = ["address.building", "address.room"]+ metadata_names = ["_id"]+ include_db_collection_in_metadata = True+mock_async_load = AsyncMock()mock_async_load.return_value = expected_documents@@ -51,7 +57,13 @@ async def test_load_mocked(expected_documents: List[Document]) -> None:new=mock_async_load,):loader = MongodbLoader(- "mongodb://localhost:27017", "test_db", "test_collection"+ "mongodb://localhost:27017",+ "test_db",+ "test_collection",+ filter_criteria=filter_criteria,+ field_names=field_names,+ metadata_names=metadata_names,+ include_db_collection_in_metadata=include_db_collection_in_metadata,)loader.collection = mock_collectiondocuments = await loader.aload()