The agent engineering platform.
Python agentsaiai-agentsanthropicchatgptdeepagentsenterpriseframeworkgeminigenerative-ailangchainlanggraphllmmultiagentopen-sourceopenaipydanticpythonragtypescript
LangChain-MongoDB: [Experimental] Driver-side index creation helper (#19359)
Jibc0fcf76e93c9
diff --git a/docs/docs/integrations/vectorstores/mongodb_atlas.ipynb b/docs/docs/integrations/vectorstores/mongodb_atlas.ipynbindex af6cc33404..c0415fed23 100644--- a/docs/docs/integrations/vectorstores/mongodb_atlas.ipynb+++ b/docs/docs/integrations/vectorstores/mongodb_atlas.ipynb@@ -143,6 +143,28 @@" }\n"," ]\n","}\n",+ "```\n",+ "\n",+ "Additionally, if you are running a MongoDB M10 cluster with server version 6.0+, you can leverage the `MongoDBAtlasVectorSearch.create_index`. To add the above index its usage would look like this.\n",+ "\n",+ "```python\n",+ "from langchain_community.embeddings.openai import OpenAIEmbeddings\n",+ "from langchain_mongodb.vectorstores import MongoDBAtlasVectorSearch\n",+ "from pymongo import MongoClient\n",+ "\n",+ "mongo_client = MongoClient(\"<YOUR-CONNECTION-STRING>\")\n",+ "collection = mongo_client[\"<db_name>\"][\"<collection_name>\"]\n",+ "embeddings = OpenAIEmbeddings()\n",+ "\n",+ "vectorstore = MongoDBAtlasVectorSearch(\n",+ " collection=collection,\n",+ " embedding=embeddings,\n",+ " index_name=\"<ATLAS_VECTOR_SEARCH_INDEX_NAME>\",\n",+ " relevance_score_fn=\"cosine\",\n",+ ")\n",+ "\n",+ "# Creates an index using the index_name provided and relevance_score_fn type\n",+ "vectorstore.create_index(dimensions=1536)\n","```"]},@@ -296,6 +318,16 @@" }\n"," ]\n","}\n",+ "```\n",+ "\n",+ "You can also update the index programmatically using the `MongoDBAtlasVectorSearch.create_index` method.\n",+ "\n",+ "```python\n",+ "vectorstore.create_index(\n",+ " dimensions=1536,\n",+ " filters=[{\"type\":\"filter\", \"path\":\"page\"}],\n",+ " update=True\n",+ ")\n","```"]},diff --git a/libs/partners/mongodb/README.md b/libs/partners/mongodb/README.mdindex ca420328a2..a74fe65947 100644--- a/libs/partners/mongodb/README.md+++ b/libs/partners/mongodb/README.md@@ -6,7 +6,7 @@ pip install -U langchain-mongodb```# Usage-- See [integrations doc](../../../docs/docs/integrations/vectorstores/mongodb.ipynb) for more in-depth usage instructions.+- See [integrations doc](../../../docs/docs/integrations/providers/mongodb_atlas.ipynb) for more in-depth usage instructions.- See [Getting Started with the LangChain Integration](https://www.mongodb.com/docs/atlas/atlas-vector-search/ai-integrations/langchain/#get-started-with-the-langchain-integration) for a walkthrough on using your first LangChain implementation with MongoDB Atlas.## Using MongoDBAtlasVectorSearchdiff --git a/libs/partners/mongodb/langchain_mongodb/index.py b/libs/partners/mongodb/langchain_mongodb/index.pynew file mode 100644index 0000000000..a2c0e10ed6--- /dev/null+++ b/libs/partners/mongodb/langchain_mongodb/index.py@@ -0,0 +1,105 @@+import logging+from typing import Any, Dict, List, Optional++from pymongo.collection import Collection+from pymongo.operations import SearchIndexModel++logger = logging.getLogger(__file__)+++def _vector_search_index_definition(+ dimensions: int,+ path: str,+ similarity: str,+ filters: Optional[List[Dict[str, str]]],+) -> Dict[str, Any]:+ return {+ "fields": [+ {+ "numDimensions": dimensions,+ "path": path,+ "similarity": similarity,+ "type": "vector",+ },+ *(filters or []),+ ]+ }+++def create_vector_search_index(+ collection: Collection,+ index_name: str,+ dimensions: int,+ path: str,+ similarity: str,+ filters: List[Dict[str, str]],+) -> None:+ """Experimental Utility function to create a vector search index++ Args:+ collection (Collection): MongoDB Collection+ index_name (str): Name of Index+ dimensions (int): Number of dimensions in embedding+ path (str): field with vector embedding+ similarity (str): The similarity score used for the index+ filters (List[Dict[str, str]]): additional filters for index definition.+ """+ logger.info("Creating Search Index %s on %s", index_name, collection.name)+ result = collection.create_search_index(+ SearchIndexModel(+ definition=_vector_search_index_definition(+ dimensions=dimensions, path=path, similarity=similarity, filters=filters+ ),+ name=index_name,+ type="vectorSearch",+ )+ )+ logger.info(result)+++def drop_vector_search_index(collection: Collection, index_name: str) -> None:+ """Drop a created vector search index++ Args:+ collection (Collection): MongoDB Collection with index to be dropped+ index_name (str): Name of the MongoDB index+ """+ logger.info(+ "Dropping Search Index %s from Collection: %s", index_name, collection.name+ )+ collection.drop_search_index(index_name)+ logger.info("Vector Search index %s.%s dropped", collection.name, index_name)+++def update_vector_search_index(+ collection: Collection,+ index_name: str,+ dimensions: int,+ path: str,+ similarity: str,+ filters: List[Dict[str, str]],+) -> None:+ """Leverages the updateSearchIndex call++ Args:+ collection (Collection): MongoDB Collection+ index_name (str): Name of Index+ dimensions (int): Number of dimensions in embedding.+ path (str): field with vector embedding.+ similarity (str): The similarity score used for the index.+ filters (List[Dict[str, str]]): additional filters for index definition.+ """++ logger.info(+ "Updating Search Index %s from Collection: %s", index_name, collection.name+ )+ collection.update_search_index(+ name=index_name,+ definition=_vector_search_index_definition(+ dimensions=dimensions,+ path=path,+ similarity=similarity,+ filters=filters,+ ),+ )+ logger.info("Update succeeded")diff --git a/libs/partners/mongodb/langchain_mongodb/utils.py b/libs/partners/mongodb/langchain_mongodb/utils.pyindex 9c345e1520..cea4b8c044 100644--- a/libs/partners/mongodb/langchain_mongodb/utils.py+++ b/libs/partners/mongodb/langchain_mongodb/utils.py@@ -18,6 +18,11 @@ logger = logging.getLogger(__name__)Matrix = Union[List[List[float]], List[np.ndarray], np.ndarray]+class FailCode:+ INDEX_NOT_FOUND = 27+ INDEX_ALREADY_EXISTS = 68++def cosine_similarity(X: Matrix, Y: Matrix) -> np.ndarray:"""Row-wise cosine similarity between two equal-width matrices."""if len(X) == 0 or len(Y) == 0:diff --git a/libs/partners/mongodb/langchain_mongodb/vectorstores.py b/libs/partners/mongodb/langchain_mongodb/vectorstores.pyindex a703d59c51..5084814619 100644--- a/libs/partners/mongodb/langchain_mongodb/vectorstores.py+++ b/libs/partners/mongodb/langchain_mongodb/vectorstores.py@@ -24,7 +24,12 @@ from langchain_core.vectorstores import VectorStorefrom pymongo import MongoClientfrom pymongo.collection import Collectionfrom pymongo.driver_info import DriverInfo+from pymongo.errors import CollectionInvalid+from langchain_mongodb.index import (+ create_vector_search_index,+ update_vector_search_index,+)from langchain_mongodb.utils import maximal_marginal_relevanceMongoDBDocumentType = TypeVar("MongoDBDocumentType", bound=Dict[str, Any])@@ -489,3 +494,42 @@ class MongoDBAtlasVectorSearch(VectorStore):lambda_mult=lambda_mult,**kwargs,)++ def create_vector_search_index(+ self,+ dimensions: int,+ filters: Optional[List[Dict[str, str]]] = None,+ update: bool = False,+ ) -> None:+ """Creates a MongoDB Atlas vectorSearch index for the VectorStore++ Note**: This method may fail as it requires a MongoDB Atlas with+ these pre-requisites:+ - M10 cluster or higher+ - https://www.mongodb.com/docs/atlas/atlas-vector-search/create-index/#prerequisites++ Args:+ dimensions (int): Number of dimensions in embedding+ filters (Optional[List[Dict[str, str]]], optional): additional filters+ for index definition.+ Defaults to None.+ update (bool, optional): Updates existing vectorSearch index.+ Defaults to False.+ """+ try:+ self._collection.database.create_collection(self._collection.name)+ except CollectionInvalid:+ pass++ index_operation = (+ update_vector_search_index if update else create_vector_search_index+ )++ index_operation(+ collection=self._collection,+ index_name=self._index_name,+ dimensions=dimensions,+ path=self._embedding_key,+ similarity=self._relevance_score_fn,+ filters=filters or [],+ )diff --git a/libs/partners/mongodb/tests/integration_tests/test_vectorstores.py b/libs/partners/mongodb/tests/integration_tests/test_vectorstores.pyindex 451ff29113..6767563e24 100644--- a/libs/partners/mongodb/tests/integration_tests/test_vectorstores.py+++ b/libs/partners/mongodb/tests/integration_tests/test_vectorstores.py@@ -3,22 +3,27 @@from __future__ import annotationsimport os-from time import sleep-from typing import Any, Dict, List+from time import monotonic, sleep+from typing import Any, Dict, List, Optionalimport pytestfrom langchain_core.documents import Documentfrom langchain_core.embeddings import Embeddingsfrom pymongo import MongoClientfrom pymongo.collection import Collection+from pymongo.errors import OperationFailurefrom langchain_mongodb import MongoDBAtlasVectorSearch+from langchain_mongodb.index import drop_vector_search_indexfrom tests.utils import ConsistentFakeEmbeddingsINDEX_NAME = "langchain-test-index-vectorstores"+INDEX_CREATION_NAME = "langchain-test-index-vectorstores-create-test"NAMESPACE = "langchain_test_db.langchain_test_vectorstores"CONNECTION_STRING = os.environ.get("MONGODB_ATLAS_URI")DB_NAME, COLLECTION_NAME = NAMESPACE.split(".")+INDEX_COLLECTION_NAME = "langchain_test_vectorstores_index"+INDEX_DB_NAME = "langchain_test_index_db"DIMENSIONS = 1536TIMEOUT = 10.0INTERVAL = 0.5@@ -28,16 +33,53 @@ class PatchedMongoDBAtlasVectorSearch(MongoDBAtlasVectorSearch):def _insert_texts(self, texts: List[str], metadatas: List[Dict[str, Any]]) -> List:"""Patched insert_texts that waits for data to be indexed before returning"""ids = super()._insert_texts(texts, metadatas)- timeout = TIMEOUT- while len(ids) != self.similarity_search("sandwich") and timeout >= 0:+ start = monotonic()+ while len(ids) != self.similarity_search("sandwich") and (+ monotonic() - start <= TIMEOUT+ ):sleep(INTERVAL)- timeout -= INTERVALreturn ids+ def create_vector_search_index(+ self,+ dimensions: int,+ filters: Optional[List[Dict[str, str]]] = None,+ update: bool = False,+ ) -> None:+ result = super().create_vector_search_index(+ dimensions=dimensions, filters=filters, update=update+ )+ start = monotonic()+ while monotonic() - start <= TIMEOUT:+ if indexes := list(+ self._collection.list_search_indexes(name=self._index_name)+ ):+ if indexes[0].get("status") == "READY":+ return result+ sleep(INTERVAL)++ raise TimeoutError(f"{self._index_name} never reached 'status: READY'")+-def get_collection() -> Collection:+def _await_index_deletion(coll: Collection, index_name: str) -> None:+ start = monotonic()+ try:+ drop_vector_search_index(coll, index_name)+ except OperationFailure:+ # This most likely means an ongoing drop request was made so skip+ pass++ while list(coll.list_search_indexes(name=index_name)):+ if monotonic() - start > TIMEOUT:+ raise TimeoutError(f"Index Name: {index_name} never dropped")+ sleep(INTERVAL)+++def get_collection(+ database_name: str = DB_NAME, collection_name: str = COLLECTION_NAME+) -> Collection:test_client: MongoClient = MongoClient(CONNECTION_STRING)- return test_client[DB_NAME][COLLECTION_NAME]+ return test_client[database_name][collection_name]@pytest.fixture()@@ -45,6 +87,11 @@ def collection() -> Collection:return get_collection()+def index_collection() -> Collection:+ return get_collection(INDEX_DB_NAME, INDEX_COLLECTION_NAME)++class TestMongoDBAtlasVectorSearch:@classmethoddef setup_class(cls) -> None:@@ -65,6 +112,11 @@ class TestMongoDBAtlasVectorSearch:# delete all the documents in the collectioncollection.delete_many({}) # type: ignore[index]+ # delete all indexes on index collection name+ _await_index_deletion(+ get_collection(INDEX_DB_NAME, INDEX_COLLECTION_NAME), INDEX_CREATION_NAME+ )+@pytest.fixturedef embedding_openai(self) -> Embeddings:return ConsistentFakeEmbeddings(DIMENSIONS)@@ -85,7 +137,6 @@ class TestMongoDBAtlasVectorSearch:collection=collection,index_name=INDEX_NAME,)- # sleep(5) # waits for mongot to update Lucene's indexoutput = vectorstore.similarity_search("Sandwich", k=1)assert len(output) == 1# Check for the presence of the metadata key@@ -150,7 +201,6 @@ class TestMongoDBAtlasVectorSearch:collection=collection,index_name=INDEX_NAME,)- # sleep(5) # waits for mongot to update Lucene's indexoutput = vectorstore.similarity_search("Sandwich", k=1)assert len(output) == 1@@ -172,7 +222,6 @@ class TestMongoDBAtlasVectorSearch:collection=collection,index_name=INDEX_NAME,)- # sleep(5) # waits for mongot to update Lucene's indexoutput = vectorstore.similarity_search("Sandwich", k=1)assert len(output) == 1# Check for the presence of the metadata key@@ -195,7 +244,6 @@ class TestMongoDBAtlasVectorSearch:collection=collection,index_name=INDEX_NAME,)- # sleep(5) # waits for mongot to update Lucene's indexoutput = vectorstore.similarity_search("Sandwich", k=1, pre_filter={"c": {"$lte": 0}})@@ -209,9 +257,25 @@ class TestMongoDBAtlasVectorSearch:collection=collection,index_name=INDEX_NAME,)- # sleep(5) # waits for mongot to update Lucene's indexquery = "foo"output = vectorstore.max_marginal_relevance_search(query, k=10, lambda_mult=0.1)assert len(output) == len(texts)assert output[0].page_content == "foo"assert output[1].page_content != "foo"++ def test_index_creation(+ self, embedding_openai: Embeddings, index_collection: Any+ ) -> None:+ vectorstore = PatchedMongoDBAtlasVectorSearch(+ index_collection, embedding_openai, index_name=INDEX_CREATION_NAME+ )+ vectorstore.create_vector_search_index(dimensions=1536)++ def test_index_update(+ self, embedding_openai: Embeddings, index_collection: Any+ ) -> None:+ vectorstore = PatchedMongoDBAtlasVectorSearch(+ index_collection, embedding_openai, index_name=INDEX_CREATION_NAME+ )+ vectorstore.create_vector_search_index(dimensions=1536)+ vectorstore.create_vector_search_index(dimensions=1536, update=True)