The agent engineering platform.
Python agentsaiai-agentsanthropicchatgptdeepagentsenterpriseframeworkgeminigenerative-ailangchainlanggraphllmmultiagentopen-sourceopenaipydanticpythonragtypescript
core[minor]: Improve support for id in VectorStore (#26660)
Christophe Bornetc4ebccfec2e1
diff --git a/libs/community/langchain_community/vectorstores/usearch.py b/libs/community/langchain_community/vectorstores/usearch.pyindex fa94d19de0..c59446de54 100644--- a/libs/community/langchain_community/vectorstores/usearch.py+++ b/libs/community/langchain_community/vectorstores/usearch.py@@ -1,6 +1,6 @@from __future__ import annotations-from typing import Any, Dict, Iterable, List, Optional, Tuple+from typing import Any, Dict, Iterable, List, Optional, Tuple, Unionimport numpy as npfrom langchain_core.documents import Document@@ -42,7 +42,7 @@ class USearch(VectorStore):self,texts: Iterable[str],metadatas: Optional[List[Dict]] = None,- ids: Optional[np.ndarray] = None,+ ids: Optional[Union[np.ndarray, list[str]]] = None,**kwargs: Any,) -> List[str]:"""Run more texts through the embeddings and add to the vectorstore.@@ -69,6 +69,8 @@ class USearch(VectorStore):last_id = int(self.ids[-1]) + 1if ids is None:ids = np.array([str(last_id + id) for id, _ in enumerate(texts)])+ elif isinstance(ids, list):+ ids = np.array(ids)self.index.add(np.array(ids), np.array(embeddings))self.docstore.add(dict(zip(ids, documents)))@@ -134,7 +136,7 @@ class USearch(VectorStore):texts: List[str],embedding: Embeddings,metadatas: Optional[List[Dict]] = None,- ids: Optional[np.ndarray] = None,+ ids: Optional[Union[np.ndarray, list[str]]] = None,metric: str = "cos",**kwargs: Any,) -> USearch:@@ -159,6 +161,8 @@ class USearch(VectorStore):documents: List[Document] = []if ids is None:ids = np.array([str(id) for id, _ in enumerate(texts)])+ elif isinstance(ids, list):+ ids = np.array(ids)for i, text in enumerate(texts):metadata = metadatas[i] if metadatas else {}documents.append(Document(page_content=text, metadata=metadata))diff --git a/libs/core/langchain_core/vectorstores/base.py b/libs/core/langchain_core/vectorstores/base.pyindex aacf897ffb..ccc966d9f2 100644--- a/libs/core/langchain_core/vectorstores/base.py+++ b/libs/core/langchain_core/vectorstores/base.py@@ -25,7 +25,7 @@ import loggingimport mathimport warningsfrom abc import ABC, abstractmethod-from collections.abc import Collection, Iterable, Sequence+from collections.abc import Collection, Iterable, Iterator, Sequencefrom itertools import cyclefrom typing import (TYPE_CHECKING,@@ -61,10 +61,8 @@ class VectorStore(ABC):self,texts: Iterable[str],metadatas: Optional[list[dict]] = None,- # One of the kwargs should be `ids` which is a list of ids- # associated with the texts.- # This is not yet enforced in the type signature for backwards compatibility- # with existing implementations.+ *,+ ids: Optional[list[str]] = None,**kwargs: Any,) -> list[str]:"""Run more texts through the embeddings and add to the vectorstore.@@ -72,6 +70,7 @@ class VectorStore(ABC):Args:texts: Iterable of strings to add to the vectorstore.metadatas: Optional list of metadatas associated with the texts.+ ids: Optional list of IDs associated with the texts.**kwargs: vectorstore specific parameters.One of the kwargs should be `ids` which is a list of idsassociated with the texts.@@ -99,10 +98,14 @@ class VectorStore(ABC):f"Got {len(metadatas)} metadatas and {len(texts_)} texts.")metadatas_ = iter(metadatas) if metadatas else cycle([{}])+ ids_: Iterator[Optional[str]] = iter(ids) if ids else cycle([None])docs = [- Document(page_content=text, metadata=metadata_)- for text, metadata_ in zip(texts, metadatas_)+ Document(id=id_, page_content=text, metadata=metadata_)+ for text, metadata_, id_ in zip(texts, metadatas_, ids_)]+ if ids is not None:+ # For backward compatibility+ kwargs["ids"] = idsreturn self.add_documents(docs, **kwargs)raise NotImplementedError(@@ -206,6 +209,8 @@ class VectorStore(ABC):self,texts: Iterable[str],metadatas: Optional[list[dict]] = None,+ *,+ ids: Optional[list[str]] = None,**kwargs: Any,) -> list[str]:"""Async run more texts through the embeddings and add to the vectorstore.@@ -214,6 +219,7 @@ class VectorStore(ABC):texts: Iterable of strings to add to the vectorstore.metadatas: Optional list of metadatas associated with the texts.Default is None.+ ids: Optional list**kwargs: vectorstore specific parameters.Returns:@@ -223,6 +229,9 @@ class VectorStore(ABC):ValueError: If the number of metadatas does not match the number of texts.ValueError: If the number of ids does not match the number of texts."""+ if ids is not None:+ # For backward compatibility+ kwargs["ids"] = idsif type(self).aadd_documents != VectorStore.aadd_documents:# Import document in local scope to avoid circular importsfrom langchain_core.documents import Document@@ -239,12 +248,12 @@ class VectorStore(ABC):f"Got {len(metadatas)} metadatas and {len(texts_)} texts.")metadatas_ = iter(metadatas) if metadatas else cycle([{}])+ ids_: Iterator[Optional[str]] = iter(ids) if ids else cycle([None])docs = [- Document(page_content=text, metadata=metadata_)- for text, metadata_ in zip(texts, metadatas_)+ Document(id=id_, page_content=text, metadata=metadata_)+ for text, metadata_, id_ in zip(texts, metadatas_, ids_)]-return await self.aadd_documents(docs, **kwargs)return await run_in_executor(None, self.add_texts, texts, metadatas, **kwargs)@@ -827,6 +836,15 @@ class VectorStore(ABC):"""texts = [d.page_content for d in documents]metadatas = [d.metadata for d in documents]++ if "ids" not in kwargs:+ ids = [doc.id for doc in documents]++ # If there's at least one valid ID, we'll assume that IDs+ # should be used.+ if any(ids):+ kwargs["ids"] = ids+return cls.from_texts(texts, embedding, metadatas=metadatas, **kwargs)@classmethod@@ -848,6 +866,15 @@ class VectorStore(ABC):"""texts = [d.page_content for d in documents]metadatas = [d.metadata for d in documents]++ if "ids" not in kwargs:+ ids = [doc.id for doc in documents]++ # If there's at least one valid ID, we'll assume that IDs+ # should be used.+ if any(ids):+ kwargs["ids"] = ids+return await cls.afrom_texts(texts, embedding, metadatas=metadatas, **kwargs)@classmethod@@ -857,6 +884,8 @@ class VectorStore(ABC):texts: list[str],embedding: Embeddings,metadatas: Optional[list[dict]] = None,+ *,+ ids: Optional[list[str]] = None,**kwargs: Any,) -> VST:"""Return VectorStore initialized from texts and embeddings.@@ -866,6 +895,7 @@ class VectorStore(ABC):embedding: Embedding function to use.metadatas: Optional list of metadatas associated with the texts.Default is None.+ ids: Optional list of IDs associated with the texts.kwargs: Additional keyword arguments.Returns:@@ -878,6 +908,8 @@ class VectorStore(ABC):texts: list[str],embedding: Embeddings,metadatas: Optional[list[dict]] = None,+ *,+ ids: Optional[list[str]] = None,**kwargs: Any,) -> VST:"""Async return VectorStore initialized from texts and embeddings.@@ -887,11 +919,14 @@ class VectorStore(ABC):embedding: Embedding function to use.metadatas: Optional list of metadatas associated with the texts.Default is None.+ ids: Optional list of IDs associated with the texts.kwargs: Additional keyword arguments.Returns:VectorStore: VectorStore initialized from texts and embeddings."""+ if ids is not None:+ kwargs["ids"] = idsreturn await run_in_executor(None, cls.from_texts, texts, embedding, metadatas, **kwargs)diff --git a/libs/core/tests/unit_tests/vectorstores/test_vectorstore.py b/libs/core/tests/unit_tests/vectorstores/test_vectorstore.pyindex 29746bbe67..aba1481c62 100644--- a/libs/core/tests/unit_tests/vectorstores/test_vectorstore.py+++ b/libs/core/tests/unit_tests/vectorstores/test_vectorstore.py@@ -10,8 +10,10 @@ import uuidfrom collections.abc import Iterable, Sequencefrom typing import Any, Optional+import pytest+from langchain_core.documents import Document-from langchain_core.embeddings import Embeddings+from langchain_core.embeddings import Embeddings, FakeEmbeddingsfrom langchain_core.vectorstores import VectorStore@@ -25,10 +27,6 @@ class CustomAddTextsVectorstore(VectorStore):self,texts: Iterable[str],metadatas: Optional[list[dict]] = None,- # One of the kwargs should be `ids` which is a list of ids- # associated with the texts.- # This is not yet enforced in the type signature for backwards compatibility- # with existing implementations.ids: Optional[list[str]] = None,**kwargs: Any,) -> list[str]:@@ -68,12 +66,59 @@ class CustomAddTextsVectorstore(VectorStore):raise NotImplementedError()-def test_default_add_documents() -> None:+class CustomAddDocumentsVectorstore(VectorStore):+ """A vectorstore that only implements add documents."""++ def __init__(self) -> None:+ self.store: dict[str, Document] = {}++ def add_documents(+ self,+ documents: list[Document],+ *,+ ids: Optional[list[str]] = None,+ **kwargs: Any,+ ) -> list[str]:+ ids_ = []+ ids_iter = iter(ids or [])+ for document in documents:+ id_ = next(ids_iter) if ids else document.id or str(uuid.uuid4())+ self.store[id_] = Document(+ id=id_, page_content=document.page_content, metadata=document.metadata+ )+ ids_.append(id_)+ return ids_++ def get_by_ids(self, ids: Sequence[str], /) -> list[Document]:+ return [self.store[id] for id in ids if id in self.store]++ @classmethod+ def from_texts( # type: ignore+ cls,+ texts: list[str],+ embedding: Embeddings,+ metadatas: Optional[list[dict]] = None,+ **kwargs: Any,+ ) -> CustomAddDocumentsVectorstore:+ vectorstore = CustomAddDocumentsVectorstore()+ vectorstore.add_texts(texts, metadatas=metadatas, **kwargs)+ return vectorstore++ def similarity_search(+ self, query: str, k: int = 4, **kwargs: Any+ ) -> list[Document]:+ raise NotImplementedError()+++ "vs_class", [CustomAddTextsVectorstore, CustomAddDocumentsVectorstore]+)+def test_default_add_documents(vs_class: type[VectorStore]) -> None:"""Test that we can implement the upsert method of the CustomVectorStoreclass without violating the Liskov Substitution Principle."""- store = CustomAddTextsVectorstore()+ store = vs_class()# Check upsert with idassert store.add_documents([Document(id="1", page_content="hello")]) == ["1"]@@ -95,8 +140,11 @@ def test_default_add_documents() -> None:assert store.get_by_ids(["6"]) == [Document(id="6", page_content="baz")]-def test_default_add_texts() -> None:- store = CustomAddTextsVectorstore()+ "vs_class", [CustomAddTextsVectorstore, CustomAddDocumentsVectorstore]+)+def test_default_add_texts(vs_class: type[VectorStore]) -> None:+ store = vs_class()# Check that default implementation of add_texts worksassert store.add_texts(["hello", "world"], ids=["3", "4"]) == ["3", "4"]@@ -122,9 +170,12 @@ def test_default_add_texts() -> None:]-async def test_default_aadd_documents() -> None:+ "vs_class", [CustomAddTextsVectorstore, CustomAddDocumentsVectorstore]+)+async def test_default_aadd_documents(vs_class: type[VectorStore]) -> None:"""Test delegation to the synchronous method."""- store = CustomAddTextsVectorstore()+ store = vs_class()# Check upsert with idassert await store.aadd_documents([Document(id="1", page_content="hello")]) == ["1"]@@ -146,10 +197,13 @@ async def test_default_aadd_documents() -> None:assert await store.aget_by_ids(["6"]) == [Document(id="6", page_content="baz")]-async def test_default_aadd_texts() -> None:+ "vs_class", [CustomAddTextsVectorstore, CustomAddDocumentsVectorstore]+)+async def test_default_aadd_texts(vs_class: type[VectorStore]) -> None:"""Test delegation to the synchronous method."""- store = CustomAddTextsVectorstore()- # Check that default implementation of add_texts works+ store = vs_class()+ # Check that default implementation of aadd_texts worksassert await store.aadd_texts(["hello", "world"], ids=["3", "4"]) == ["3", "4"]assert await store.aget_by_ids(["3", "4"]) == [@@ -172,3 +226,61 @@ async def test_default_aadd_texts() -> None:Document(id=ids_2[0], page_content="foo", metadata={"foo": "bar"}),Document(id=ids_2[1], page_content="bar", metadata={"foo": "bar"}),]+++ "vs_class", [CustomAddTextsVectorstore, CustomAddDocumentsVectorstore]+)+def test_default_from_documents(vs_class: type[VectorStore]) -> None:+ embeddings = FakeEmbeddings(size=1)+ store = vs_class.from_documents(+ [Document(id="1", page_content="hello", metadata={"foo": "bar"})], embeddings+ )++ assert store.get_by_ids(["1"]) == [+ Document(id="1", page_content="hello", metadata={"foo": "bar"})+ ]++ # from_documents with ids in args+ store = vs_class.from_documents(+ [Document(page_content="hello", metadata={"foo": "bar"})], embeddings, ids=["1"]+ )++ assert store.get_by_ids(["1"]) == [+ Document(id="1", page_content="hello", metadata={"foo": "bar"})+ ]++ # Test from_documents with id specified in both document and ids+ original_document = Document(id="7", page_content="baz")+ store = vs_class.from_documents([original_document], embeddings, ids=["6"])+ assert original_document.id == "7" # original document should not be modified+ assert store.get_by_ids(["6"]) == [Document(id="6", page_content="baz")]+++ "vs_class", [CustomAddTextsVectorstore, CustomAddDocumentsVectorstore]+)+async def test_default_afrom_documents(vs_class: type[VectorStore]) -> None:+ embeddings = FakeEmbeddings(size=1)+ store = await vs_class.afrom_documents(+ [Document(id="1", page_content="hello", metadata={"foo": "bar"})], embeddings+ )++ assert await store.aget_by_ids(["1"]) == [+ Document(id="1", page_content="hello", metadata={"foo": "bar"})+ ]++ # from_documents with ids in args+ store = await vs_class.afrom_documents(+ [Document(page_content="hello", metadata={"foo": "bar"})], embeddings, ids=["1"]+ )++ assert await store.aget_by_ids(["1"]) == [+ Document(id="1", page_content="hello", metadata={"foo": "bar"})+ ]++ # Test afrom_documents with id specified in both document and ids+ original_document = Document(id="7", page_content="baz")+ store = await vs_class.afrom_documents([original_document], embeddings, ids=["6"])+ assert original_document.id == "7" # original document should not be modified+ assert await store.aget_by_ids(["6"]) == [Document(id="6", page_content="baz")]