Skip to content

The agent engineering platform.

tests[patch]: populate API reference for vector stores (#28520)

ccurmef4597544702f
diff --git a/libs/standard-tests/langchain_tests/integration_tests/vectorstores.py b/libs/standard-tests/langchain_tests/integration_tests/vectorstores.py
index 08b0358dcf..07d5ced720 100644
--- a/libs/standard-tests/langchain_tests/integration_tests/vectorstores.py
+++ b/libs/standard-tests/langchain_tests/integration_tests/vectorstores.py
@@ -15,18 +15,68 @@ EMBEDDING_SIZE = 6
class ReadWriteTestSuite(BaseStandardTests):
- """Test suite for checking the read-write API of a vectorstore.
-
- This test suite verifies the basic read-write API of a vectorstore.
-
- The test suite is designed for synchronous vectorstores.
+ """Test suite for checking the synchronous read-write API of a vector store.
Implementers should subclass this test suite and provide a fixture
- that returns an empty vectorstore for each test.
+ that returns an empty vector store for each test.
- The fixture should use the `get_embeddings` method to get a pre-defined
+ The fixture should use the ``get_embeddings`` method to get a pre-defined
embeddings model that should be used for this test suite.
- """
+
+ Here is a template:
+
+ .. code-block:: python
+
+ from typing import Generator
+
+ import pytest
+ from langchain_core.vectorstores import VectorStore
+ from langchain_parrot_link.vectorstores import ParrotVectorStore
+ from langchain_tests.integration_tests.vectorstores import ReadWriteTestSuite
+
+
+ class TestSync(ReadWriteTestSuite):
+ @pytest.fixture()
+ def vectorstore(self) -> Generator[VectorStore, None, None]: # type: ignore
+ \"\"\"Get an empty vectorstore.\"\"\"
+ store = ParrotVectorStore(self.get_embeddings())
+ # note: store should be EMPTY at this point
+ # if you need to delete data, you may do so here
+ try:
+ yield store
+ finally:
+ # cleanup operations, or deleting data
+ pass
+
+ In the fixture, before the ``yield`` we instantiate an empty vector store. In the
+ ``finally`` block, we call whatever logic is necessary to bring the vector store
+ to a clean state.
+
+ Example:
+
+ .. code-block:: python
+
+ from typing import Generator
+
+ import pytest
+ from langchain_core.vectorstores import VectorStore
+ from langchain_tests.integration_tests.vectorstores import ReadWriteTestSuite
+
+ from langchain_chroma import Chroma
+
+
+ class TestSync(ReadWriteTestSuite):
+ @pytest.fixture()
+ def vectorstore(self) -> Generator[VectorStore, None, None]: # type: ignore
+ \"\"\"Get an empty vectorstore.\"\"\"
+ store = Chroma(embedding_function=self.get_embeddings())
+ try:
+ yield store
+ finally:
+ store.delete_collection()
+ pass
+
+ """ # noqa: E501
@abstractmethod
@pytest.fixture
@@ -38,17 +88,39 @@ class ReadWriteTestSuite(BaseStandardTests):
@staticmethod
def get_embeddings() -> Embeddings:
- """A pre-defined embeddings model that should be used for this test."""
+ """A pre-defined embeddings model that should be used for this test.
+
+ This currently uses ``DeterministicFakeEmbedding`` from ``langchain-core``,
+ which uses numpy to generate random numbers based on a hash of the input text.
+
+ The resulting embeddings are not meaningful, but they are deterministic.
+ """
return DeterministicFakeEmbedding(
size=EMBEDDING_SIZE,
)
def test_vectorstore_is_empty(self, vectorstore: VectorStore) -> None:
- """Test that the vectorstore is empty."""
+ """Test that the vectorstore is empty.
+
+ .. dropdown:: Troubleshooting
+
+ If this test fails, check that the test class (i.e., sub class of
+ ``ReadWriteTestSuite``) initializes an empty vector store in the
+ ``vectorestore`` fixture.
+ """
assert vectorstore.similarity_search("foo", k=1) == []
def test_add_documents(self, vectorstore: VectorStore) -> None:
- """Test adding documents into the vectorstore."""
+ """Test adding documents into the vectorstore.
+
+ .. dropdown:: Troubleshooting
+
+ If this test fails, check that:
+
+ 1. We correctly initialize an empty vector store in the ``vectorestore`` fixture.
+ 2. Calling ``.similarity_search`` for the top ``k`` similar documents does not threshold by score.
+ 3. We do not mutate the original document object when adding it to the vector store (e.g., by adding an ID).
+ """ # noqa: E501
original_documents = [
Document(page_content="foo", metadata={"id": 1}),
Document(page_content="bar", metadata={"id": 2}),
@@ -71,11 +143,24 @@ class ReadWriteTestSuite(BaseStandardTests):
This just verifies that the fixture is set up properly to be empty
after each test.
+
+ .. dropdown:: Troubleshooting
+
+ If this test fails, check that the test class (i.e., sub class of
+ ``ReadWriteTestSuite``) correctly clears the vector store in the
+ ``finally`` block.
"""
assert vectorstore.similarity_search("foo", k=1) == []
def test_deleting_documents(self, vectorstore: VectorStore) -> None:
- """Test deleting documents from the vectorstore."""
+ """Test deleting documents from the vectorstore.
+
+ .. dropdown:: Troubleshooting
+
+ If this test fails, check that ``add_documents`` preserves identifiers
+ passed in through ``ids``, and that ``delete`` correctly removes
+ documents.
+ """
documents = [
Document(page_content="foo", metadata={"id": 1}),
Document(page_content="bar", metadata={"id": 2}),
@@ -87,7 +172,13 @@ class ReadWriteTestSuite(BaseStandardTests):
assert documents == [Document(page_content="bar", metadata={"id": 2}, id="2")]
def test_deleting_bulk_documents(self, vectorstore: VectorStore) -> None:
- """Test that we can delete several documents at once."""
+ """Test that we can delete several documents at once.
+
+ .. dropdown:: Troubleshooting
+
+ If this test fails, check that ``delete`` correctly removes multiple
+ documents when given a list of IDs.
+ """
documents = [
Document(page_content="foo", metadata={"id": 1}),
Document(page_content="bar", metadata={"id": 2}),
@@ -100,14 +191,27 @@ class ReadWriteTestSuite(BaseStandardTests):
assert documents == [Document(page_content="baz", metadata={"id": 3}, id="3")]
def test_delete_missing_content(self, vectorstore: VectorStore) -> None:
- """Deleting missing content should not raise an exception."""
+ """Deleting missing content should not raise an exception.
+
+ .. dropdown:: Troubleshooting
+
+ If this test fails, check that ``delete`` does not raise an exception
+ when deleting IDs that do not exist.
+ """
vectorstore.delete(["1"])
vectorstore.delete(["1", "2", "3"])
def test_add_documents_with_ids_is_idempotent(
self, vectorstore: VectorStore
) -> None:
- """Adding by ID should be idempotent."""
+ """Adding by ID should be idempotent.
+
+ .. dropdown:: Troubleshooting
+
+ If this test fails, check that adding the same document twice with the
+ same IDs has the same effect as adding it once (i.e., it does not
+ duplicate the documents).
+ """
documents = [
Document(page_content="foo", metadata={"id": 1}),
Document(page_content="bar", metadata={"id": 2}),
@@ -121,7 +225,14 @@ class ReadWriteTestSuite(BaseStandardTests):
]
def test_add_documents_by_id_with_mutation(self, vectorstore: VectorStore) -> None:
- """Test that we can overwrite by ID using add_documents."""
+ """Test that we can overwrite by ID using add_documents.
+
+ .. dropdown:: Troubleshooting
+
+ If this test fails, check that when ``add_documents`` is called with an
+ ID that already exists in the vector store, the content is updated
+ rather than duplicated.
+ """
documents = [
Document(page_content="foo", metadata={"id": 1}),
Document(page_content="bar", metadata={"id": 2}),
@@ -150,7 +261,26 @@ class ReadWriteTestSuite(BaseStandardTests):
]
def test_get_by_ids(self, vectorstore: VectorStore) -> None:
- """Test get by IDs."""
+ """Test get by IDs.
+
+ This test requires that ``get_by_ids`` be implemented on the vector store.
+
+ .. dropdown:: Troubleshooting
+
+ If this test fails, check that ``get_by_ids`` is implemented and returns
+ documents in the same order as the IDs passed in.
+
+ .. note::
+ ``get_by_ids`` was added to the ``VectorStore`` interface in
+ ``langchain-core`` version 0.2.11. If difficult to implement, this
+ test can be skipped using a pytest ``xfail`` on the test class:
+
+ .. code-block:: python
+
+ @pytest.mark.xfail(reason=("get_by_ids not implemented."))
+ def test_get_by_ids(self, vectorstore: VectorStore) -> None:
+ super().test_get_by_ids(vectorstore)
+ """
documents = [
Document(page_content="foo", metadata={"id": 1}),
Document(page_content="bar", metadata={"id": 2}),
@@ -163,13 +293,50 @@ class ReadWriteTestSuite(BaseStandardTests):
]
def test_get_by_ids_missing(self, vectorstore: VectorStore) -> None:
- """Test get by IDs with missing IDs."""
+ """Test get by IDs with missing IDs.
+
+ .. dropdown:: Troubleshooting
+
+ If this test fails, check that ``get_by_ids`` is implemented and does not
+ raise an exception when given IDs that do not exist.
+
+ .. note::
+ ``get_by_ids`` was added to the ``VectorStore`` interface in
+ ``langchain-core`` version 0.2.11. If difficult to implement, this
+ test can be skipped using a pytest ``xfail`` on the test class:
+
+ .. code-block:: python
+
+ @pytest.mark.xfail(reason=("get_by_ids not implemented."))
+ def test_get_by_ids_missing(self, vectorstore: VectorStore) -> None:
+ super().test_get_by_ids_missing(vectorstore)
+ """ # noqa: E501
# This should not raise an exception
documents = vectorstore.get_by_ids(["1", "2", "3"])
assert documents == []
def test_add_documents_documents(self, vectorstore: VectorStore) -> None:
- """Run add_documents tests."""
+ """Run add_documents tests.
+
+ .. dropdown:: Troubleshooting
+
+ If this test fails, check that ``get_by_ids`` is implemented and returns
+ documents in the same order as the IDs passed in.
+
+ Check also that ``add_documents`` will correctly generate string IDs if
+ none are provided.
+
+ .. note::
+ ``get_by_ids`` was added to the ``VectorStore`` interface in
+ ``langchain-core`` version 0.2.11. If difficult to implement, this
+ test can be skipped using a pytest ``xfail`` on the test class:
+
+ .. code-block:: python
+
+ @pytest.mark.xfail(reason=("get_by_ids not implemented."))
+ def test_add_documents_documents(self, vectorstore: VectorStore) -> None:
+ super().test_add_documents_documents(vectorstore)
+ """ # noqa: E501
documents = [
Document(page_content="foo", metadata={"id": 1}),
Document(page_content="bar", metadata={"id": 2}),
@@ -181,7 +348,29 @@ class ReadWriteTestSuite(BaseStandardTests):
]
def test_add_documents_with_existing_ids(self, vectorstore: VectorStore) -> None:
- """Test that add_documentsing with existing IDs is idempotent."""
+ """Test that add_documents with existing IDs is idempotent.
+
+ .. dropdown:: Troubleshooting
+
+ If this test fails, check that ``get_by_ids`` is implemented and returns
+ documents in the same order as the IDs passed in.
+
+ This test also verifies that:
+
+ 1. IDs specified in the ``Document.id`` field are assigned when adding documents.
+ 2. If some documents include IDs and others don't string IDs are generated for the latter.
+
+ .. note::
+ ``get_by_ids`` was added to the ``VectorStore`` interface in
+ ``langchain-core`` version 0.2.11. If difficult to implement, this
+ test can be skipped using a pytest ``xfail`` on the test class:
+
+ .. code-block:: python
+
+ @pytest.mark.xfail(reason=("get_by_ids not implemented."))
+ def test_add_documents_with_existing_ids(self, vectorstore: VectorStore) -> None:
+ super().test_add_documents_with_existing_ids(vectorstore)
+ """ # noqa: E501
documents = [
Document(id="foo", page_content="foo", metadata={"id": 1}),
Document(page_content="bar", metadata={"id": 2}),
@@ -195,18 +384,68 @@ class ReadWriteTestSuite(BaseStandardTests):
class AsyncReadWriteTestSuite(BaseStandardTests):
- """Test suite for checking the **async** read-write API of a vectorstore.
-
- This test suite verifies the basic read-write API of a vectorstore.
-
- The test suite is designed for asynchronous vectorstores.
+ """Test suite for checking the async read-write API of a vector store.
Implementers should subclass this test suite and provide a fixture
- that returns an empty vectorstore for each test.
+ that returns an empty vector store for each test.
- The fixture should use the `get_embeddings` method to get a pre-defined
+ The fixture should use the ``get_embeddings`` method to get a pre-defined
embeddings model that should be used for this test suite.
- """
+
+ Here is a template:
+
+ .. code-block:: python
+
+ from typing import AsyncGenerator
+
+ import pytest
+ from langchain_core.vectorstores import VectorStore
+ from langchain_parrot_link.vectorstores import ParrotVectorStore
+ from langchain_tests.integration_tests.vectorstores import AsyncReadWriteTestSuite
+
+
+ class TestAsync(AsyncReadWriteTestSuite):
+ @pytest.fixture()
+ def vectorstore(self) -> AsyncGenerator[VectorStore, None]: # type: ignore
+ \"\"\"Get an empty vectorstore.\"\"\"
+ store = ParrotVectorStore(self.get_embeddings())
+ # note: store should be EMPTY at this point
+ # if you need to delete data, you may do so here
+ try:
+ yield store
+ finally:
+ # cleanup operations, or deleting data
+ pass
+
+ In the fixture, before the ``yield`` we instantiate an empty vector store. In the
+ ``finally`` block, we call whatever logic is necessary to bring the vector store
+ to a clean state.
+
+ Example:
+
+ .. code-block:: python
+
+ from typing import AsyncGenerator, Generator
+
+ import pytest
+ from langchain_core.vectorstores import VectorStore
+ from langchain_tests.integration_tests.vectorstores import AsyncReadWriteTestSuite
+
+ from langchain_chroma import Chroma
+
+
+ class TestAsync(AsyncReadWriteTestSuite):
+ @pytest.fixture()
+ async def vectorstore(self) -> AsyncGenerator[VectorStore, None]: # type: ignore
+ \"\"\"Get an empty vectorstore for unit tests.\"\"\"
+ store = Chroma(embedding_function=self.get_embeddings())
+ try:
+ yield store
+ finally:
+ store.delete_collection()
+ pass
+
+ """ # noqa: E501
@abstractmethod
@pytest.fixture
@@ -218,17 +457,39 @@ class AsyncReadWriteTestSuite(BaseStandardTests):
@staticmethod
def get_embeddings() -> Embeddings:
- """A pre-defined embeddings model that should be used for this test."""
+ """A pre-defined embeddings model that should be used for this test.
+
+ This currently uses ``DeterministicFakeEmbedding`` from ``langchain-core``,
+ which uses numpy to generate random numbers based on a hash of the input text.
+
+ The resulting embeddings are not meaningful, but they are deterministic.
+ """
return DeterministicFakeEmbedding(
size=EMBEDDING_SIZE,
)
async def test_vectorstore_is_empty(self, vectorstore: VectorStore) -> None:
- """Test that the vectorstore is empty."""
+ """Test that the vectorstore is empty.
+
+ .. dropdown:: Troubleshooting
+
+ If this test fails, check that the test class (i.e., sub class of
+ ``AsyncReadWriteTestSuite``) initializes an empty vector store in the
+ ``vectorestore`` fixture.
+ """
assert await vectorstore.asimilarity_search("foo", k=1) == []
async def test_add_documents(self, vectorstore: VectorStore) -> None:
- """Test adding documents into the vectorstore."""
+ """Test adding documents into the vectorstore.
+
+ .. dropdown:: Troubleshooting
+
+ If this test fails, check that:
+
+ 1. We correctly initialize an empty vector store in the ``vectorestore`` fixture.
+ 2. Calling ``.asimilarity_search`` for the top ``k`` similar documents does not threshold by score.
+ 3. We do not mutate the original document object when adding it to the vector store (e.g., by adding an ID).
+ """ # noqa: E501
original_documents = [
Document(page_content="foo", metadata={"id": 1}),
Document(page_content="bar", metadata={"id": 2}),
@@ -252,11 +513,24 @@ class AsyncReadWriteTestSuite(BaseStandardTests):
This just verifies that the fixture is set up properly to be empty
after each test.
+
+ .. dropdown:: Troubleshooting
+
+ If this test fails, check that the test class (i.e., sub class of
+ ``AsyncReadWriteTestSuite``) correctly clears the vector store in the
+ ``finally`` block.
"""
assert await vectorstore.asimilarity_search("foo", k=1) == []
async def test_deleting_documents(self, vectorstore: VectorStore) -> None:
- """Test deleting documents from the vectorstore."""
+ """Test deleting documents from the vectorstore.
+
+ .. dropdown:: Troubleshooting
+
+ If this test fails, check that ``aadd_documents`` preserves identifiers
+ passed in through ``ids``, and that ``delete`` correctly removes
+ documents.
+ """
documents = [
Document(page_content="foo", metadata={"id": 1}),
Document(page_content="bar", metadata={"id": 2}),
@@ -268,7 +542,13 @@ class AsyncReadWriteTestSuite(BaseStandardTests):
assert documents == [Document(page_content="bar", metadata={"id": 2}, id="2")]
async def test_deleting_bulk_documents(self, vectorstore: VectorStore) -> None:
- """Test that we can delete several documents at once."""
+ """Test that we can delete several documents at once.
+
+ .. dropdown:: Troubleshooting
+
+ If this test fails, check that ``adelete`` correctly removes multiple
+ documents when given a list of IDs.
+ """
documents = [
Document(page_content="foo", metadata={"id": 1}),
Document(page_content="bar", metadata={"id": 2}),
@@ -281,14 +561,27 @@ class AsyncReadWriteTestSuite(BaseStandardTests):
assert documents == [Document(page_content="baz", metadata={"id": 3}, id="3")]
async def test_delete_missing_content(self, vectorstore: VectorStore) -> None:
- """Deleting missing content should not raise an exception."""
+ """Deleting missing content should not raise an exception.
+
+ .. dropdown:: Troubleshooting
+
+ If this test fails, check that ``adelete`` does not raise an exception
+ when deleting IDs that do not exist.
+ """
await vectorstore.adelete(["1"])
await vectorstore.adelete(["1", "2", "3"])
async def test_add_documents_with_ids_is_idempotent(
self, vectorstore: VectorStore
) -> None:
- """Adding by ID should be idempotent."""
+ """Adding by ID should be idempotent.
+
+ .. dropdown:: Troubleshooting
+
+ If this test fails, check that adding the same document twice with the
+ same IDs has the same effect as adding it once (i.e., it does not
+ duplicate the documents).
+ """
documents = [
Document(page_content="foo", metadata={"id": 1}),
Document(page_content="bar", metadata={"id": 2}),
@@ -304,7 +597,14 @@ class AsyncReadWriteTestSuite(BaseStandardTests):
async def test_add_documents_by_id_with_mutation(
self, vectorstore: VectorStore
) -> None:
- """Test that we can overwrite by ID using add_documents."""
+ """Test that we can overwrite by ID using add_documents.
+
+ .. dropdown:: Troubleshooting
+
+ If this test fails, check that when ``aadd_documents`` is called with an
+ ID that already exists in the vector store, the content is updated
+ rather than duplicated.
+ """
documents = [
Document(page_content="foo", metadata={"id": 1}),
Document(page_content="bar", metadata={"id": 2}),
@@ -333,7 +633,26 @@ class AsyncReadWriteTestSuite(BaseStandardTests):
]
async def test_get_by_ids(self, vectorstore: VectorStore) -> None:
- """Test get by IDs."""
+ """Test get by IDs.
+
+ This test requires that ``get_by_ids`` be implemented on the vector store.
+
+ .. dropdown:: Troubleshooting
+
+ If this test fails, check that ``get_by_ids`` is implemented and returns
+ documents in the same order as the IDs passed in.
+
+ .. note::
+ ``get_by_ids`` was added to the ``VectorStore`` interface in
+ ``langchain-core`` version 0.2.11. If difficult to implement, this
+ test can be skipped using a pytest ``xfail`` on the test class:
+
+ .. code-block:: python
+
+ @pytest.mark.xfail(reason=("get_by_ids not implemented."))
+ async def test_get_by_ids(self, vectorstore: VectorStore) -> None:
+ await super().test_get_by_ids(vectorstore)
+ """
documents = [
Document(page_content="foo", metadata={"id": 1}),
Document(page_content="bar", metadata={"id": 2}),
@@ -346,12 +665,49 @@ class AsyncReadWriteTestSuite(BaseStandardTests):
]
async def test_get_by_ids_missing(self, vectorstore: VectorStore) -> None:
- """Test get by IDs with missing IDs."""
+ """Test get by IDs with missing IDs.
+
+ .. dropdown:: Troubleshooting
+
+ If this test fails, check that ``get_by_ids`` is implemented and does not
+ raise an exception when given IDs that do not exist.
+
+ .. note::
+ ``get_by_ids`` was added to the ``VectorStore`` interface in
+ ``langchain-core`` version 0.2.11. If difficult to implement, this
+ test can be skipped using a pytest ``xfail`` on the test class:
+
+ .. code-block:: python
+
+ @pytest.mark.xfail(reason=("get_by_ids not implemented."))
+ async def test_get_by_ids_missing(self, vectorstore: VectorStore) -> None:
+ await super().test_get_by_ids_missing(vectorstore)
+ """ # noqa: E501
# This should not raise an exception
assert await vectorstore.aget_by_ids(["1", "2", "3"]) == []
async def test_add_documents_documents(self, vectorstore: VectorStore) -> None:
- """Run add_documents tests."""
+ """Run add_documents tests.
+
+ .. dropdown:: Troubleshooting
+
+ If this test fails, check that ``get_by_ids`` is implemented and returns
+ documents in the same order as the IDs passed in.
+
+ Check also that ``aadd_documents`` will correctly generate string IDs if
+ none are provided.
+
+ .. note::
+ ``get_by_ids`` was added to the ``VectorStore`` interface in
+ ``langchain-core`` version 0.2.11. If difficult to implement, this
+ test can be skipped using a pytest ``xfail`` on the test class:
+
+ .. code-block:: python
+
+ @pytest.mark.xfail(reason=("get_by_ids not implemented."))
+ async def test_add_documents_documents(self, vectorstore: VectorStore) -> None:
+ await super().test_add_documents_documents(vectorstore)
+ """ # noqa: E501
documents = [
Document(page_content="foo", metadata={"id": 1}),
Document(page_content="bar", metadata={"id": 2}),
@@ -365,7 +721,29 @@ class AsyncReadWriteTestSuite(BaseStandardTests):
async def test_add_documents_with_existing_ids(
self, vectorstore: VectorStore
) -> None:
- """Test that add_documentsing with existing IDs is idempotent."""
+ """Test that add_documents with existing IDs is idempotent.
+
+ .. dropdown:: Troubleshooting
+
+ If this test fails, check that ``get_by_ids`` is implemented and returns
+ documents in the same order as the IDs passed in.
+
+ This test also verifies that:
+
+ 1. IDs specified in the ``Document.id`` field are assigned when adding documents.
+ 2. If some documents include IDs and others don't string IDs are generated for the latter.
+
+ .. note::
+ ``get_by_ids`` was added to the ``VectorStore`` interface in
+ ``langchain-core`` version 0.2.11. If difficult to implement, this
+ test can be skipped using a pytest ``xfail`` on the test class:
+
+ .. code-block:: python
+
+ @pytest.mark.xfail(reason=("get_by_ids not implemented."))
+ async def test_add_documents_with_existing_ids(self, vectorstore: VectorStore) -> None:
+ await super().test_add_documents_with_existing_ids(vectorstore)
+ """ # noqa: E501
documents = [
Document(id="foo", page_content="foo", metadata={"id": 1}),
Document(page_content="bar", metadata={"id": 2}),