The agent engineering platform.
Python agentsaiai-agentsanthropicchatgptdeepagentsenterpriseframeworkgeminigenerative-ailangchainlanggraphllmmultiagentopen-sourceopenaipydanticpythonragtypescript
core[patch]: Reverts PR #25754 and add unit tests (#28702)
Keiichi Hirobeda28cf1f54c1
diff --git a/libs/core/langchain_core/indexing/api.py b/libs/core/langchain_core/indexing/api.pyindex c310deb241..75efe1f565 100644--- a/libs/core/langchain_core/indexing/api.py+++ b/libs/core/langchain_core/indexing/api.py@@ -408,18 +408,17 @@ def index(# mypy isn't good enough to determine that source ids cannot be None# here due to a check that's happening above, so we check again.- if any(source_id is None for source_id in source_ids):- msg = "Source ids cannot be if cleanup=='incremental'."- raise AssertionError(msg)+ for source_id in source_ids:+ if source_id is None:+ msg = "Source ids cannot be None here."+ raise AssertionError(msg)- indexed_source_ids = cast(- Sequence[str], [source_id_assigner(doc) for doc in docs_to_index]- )+ _source_ids = cast(Sequence[str], source_ids)uids_to_delete = record_manager.list_keys(- group_ids=indexed_source_ids, before=index_start_dt+ group_ids=_source_ids, before=index_start_dt)- if indexed_source_ids and uids_to_delete:+ if uids_to_delete:# Then delete from vector store.destination.delete(uids_to_delete)# First delete from record store.@@ -669,18 +668,17 @@ async def aindex(# mypy isn't good enough to determine that source ids cannot be None# here due to a check that's happening above, so we check again.- if any(source_id is None for source_id in source_ids):- msg = "Source ids cannot be if cleanup=='incremental'."- raise AssertionError(msg)+ for source_id in source_ids:+ if source_id is None:+ msg = "Source ids cannot be None here."+ raise AssertionError(msg)- indexed_source_ids = cast(- Sequence[str], [source_id_assigner(doc) for doc in docs_to_index]- )+ _source_ids = cast(Sequence[str], source_ids)uids_to_delete = await record_manager.alist_keys(- group_ids=indexed_source_ids, before=index_start_dt+ group_ids=_source_ids, before=index_start_dt)- if indexed_source_ids and uids_to_delete:+ if uids_to_delete:# Then delete from vector store.await destination.adelete(uids_to_delete)# First delete from record store.diff --git a/libs/core/tests/unit_tests/indexing/test_indexing.py b/libs/core/tests/unit_tests/indexing/test_indexing.pyindex 287b6b49f6..4b5874b85b 100644--- a/libs/core/tests/unit_tests/indexing/test_indexing.py+++ b/libs/core/tests/unit_tests/indexing/test_indexing.py@@ -544,7 +544,7 @@ def test_incremental_delete()with patch.object(- record_manager, "get_time", return_value=datetime(2021, 1, 2).timestamp()+ record_manager, "get_time", return_value=datetime(2021, 1, 1).timestamp()):assert index(loader,@@ -630,26 +630,18 @@ def test_incremental_delete(}-def test_incremental_indexing_with_batch_size(+def test_incremental_delete_with_same_source(record_manager: InMemoryRecordManager, vector_store: InMemoryVectorStore) -> None:- """Test indexing with incremental indexing"""+ """Test indexing with incremental deletion strategy."""loader = ToyLoader(documents=[Document(- page_content="1",- metadata={"source": "1"},- ),- Document(- page_content="2",- metadata={"source": "1"},- ),- Document(- page_content="3",+ page_content="This is a test document.",metadata={"source": "1"},),Document(- page_content="4",+ page_content="This is another document.",metadata={"source": "1"},),]@@ -664,9 +656,8 @@ def test_incremental_indexing_with_batch_size(vector_store,cleanup="incremental",source_id_key="source",- batch_size=2,) == {- "num_added": 4,+ "num_added": 2,"num_deleted": 0,"num_skipped": 0,"num_updated": 0,@@ -677,7 +668,17 @@ def test_incremental_indexing_with_batch_size(vector_store.get_by_ids([uid])[0].page_content # type: ignorefor uid in vector_store.store}- assert doc_texts == {"1", "2", "3", "4"}+ assert doc_texts == {"This is another document.", "This is a test document."}++ # Delete 1 document and unchange 1 document+ loader = ToyLoader(+ documents=[+ Document(+ page_content="This is another document.", # <-- Same as original+ metadata={"source": "1"},+ ),+ ]+ )with patch.object(record_manager, "get_time", return_value=datetime(2021, 1, 2).timestamp()@@ -688,11 +689,10 @@ def test_incremental_indexing_with_batch_size(vector_store,cleanup="incremental",source_id_key="source",- batch_size=2,) == {"num_added": 0,- "num_deleted": 0,- "num_skipped": 4,+ "num_deleted": 1,+ "num_skipped": 1,"num_updated": 0,}@@ -701,40 +701,41 @@ def test_incremental_indexing_with_batch_size(vector_store.get_by_ids([uid])[0].page_content # type: ignorefor uid in vector_store.store}- assert doc_texts == {"1", "2", "3", "4"}+ assert doc_texts == {+ "This is another document.",+ }-def test_incremental_indexing_with_batch_size_with_optimization(+def test_incremental_indexing_with_batch_size(record_manager: InMemoryRecordManager, vector_store: InMemoryVectorStore) -> None:- """Test case when batch_size < num of docs.-- Here, we'll verify that an indexing optimization works as expected.- """- documents = [- Document(- page_content="1",- metadata={"source": "1"},- ),- Document(- page_content="2",- metadata={"source": "1"},- ),- Document(- page_content="3",- metadata={"source": "1"},- ),- Document(- page_content="4",- metadata={"source": "1"},- ),- ]+ """Test indexing with incremental indexing"""+ loader = ToyLoader(+ documents=[+ Document(+ page_content="1",+ metadata={"source": "1"},+ ),+ Document(+ page_content="2",+ metadata={"source": "1"},+ ),+ Document(+ page_content="3",+ metadata={"source": "1"},+ ),+ Document(+ page_content="4",+ metadata={"source": "1"},+ ),+ ]+ )with patch.object(record_manager, "get_time", return_value=datetime(2021, 1, 1).timestamp()):assert index(- documents,+ loader,record_manager,vector_store,cleanup="incremental",@@ -754,88 +755,20 @@ def test_incremental_indexing_with_batch_size_with_optimization(}assert doc_texts == {"1", "2", "3", "4"}- # Mutate content in first batch- doc_first_batch_mutation = [- Document(- page_content="updated 1",- metadata={"source": "1"},- ),- Document(- page_content="2",- metadata={"source": "1"},- ),- Document(- page_content="3",- metadata={"source": "1"},- ),- Document(- page_content="4",- metadata={"source": "1"},- ),- ]-with patch.object(record_manager, "get_time", return_value=datetime(2021, 1, 2).timestamp()):assert index(- doc_first_batch_mutation,- record_manager,- vector_store,- cleanup="incremental",- source_id_key="source",- batch_size=2,- ) == {- # Cannot optimize here since the first batch was changed- # So only skpping indexing the document with content "2"- "num_added": 3,- "num_deleted": 3,- "num_skipped": 1,- "num_updated": 0,- }-- doc_texts = {- # Ignoring type since doc should be in the store and not a None- vector_store.get_by_ids([uid])[0].page_content # type: ignore- for uid in vector_store.store- }- assert doc_texts == {"updated 1", "2", "3", "4"}-- # Mutate content in second batch- doc_second_batch_mutation = [- Document(- page_content="updated 1", # <-- This was already previously updated- metadata={"source": "1"},- ),- Document(- page_content="2",- metadata={"source": "1"},- ),- Document(- page_content="3",- metadata={"source": "1"},- ),- Document(- page_content="updated 4",- metadata={"source": "1"},- ),- ]-- with patch.object(- record_manager, "get_time", return_value=datetime(2021, 1, 3).timestamp()- ):- assert index(- doc_second_batch_mutation,+ loader,record_manager,vector_store,cleanup="incremental",source_id_key="source",batch_size=2,) == {- # Skips updating content from the first batch, only updates- # the `updated 4` content- "num_added": 1,- "num_deleted": 1,- "num_skipped": 3,+ "num_added": 2,+ "num_deleted": 2,+ "num_skipped": 2,"num_updated": 0,}@@ -844,7 +777,7 @@ def test_incremental_indexing_with_batch_size_with_optimization(vector_store.get_by_ids([uid])[0].page_content # type: ignorefor uid in vector_store.store}- assert doc_texts == {"updated 1", "2", "3", "updated 4"}+ assert doc_texts == {"1", "2", "3", "4"}def test_incremental_delete_with_batch_size(@@ -1588,149 +1521,6 @@ async def test_aindex_into_document_index(}-async def test_incremental_aindexing_with_batch_size_with_optimization(- arecord_manager: InMemoryRecordManager, vector_store: InMemoryVectorStore-) -> None:- """Test case when batch_size < num of docs.-- Here, we'll verify that an indexing optimization works as expected.- """- documents = [- Document(- page_content="1",- metadata={"source": "1"},- ),- Document(- page_content="2",- metadata={"source": "1"},- ),- Document(- page_content="3",- metadata={"source": "1"},- ),- Document(- page_content="4",- metadata={"source": "1"},- ),- ]-- with patch.object(- arecord_manager, "get_time", return_value=datetime(2021, 1, 1).timestamp()- ):- assert await aindex(- documents,- arecord_manager,- vector_store,- cleanup="incremental",- source_id_key="source",- batch_size=2,- ) == {- "num_added": 4,- "num_deleted": 0,- "num_skipped": 0,- "num_updated": 0,- }-- doc_texts = {- # Ignoring type since doc should be in the store and not a None- vector_store.get_by_ids([uid])[0].page_content # type: ignore- for uid in vector_store.store- }- assert doc_texts == {"1", "2", "3", "4"}-- # Mutate content in first batch- doc_first_batch_mutation = [- Document(- page_content="updated 1",- metadata={"source": "1"},- ),- Document(- page_content="2",- metadata={"source": "1"},- ),- Document(- page_content="3",- metadata={"source": "1"},- ),- Document(- page_content="4",- metadata={"source": "1"},- ),- ]-- with patch.object(- arecord_manager, "get_time", return_value=datetime(2021, 1, 2).timestamp()- ):- assert await aindex(- doc_first_batch_mutation,- arecord_manager,- vector_store,- cleanup="incremental",- source_id_key="source",- batch_size=2,- ) == {- # Cannot optimize here since the first batch was changed- # So only skpping indexing the document with content "2"- "num_added": 3,- "num_deleted": 3,- "num_skipped": 1,- "num_updated": 0,- }-- doc_texts = {- # Ignoring type since doc should be in the store and not a None- vector_store.get_by_ids([uid])[0].page_content # type: ignore- for uid in vector_store.store- }- assert doc_texts == {"updated 1", "2", "3", "4"}-- # Mutate content in second batch- doc_second_batch_mutation = [- Document(- page_content="updated 1", # <-- This was already previously updated- metadata={"source": "1"},- ),- Document(- page_content="2",- metadata={"source": "1"},- ),- Document(- page_content="3",- metadata={"source": "1"},- ),- Document(- page_content="updated 4",- metadata={"source": "1"},- ),- ]-- with patch.object(- arecord_manager, "get_time", return_value=datetime(2021, 1, 3).timestamp()- ):- assert await aindex(- doc_second_batch_mutation,- arecord_manager,- vector_store,- cleanup="incremental",- source_id_key="source",- batch_size=2,- ) == {- # Skips updating content from the first batch, only updates- # the `updated 4` content- "num_added": 1,- "num_deleted": 1,- "num_skipped": 3,- "num_updated": 0,- }-- doc_texts = {- # Ignoring type since doc should be in the store and not a None- vector_store.get_by_ids([uid])[0].page_content # type: ignore- for uid in vector_store.store- }- assert doc_texts == {"updated 1", "2", "3", "updated 4"}--def test_index_with_upsert_kwargs(record_manager: InMemoryRecordManager, upserting_vector_store: InMemoryVectorStore) -> None: