The agent engineering platform.
Python agentsaiai-agentsanthropicchatgptdeepagentsenterpriseframeworkgeminigenerative-ailangchainlanggraphllmmultiagentopen-sourceopenaipydanticpythonragtypescript
langchain: add id_key option to EnsembleRetriever for metadata-based document merging (#22950)
shimajiroxyz3e835a1aa115
diff --git a/libs/langchain/langchain/retrievers/ensemble.py b/libs/langchain/langchain/retrievers/ensemble.pyindex 61f2bfdcb1..0eb779544d 100644--- a/libs/langchain/langchain/retrievers/ensemble.py+++ b/libs/langchain/langchain/retrievers/ensemble.py@@ -66,11 +66,14 @@ class EnsembleRetriever(BaseRetriever):c: A constant added to the rank, controlling the balance between the importanceof high-ranked items and the consideration given to lower-ranked items.Default is 60.+ id_key: The key in the document's metadata used to determine unique documents.+ If not specified, page_content is used."""retrievers: List[RetrieverLike]weights: List[float]c: int = 60+ id_key: Optional[str] = None@propertydef config_specs(self) -> List[ConfigurableFieldSpec]:@@ -305,13 +308,24 @@ class EnsembleRetriever(BaseRetriever):rrf_score: Dict[str, float] = defaultdict(float)for doc_list, weight in zip(doc_lists, self.weights):for rank, doc in enumerate(doc_list, start=1):- rrf_score[doc.page_content] += weight / (rank + self.c)+ rrf_score[+ doc.page_content+ if self.id_key is None+ else doc.metadata[self.id_key]+ ] += weight / (rank + self.c)# Docs are deduplicated by their contents then sorted by their scoresall_docs = chain.from_iterable(doc_lists)sorted_docs = sorted(- unique_by_key(all_docs, lambda doc: doc.page_content),+ unique_by_key(+ all_docs,+ lambda doc: doc.page_content+ if self.id_key is None+ else doc.metadata[self.id_key],+ ),reverse=True,- key=lambda doc: rrf_score[doc.page_content],+ key=lambda doc: rrf_score[+ doc.page_content if self.id_key is None else doc.metadata[self.id_key]+ ],)return sorted_docsdiff --git a/libs/langchain/tests/unit_tests/retrievers/test_ensemble.py b/libs/langchain/tests/unit_tests/retrievers/test_ensemble.pynew file mode 100644index 0000000000..4c5e9837c0--- /dev/null+++ b/libs/langchain/tests/unit_tests/retrievers/test_ensemble.py@@ -0,0 +1,88 @@+from typing import List, Optional++from langchain_core.callbacks.manager import CallbackManagerForRetrieverRun+from langchain_core.documents import Document+from langchain_core.retrievers import BaseRetriever++from langchain.retrievers.ensemble import EnsembleRetriever+++class MockRetriever(BaseRetriever):+ docs: List[Document]++ def _get_relevant_documents(+ self,+ query: str,+ *,+ run_manager: Optional[CallbackManagerForRetrieverRun] = None,+ ) -> List[Document]:+ """Return the documents"""+ return self.docs+++def test_invoke() -> None:+ documents1 = [+ Document(page_content="a", metadata={"id": 1}),+ Document(page_content="b", metadata={"id": 2}),+ Document(page_content="c", metadata={"id": 3}),+ ]+ documents2 = [Document(page_content="b")]++ retriever1 = MockRetriever(docs=documents1)+ retriever2 = MockRetriever(docs=documents2)++ ensemble_retriever = EnsembleRetriever(+ retrievers=[retriever1, retriever2], weights=[0.5, 0.5], id_key=None+ )+ ranked_documents = ensemble_retriever.invoke("_")++ # The document with page_content "b" in documents2+ # will be merged with the document with page_content "b"+ # in documents1, so the length of ranked_documents should be 3.+ # Additionally, the document with page_content "b" will be ranked 1st.+ assert len(ranked_documents) == 3+ assert ranked_documents[0].page_content == "b"++ documents1 = [+ Document(page_content="a", metadata={"id": 1}),+ Document(page_content="b", metadata={"id": 2}),+ Document(page_content="c", metadata={"id": 3}),+ ]+ documents2 = [Document(page_content="d")]++ retriever1 = MockRetriever(docs=documents1)+ retriever2 = MockRetriever(docs=documents2)++ ensemble_retriever = EnsembleRetriever(+ retrievers=[retriever1, retriever2], weights=[0.5, 0.5], id_key=None+ )+ ranked_documents = ensemble_retriever.invoke("_")++ # The document with page_content "d" in documents2 will not be merged+ # with any document in documents1, so the length of ranked_documents+ # should be 4. The document with page_content "a" and the document+ # with page_content "d" will have the same score, but the document+ # with page_content "a" will be ranked 1st because retriever1 has a smaller index.+ assert len(ranked_documents) == 4+ assert ranked_documents[0].page_content == "a"++ documents1 = [+ Document(page_content="a", metadata={"id": 1}),+ Document(page_content="b", metadata={"id": 2}),+ Document(page_content="c", metadata={"id": 3}),+ ]+ documents2 = [Document(page_content="d", metadata={"id": 2})]++ retriever1 = MockRetriever(docs=documents1)+ retriever2 = MockRetriever(docs=documents2)++ ensemble_retriever = EnsembleRetriever(+ retrievers=[retriever1, retriever2], weights=[0.5, 0.5], id_key="id"+ )+ ranked_documents = ensemble_retriever.invoke("_")++ # Since id_key is specified, the document with id 2 will be merged.+ # Therefore, the length of ranked_documents should be 3.+ # Additionally, the document with page_content "b" will be ranked 1st.+ assert len(ranked_documents) == 3+ assert ranked_documents[0].page_content == "b"