RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(api): return document metadata from /dify/retrieval (#19686)
Ehsanc71bb0198f2f
diff --git a/api/apps/restful_apis/dify_retrieval_api.py b/api/apps/restful_apis/dify_retrieval_api.pyindex 0640ff799c..68ebd16098 100644--- a/api/apps/restful_apis/dify_retrieval_api.py+++ b/api/apps/restful_apis/dify_retrieval_api.py@@ -291,6 +291,7 @@ async def retrieval(tenant_id):doc_ids = list(set([c["doc_id"] for c in ranks["chunks"]]))docs = DocumentService.get_by_ids(doc_ids)doc_map = {doc.id: doc for doc in docs}+ metadata_map = DocMetadataService.get_metadata_for_documents(doc_ids, kb_id) if doc_ids else {}records = []for c in ranks["chunks"]:@@ -298,7 +299,8 @@ async def retrieval(tenant_id):if not doc:continuec.pop("vector", None)- meta = getattr(doc, "meta_fields", {})+ # Copied because several chunks of one document share a map entry.+ meta = dict(metadata_map.get(c["doc_id"]) or {})meta["doc_id"] = c["doc_id"]# Dify expects metadata.document_id for external retrieval sources.meta["document_id"] = c["doc_id"]diff --git a/test/unit_test/api/apps/sdk/test_dify_retrieval.py b/test/unit_test/api/apps/sdk/test_dify_retrieval.pyindex f394a45ea4..80175ec3e7 100644--- a/test/unit_test/api/apps/sdk/test_dify_retrieval.py+++ b/test/unit_test/api/apps/sdk/test_dify_retrieval.py@@ -113,6 +113,7 @@ def _load_dify_retrieval(request_args=None,kg_content="",meta_filter_ids=None,+ doc_metadata=None,):"""Load dify_retrieval_api.py with minimum stubs to exercise the retrieval handler."""@@ -142,14 +143,20 @@ def _load_dify_retrieval(monkeypatch,"api.db.services.document_service",DocumentService=SimpleNamespace(- get_by_id=lambda _id: (True, SimpleNamespace(id=_id, meta_fields={})),- get_by_ids=lambda ids, cols=None: [SimpleNamespace(id=doc_id, meta_fields={}) for doc_id in ids],+ get_by_id=lambda _id: (True, SimpleNamespace(id=_id)),+ get_by_ids=lambda ids, cols=None: [SimpleNamespace(id=doc_id) for doc_id in ids],),)+ # Keyed by doc_id, and the same dict object is handed back on every lookup,+ # which is what DocMetadataService.get_metadata_for_documents does.+ doc_metadata = doc_metadata or {}_stub(monkeypatch,"api.db.services.doc_metadata_service",- DocMetadataService=SimpleNamespace(get_flatted_meta_by_kbs=lambda _ids: {}),+ DocMetadataService=SimpleNamespace(+ get_flatted_meta_by_kbs=lambda _ids: {},+ get_metadata_for_documents=lambda ids, kb_id: {i: doc_metadata[i] for i in ids if i in doc_metadata},+ ),)acc_fn = accessible if callable(accessible) else (lambda *_a, **_k: accessible)@@ -626,6 +633,49 @@ class TestDifyRetrievalRetrievalBehavior:assert record["metadata"]["doc_id"] == "d1"assert record["metadata"]["document_id"] == "d1"+ @pytest.mark.p1+ def test_records_carry_the_document_metadata(self, monkeypatch):+ owner_kb = SimpleNamespace(id="kb-owner", tenant_id="tenant-owner", tenant_embd_id="", embd_id="bge")+ module = _load_dify_retrieval(+ monkeypatch,+ kb=(True, owner_kb),+ accessible=lambda _id, _u: True,+ request_body={"knowledge_id": "kb-owner", "query": "hello"},+ tenant_id="tenant-owner",+ chunks=[{"doc_id": "d1", "content_with_weight": "hello world", "similarity": 0.8, "docnm_kwd": "doc.txt"}],+ doc_metadata={"d1": {"author": "kb-owner", "year": "2025"}},+ )++ result = asyncio.run(module.retrieval())++ metadata = result["records"][0]["metadata"]+ assert metadata["author"] == "kb-owner"+ assert metadata["year"] == "2025"+ assert metadata["doc_id"] == "d1"+ assert metadata["document_id"] == "d1"++ @pytest.mark.p1+ def test_chunks_of_one_document_get_separate_metadata_dicts(self, monkeypatch):+ owner_kb = SimpleNamespace(id="kb-owner", tenant_id="tenant-owner", tenant_embd_id="", embd_id="bge")+ module = _load_dify_retrieval(+ monkeypatch,+ kb=(True, owner_kb),+ accessible=lambda _id, _u: True,+ request_body={"knowledge_id": "kb-owner", "query": "hello"},+ tenant_id="tenant-owner",+ chunks=[+ {"doc_id": "d1", "content_with_weight": "first", "similarity": 0.8, "docnm_kwd": "doc.txt"},+ {"doc_id": "d1", "content_with_weight": "second", "similarity": 0.7, "docnm_kwd": "doc.txt"},+ ],+ doc_metadata={"d1": {"author": "kb-owner"}},+ )++ result = asyncio.run(module.retrieval())++ first, second = result["records"]+ assert first["metadata"] is not second["metadata"]+ assert first["metadata"] == second["metadata"] == {"author": "kb-owner", "doc_id": "d1", "document_id": "d1"}+@pytest.mark.p1def test_no_chunks_returns_empty_records(self, monkeypatch):owner_kb = SimpleNamespace(id="kb-owner", tenant_id="tenant-owner", tenant_embd_id="", embd_id="bge")