Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(api): return document metadata from /dify/retrieval (#19686)

Ehsanc71bb0198f2f
diff --git a/api/apps/restful_apis/dify_retrieval_api.py b/api/apps/restful_apis/dify_retrieval_api.py
index 0640ff799c..68ebd16098 100644
--- a/api/apps/restful_apis/dify_retrieval_api.py
+++ b/api/apps/restful_apis/dify_retrieval_api.py
@@ -291,6 +291,7 @@ async def retrieval(tenant_id):
doc_ids = list(set([c["doc_id"] for c in ranks["chunks"]]))
docs = DocumentService.get_by_ids(doc_ids)
doc_map = {doc.id: doc for doc in docs}
+ metadata_map = DocMetadataService.get_metadata_for_documents(doc_ids, kb_id) if doc_ids else {}
records = []
for c in ranks["chunks"]:
@@ -298,7 +299,8 @@ async def retrieval(tenant_id):
if not doc:
continue
c.pop("vector", None)
- meta = getattr(doc, "meta_fields", {})
+ # Copied because several chunks of one document share a map entry.
+ meta = dict(metadata_map.get(c["doc_id"]) or {})
meta["doc_id"] = c["doc_id"]
# Dify expects metadata.document_id for external retrieval sources.
meta["document_id"] = c["doc_id"]
diff --git a/test/unit_test/api/apps/sdk/test_dify_retrieval.py b/test/unit_test/api/apps/sdk/test_dify_retrieval.py
index f394a45ea4..80175ec3e7 100644
--- a/test/unit_test/api/apps/sdk/test_dify_retrieval.py
+++ b/test/unit_test/api/apps/sdk/test_dify_retrieval.py
@@ -113,6 +113,7 @@ def _load_dify_retrieval(
request_args=None,
kg_content="",
meta_filter_ids=None,
+ doc_metadata=None,
):
"""Load dify_retrieval_api.py with minimum stubs to exercise the retrieval handler."""
@@ -142,14 +143,20 @@ def _load_dify_retrieval(
monkeypatch,
"api.db.services.document_service",
DocumentService=SimpleNamespace(
- get_by_id=lambda _id: (True, SimpleNamespace(id=_id, meta_fields={})),
- get_by_ids=lambda ids, cols=None: [SimpleNamespace(id=doc_id, meta_fields={}) for doc_id in ids],
+ get_by_id=lambda _id: (True, SimpleNamespace(id=_id)),
+ get_by_ids=lambda ids, cols=None: [SimpleNamespace(id=doc_id) for doc_id in ids],
),
)
+ # Keyed by doc_id, and the same dict object is handed back on every lookup,
+ # which is what DocMetadataService.get_metadata_for_documents does.
+ doc_metadata = doc_metadata or {}
_stub(
monkeypatch,
"api.db.services.doc_metadata_service",
- DocMetadataService=SimpleNamespace(get_flatted_meta_by_kbs=lambda _ids: {}),
+ DocMetadataService=SimpleNamespace(
+ get_flatted_meta_by_kbs=lambda _ids: {},
+ get_metadata_for_documents=lambda ids, kb_id: {i: doc_metadata[i] for i in ids if i in doc_metadata},
+ ),
)
acc_fn = accessible if callable(accessible) else (lambda *_a, **_k: accessible)
@@ -626,6 +633,49 @@ class TestDifyRetrievalRetrievalBehavior:
assert record["metadata"]["doc_id"] == "d1"
assert record["metadata"]["document_id"] == "d1"
+ @pytest.mark.p1
+ def test_records_carry_the_document_metadata(self, monkeypatch):
+ owner_kb = SimpleNamespace(id="kb-owner", tenant_id="tenant-owner", tenant_embd_id="", embd_id="bge")
+ module = _load_dify_retrieval(
+ monkeypatch,
+ kb=(True, owner_kb),
+ accessible=lambda _id, _u: True,
+ request_body={"knowledge_id": "kb-owner", "query": "hello"},
+ tenant_id="tenant-owner",
+ chunks=[{"doc_id": "d1", "content_with_weight": "hello world", "similarity": 0.8, "docnm_kwd": "doc.txt"}],
+ doc_metadata={"d1": {"author": "kb-owner", "year": "2025"}},
+ )
+
+ result = asyncio.run(module.retrieval())
+
+ metadata = result["records"][0]["metadata"]
+ assert metadata["author"] == "kb-owner"
+ assert metadata["year"] == "2025"
+ assert metadata["doc_id"] == "d1"
+ assert metadata["document_id"] == "d1"
+
+ @pytest.mark.p1
+ def test_chunks_of_one_document_get_separate_metadata_dicts(self, monkeypatch):
+ owner_kb = SimpleNamespace(id="kb-owner", tenant_id="tenant-owner", tenant_embd_id="", embd_id="bge")
+ module = _load_dify_retrieval(
+ monkeypatch,
+ kb=(True, owner_kb),
+ accessible=lambda _id, _u: True,
+ request_body={"knowledge_id": "kb-owner", "query": "hello"},
+ tenant_id="tenant-owner",
+ chunks=[
+ {"doc_id": "d1", "content_with_weight": "first", "similarity": 0.8, "docnm_kwd": "doc.txt"},
+ {"doc_id": "d1", "content_with_weight": "second", "similarity": 0.7, "docnm_kwd": "doc.txt"},
+ ],
+ doc_metadata={"d1": {"author": "kb-owner"}},
+ )
+
+ result = asyncio.run(module.retrieval())
+
+ first, second = result["records"]
+ assert first["metadata"] is not second["metadata"]
+ assert first["metadata"] == second["metadata"] == {"author": "kb-owner", "doc_id": "d1", "document_id": "d1"}
+
@pytest.mark.p1
def test_no_chunks_returns_empty_records(self, monkeypatch):
owner_kb = SimpleNamespace(id="kb-owner", tenant_id="tenant-owner", tenant_embd_id="", embd_id="bge")