Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.
Python agentagentic-aiagentic-frameworkagentic-workflowaiautomationclaudedeepseekgenaigptllmlow-codemcpnextjsno-codeopenaipythonskillsworkflow
perf(rag): reduce knowledge retrieval post-processing overhead (#43665)
呆萌闷油瓶e67479bd8c5a
diff --git a/api/core/rag/rerank/weight_rerank.py b/api/core/rag/rerank/weight_rerank.pyindex ca96496c593..eae44a2fa93 100644--- a/api/core/rag/rerank/weight_rerank.py+++ b/api/core/rag/rerank/weight_rerank.py@@ -160,29 +160,33 @@ class WeightRerankRunner(BaseRerankRunner):self, tenant_id: str, query: str, documents: list[Document], vector_setting: VectorSetting) -> list[float]:"""- Calculate Cosine scores+ Calculate cosine scores, resolving the query vector only for unscored documents.:param query: search query:param documents: documents for reranking:return:"""query_vector_scores = []-- model_manager = ModelManager.for_tenant(tenant_id=tenant_id)-- embedding_model = model_manager.get_model_instance(- tenant_id=tenant_id,- provider=vector_setting.embedding_provider_name,- model_type=ModelType.TEXT_EMBEDDING,- model=vector_setting.embedding_model_name,- )- cache_embedding = CacheEmbedding(embedding_model)- query_vector = cache_embedding.embed_query(query)+ query_vector: list[float] | None = Nonefor document in documents:# calculate cosine similarityif document.metadata and "score" in document.metadata:query_vector_scores.append(document.metadata["score"])else:+ if query_vector is None:+ query_vector = CacheEmbedding.get_cached_query_embedding(+ vector_setting.embedding_provider_name, vector_setting.embedding_model_name, query+ )+ if query_vector is None:+ model_manager = ModelManager.for_tenant(tenant_id=tenant_id)+ embedding_model = model_manager.get_model_instance(+ tenant_id=tenant_id,+ provider=vector_setting.embedding_provider_name,+ model_type=ModelType.TEXT_EMBEDDING,+ model=vector_setting.embedding_model_name,+ )+ query_vector = CacheEmbedding(embedding_model).embed_query(query)+# transform to NumPyvec1 = np.array(query_vector)vec2 = np.array(document.vector)diff --git a/api/core/rag/retrieval/dataset_retrieval.py b/api/core/rag/retrieval/dataset_retrieval.pyindex de5d9429888..abe451ffb01 100644--- a/api/core/rag/retrieval/dataset_retrieval.py+++ b/api/core/rag/retrieval/dataset_retrieval.py@@ -95,6 +95,7 @@ from models.dataset import Document as DatasetDocumentfrom models.dataset import Document as DocumentModelfrom models.enums import CreatorUserRole, DatasetQuerySourcefrom repositories.knowledge.dataset_read_repository import get_dataset_available_document_count+from repositories.knowledge.retrieval_source_repository import get_retrieval_source_documentsfrom repositories.knowledge.segment_read_adapter import sign_segment_contentfrom services.feature_service import FeatureServicefrom services.knowledge.external.service import ExternalDatasetService@@ -317,9 +318,12 @@ class DatasetRetrieval:dataset_ids = [i.segment.dataset_id for i in records]document_ids = [i.segment.document_id for i in records]datasets = retrieval_session.scalars(select(Dataset).where(Dataset.id.in_(dataset_ids))).all()- documents = retrieval_session.scalars(- select(DatasetDocument).where(DatasetDocument.id.in_(document_ids))- ).all()+ documents = get_retrieval_source_documents(+ tenant_id=request.tenant_id,+ dataset_ids=available_datasets_ids,+ document_ids=document_ids,+ session=retrieval_session,+ )dataset_map = {i.id: i for i in datasets}document_map = {i.id: i for i in documents}diff --git a/api/repositories/knowledge/retrieval_source_repository.py b/api/repositories/knowledge/retrieval_source_repository.pynew file mode 100644index 00000000000..3ffb1038479--- /dev/null+++ b/api/repositories/knowledge/retrieval_source_repository.py@@ -0,0 +1,39 @@+"""Minimal document reads for retrieval source citations in a caller-owned session."""++from collections.abc import Sequence++from sqlalchemy import select+from sqlalchemy.orm import Session, load_only++from models.dataset import Document+++def get_retrieval_source_documents(+ *, tenant_id: str, dataset_ids: Sequence[str], document_ids: Sequence[str], session: Session+) -> Sequence[Document]:+ """Read citation fields without loading document processing payloads.++ Other document fields are intentionally unavailable; this read must not grow+ implicit per-document queries when response assembly changes.+ """+ if not dataset_ids or not document_ids:+ return []+ return session.scalars(+ select(Document)+ .where(+ Document.tenant_id == tenant_id,+ Document.dataset_id.in_(set(dataset_ids)),+ Document.id.in_(set(document_ids)),+ )+ .options(+ load_only(+ Document.id,+ Document.tenant_id,+ Document.dataset_id,+ Document.name,+ Document.data_source_type,+ Document.doc_metadata,+ raiseload=True,+ )+ )+ ).all()diff --git a/api/tests/unit_tests/core/rag/rerank/test_reranker.py b/api/tests/unit_tests/core/rag/rerank/test_reranker.pyindex faf5a85d18e..270ec9ab367 100644--- a/api/tests/unit_tests/core/rag/rerank/test_reranker.py+++ b/api/tests/unit_tests/core/rag/rerank/test_reranker.py@@ -650,6 +650,7 @@ class TestWeightRerankRunner:def mock_cache_embedding(self):"""Mock CacheEmbedding for vector operations."""with patch("core.rag.rerank.weight_rerank.CacheEmbedding", autospec=True) as mock_cache:+ mock_cache.get_cached_query_embedding.return_value = Noneyield mock_cache@pytest.fixture@@ -670,6 +671,56 @@ class TestWeightRerankRunner:keyword_setting=KeywordSetting(keyword_weight=0.4),)+ @pytest.mark.parametrize("scores", [[], [0.0, 0.95, -0.2]])+ def test_cosine_skips_embedding_when_scores_are_available(+ self, scores, weights_config, mock_model_manager, mock_cache_embedding+ ):+ runner = WeightRerankRunner(tenant_id="tenant123", weights=weights_config)+ documents = [Document(page_content="content", metadata={"score": score}) for score in scores]++ result = runner._calculate_cosine("tenant123", "query", documents, weights_config.vector_setting)++ assert result == scores+ mock_model_manager.assert_not_called()+ mock_cache_embedding.assert_not_called()+ mock_cache_embedding.get_cached_query_embedding.assert_not_called()++ @pytest.mark.parametrize("cache_hit", [True, False])+ def test_cosine_resolves_query_vector_once_for_mixed_documents(+ self, cache_hit, weights_config, mock_model_manager, mock_cache_embedding+ ):+ runner = WeightRerankRunner(tenant_id="tenant123", weights=weights_config)+ documents = [+ Document(page_content="scored", metadata={"score": 0.25}),+ Document(page_content="orthogonal", vector=[0.0, 1.0]),+ Document(page_content="parallel", vector=[1.0, 0.0]),+ ]+ query_vector = [1.0, 0.0]+ mock_cache_embedding.get_cached_query_embedding.return_value = query_vector if cache_hit else None+ mock_cache_embedding.return_value.embed_query.return_value = query_vector++ result = runner._calculate_cosine("tenant123", "query", documents, weights_config.vector_setting)++ assert result == pytest.approx([0.25, 0.0, 1.0])+ mock_cache_embedding.get_cached_query_embedding.assert_called_once_with(+ "openai", "text-embedding-ada-002", "query"+ )+ if cache_hit:+ mock_model_manager.assert_not_called()+ mock_cache_embedding.assert_not_called()+ else:+ mock_model_manager.assert_called_once_with(tenant_id="tenant123")+ mock_model_manager.return_value.get_model_instance.assert_called_once_with(+ tenant_id="tenant123",+ provider="openai",+ model_type=ModelType.TEXT_EMBEDDING,+ model="text-embedding-ada-002",+ )+ mock_cache_embedding.assert_called_once_with(+ mock_model_manager.return_value.get_model_instance.return_value+ )+ mock_cache_embedding.return_value.embed_query.assert_called_once_with("query")+@pytest.fixturedef sample_documents_with_vectors(self):"""Create sample documents with vector embeddings."""@@ -1583,6 +1634,7 @@ class TestRerankEdgeCases(_UsesSQLiteSession):mock_cache_instance = MagicMock()mock_cache_instance.embed_query.return_value = [0.1, 0.2, 0.3]mock_cache.return_value = mock_cache_instance+ mock_cache.get_cached_query_embedding.return_value = None# Act: Run rerankingresult = runner.run(query="test", documents=documents)@@ -1727,6 +1779,7 @@ class TestRerankPerformance(_UsesSQLiteSession):mock_cache_instance = MagicMock()mock_cache_instance.embed_query.return_value = [0.1, 0.2]mock_cache.return_value = mock_cache_instance+ mock_cache.get_cached_query_embedding.return_value = None# Act: Run rerankingresult = runner.run(query="test", documents=documents)@@ -1866,6 +1919,7 @@ class TestRerankErrorHandling(_UsesSQLiteSession):mock_cache_instance = MagicMock()mock_cache_instance.embed_query.return_value = [0.1, 0.2]mock_cache.return_value = mock_cache_instance+ mock_cache.get_cached_query_embedding.return_value = None# Act & Assert: Should raise TypeError when processing None vector# The numpy array() call on None vector will faildiff --git a/api/tests/unit_tests/core/rag/retrieval/test_retrieval_source_query.py b/api/tests/unit_tests/core/rag/retrieval/test_retrieval_source_query.pynew file mode 100644index 00000000000..fc918dbc29a--- /dev/null+++ b/api/tests/unit_tests/core/rag/retrieval/test_retrieval_source_query.py@@ -0,0 +1,174 @@+"""Exercise the SQL emitted when workflow retrieval assembles source citations."""++from unittest.mock import patch+from uuid import uuid4++import pytest+from sqlalchemy import event, inspect+from sqlalchemy.exc import InvalidRequestError+from sqlalchemy.orm import Session++from core.rag.embedding.retrieval import RetrievalSegments+from core.rag.models.document import Document as RetrievedDocument+from core.rag.retrieval.dataset_retrieval import DatasetRetrieval+from core.workflow.nodes.knowledge_retrieval.retrieval import KnowledgeRetrievalRequest+from models.dataset import Dataset, Document, DocumentSegment+from repositories.knowledge.retrieval_source_repository import get_retrieval_source_documents+++def test_source_citations_do_not_fetch_unused_document_fields(sqlite_session: Session) -> None:+ tenant_id, creator_id, dataset_id, document_id = (str(uuid4()) for _ in range(4))+ dataset = Dataset(id=dataset_id, tenant_id=tenant_id, name="Knowledge", created_by=creator_id)+ document = Document(+ id=document_id,+ tenant_id=tenant_id,+ dataset_id=dataset_id,+ position=1,+ data_source_type="upload_file",+ batch="batch-1",+ name="source.txt",+ created_from="web",+ created_by=creator_id,+ enabled=True,+ archived=False,+ doc_metadata={"category": "transport"},+ data_source_info="x" * 1_000_000,+ error="y" * 1_000_000,+ )+ segments = [+ DocumentSegment(+ tenant_id=tenant_id,+ dataset_id=dataset_id,+ document_id=document_id,+ position=position,+ content=f"content-{position}",+ word_count=10,+ tokens=3,+ created_by=creator_id,+ enabled=True,+ index_node_id=f"node-{position}",+ index_node_hash=f"hash-{position}",+ )+ for position in (1, 2)+ ]+ sqlite_session.add_all([dataset, document, *segments])+ sqlite_session.commit()+ records = [+ RetrievalSegments(segment=segment, score=score) for segment, score in zip(segments, (0.8, 0.9), strict=True)+ ]+ retrieved = [RetrievedDocument(provider="dify", page_content="content", metadata={})]+ request = KnowledgeRetrievalRequest(+ tenant_id=tenant_id,+ user_id=creator_id,+ app_id=str(uuid4()),+ user_from="web",+ dataset_ids=[dataset_id],+ query="小动物如何运输",+ retrieval_mode="multiple",+ metadata_filtering_mode="disabled",+ )+ queries: list[str] = []++ def capture_query(_conn, _cursor, statement: str, _parameters, _context, _executemany) -> None:+ if "FROM documents" in statement:+ queries.append(statement)++ engine = sqlite_session.get_bind()+ retrieval = DatasetRetrieval()+ event.listen(engine, "before_cursor_execute", capture_query)+ try:+ with (+ patch.object(retrieval, "_check_knowledge_rate_limit"),+ patch.object(retrieval, "_get_available_datasets", return_value=[dataset]),+ patch.object(retrieval, "multiple_retrieve", return_value=retrieved),+ patch(+ "core.rag.retrieval.dataset_retrieval.RetrievalService.format_retrieval_documents",+ return_value=records,+ ),+ ):+ sources = retrieval.knowledge_retrieval(sqlite_session, request)+ finally:+ event.remove(engine, "before_cursor_execute", capture_query)++ assert len(sources) == 2+ assert [source.content for source in sources] == ["content-2", "content-1"]+ for source in sources:+ assert source.title == "source.txt"+ assert source.metadata.document_id == document_id+ assert source.metadata.dataset_name == "Knowledge"+ assert source.metadata.data_source_type == "upload_file"+ assert source.metadata.doc_metadata == {"category": "transport"}+ assert len(queries) == 1, "Citation fields must not cause additional lazy SQL queries"+ assert "documents.data_source_info" not in queries[0]+ assert "documents.error" not in queries[0]+ assert "documents.indexing_latency" not in queries[0]+++def test_source_document_read_is_scoped_and_has_no_lazy_queries(sqlite_session: Session) -> None:+ tenant_id, foreign_tenant_id, dataset_id, foreign_dataset_id = (str(uuid4()) for _ in range(4))+ documents = [+ Document(+ id=str(uuid4()),+ tenant_id=owner,+ dataset_id=dataset,+ position=1,+ data_source_type="upload_file",+ batch="batch-1",+ name="source.txt",+ created_from="web",+ created_by=str(uuid4()),+ doc_metadata=None,+ )+ for owner, dataset in (+ (tenant_id, dataset_id),+ (foreign_tenant_id, dataset_id),+ (tenant_id, foreign_dataset_id),+ )+ ]+ sqlite_session.add_all(documents)+ sqlite_session.flush()+ document_ids = [document.id for document in documents]+ sqlite_session.expunge_all()+ queries: list[str] = []++ def capture_query(_conn, _cursor, statement: str, _parameters, _context, _executemany) -> None:+ queries.append(statement)++ engine = sqlite_session.get_bind()+ event.listen(engine, "before_cursor_execute", capture_query)+ try:+ result = get_retrieval_source_documents(+ tenant_id=tenant_id,+ dataset_ids=[dataset_id],+ document_ids=[*document_ids, document_ids[0]],+ session=sqlite_session,+ )+ assert [document.id for document in result] == [document_ids[0]]+ assert result[0].name == "source.txt"+ assert result[0].data_source_type == "upload_file"+ assert result[0].doc_metadata is None+ assert set(inspect(result[0]).dict) - {"_sa_instance_state"} == {+ "id",+ "tenant_id",+ "dataset_id",+ "name",+ "data_source_type",+ "doc_metadata",+ }+ with pytest.raises(InvalidRequestError, match="raiseload"):+ _ = result[0].data_source_info+ assert (+ get_retrieval_source_documents(+ tenant_id=tenant_id, dataset_ids=[], document_ids=document_ids, session=sqlite_session+ )+ == []+ )+ assert (+ get_retrieval_source_documents(+ tenant_id=tenant_id, dataset_ids=[dataset_id], document_ids=[], session=sqlite_session+ )+ == []+ )+ finally:+ event.remove(engine, "before_cursor_execute", capture_query)+ assert len(queries) == 1