Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.
Python agentagentic-aiagentic-frameworkagentic-workflowaiautomationclaudedeepseekgenaigptllmlow-codemcpnextjsno-codeopenaipythonskillsworkflow
fix(vdb): balance multi-keyword Qdrant results (#44352)
mika8e26f026af91
diff --git a/api/providers/vdb/vdb-qdrant/src/dify_vdb_qdrant/qdrant_vector.py b/api/providers/vdb/vdb-qdrant/src/dify_vdb_qdrant/qdrant_vector.pyindex 0be6534ee35..3c2c010d729 100644--- a/api/providers/vdb/vdb-qdrant/src/dify_vdb_qdrant/qdrant_vector.py+++ b/api/providers/vdb/vdb-qdrant/src/dify_vdb_qdrant/qdrant_vector.py@@ -443,8 +443,12 @@ class QdrantVector(BaseVector):return []top_k = kwargs.get("top_k", 2)+ if top_k <= 0:+ return []+seen_ids: set[str | int] = set()documents: list[Document] = []+ keyword_results: list[list[rest.Record]] = []# Search each keyword separately and merge results.# This ensures each keyword gets its own search, preventing one keyword's@@ -468,14 +472,29 @@ class QdrantVector(BaseVector):with_vectors=True,)results = response[0]-- for result in results:- if result and result.id not in seen_ids:- seen_ids.add(result.id)- document = self._document_from_scored_point(result, Field.CONTENT_KEY, Field.METADATA_KEY)- documents.append(document)- if len(documents) >= top_k:- return documents+ keyword_results.append(results)++ # Select round-robin so a common first keyword cannot fill top_k before+ # later keywords contribute their matches.+ result_indexes = [0] * len(keyword_results)+ while True:+ found_result = False+ for index, results in enumerate(keyword_results):+ while result_indexes[index] < len(results):+ result = results[result_indexes[index]]+ result_indexes[index] += 1+ if result.id not in seen_ids:+ seen_ids.add(result.id)+ document = self._document_from_scored_point(result, Field.CONTENT_KEY, Field.METADATA_KEY)+ documents.append(document)+ found_result = True+ break+ if len(documents) >= top_k:+ break+ if len(documents) >= top_k:+ break+ if not found_result:+ breakreturn documentsdiff --git a/api/providers/vdb/vdb-qdrant/tests/unit_tests/test_qdrant_vector.py b/api/providers/vdb/vdb-qdrant/tests/unit_tests/test_qdrant_vector.pyindex e7b3c68b24b..2e5342bfc94 100644--- a/api/providers/vdb/vdb-qdrant/tests/unit_tests/test_qdrant_vector.py+++ b/api/providers/vdb/vdb-qdrant/tests/unit_tests/test_qdrant_vector.py@@ -6,7 +6,7 @@ import typesfrom collections import UserDictfrom pathlib import Pathfrom types import SimpleNamespace-from typing import override+from typing import Protocol, cast, overridefrom unittest.mock import MagicMock, patchimport pytest@@ -19,6 +19,13 @@ from models.dataset import Dataset, DatasetCollectionBindingfrom models.enums import CollectionBindingType+class ScrollMock(Protocol):+ side_effect: object+ call_count: int++ def assert_not_called(self) -> None: ...++def _build_fake_qdrant_modules():qdrant_client = types.ModuleType("qdrant_client")qdrant_http = types.ModuleType("qdrant_client.http")@@ -279,7 +286,7 @@ def test_search_and_helper_methods(qdrant_module):None,),]- vector._client.scroll.side_effect = scroll_results+ cast(ScrollMock, vector._client.scroll).side_effect = scroll_resultsdocs = vector.search_by_full_text("hello world", top_k=2, document_ids_filter=["d-1"])assert len(docs) == 2assert vector.search_by_full_text(" ", top_k=2) == []@@ -297,6 +304,72 @@ def test_search_and_helper_methods(qdrant_module):assert doc.page_content == "doc"+def test_qdrant_full_text_search_does_not_starve_later_keywords(qdrant_module: types.ModuleType) -> None:+ vector = qdrant_module.QdrantVector("collection_1", "group-1", _config(qdrant_module))+ common_first = qdrant_module.rest.PointStruct(+ id="p1", payload={"page_content": "common", "metadata": {"doc_id": "1"}}, vector=[0.1]+ )+ common_second = qdrant_module.rest.PointStruct(+ id="p2", payload={"page_content": "common", "metadata": {"doc_id": "2"}}, vector=[0.2]+ )+ rare_match = qdrant_module.rest.PointStruct(+ id="p3", payload={"page_content": "rare", "metadata": {"doc_id": "3"}}, vector=[0.3]+ )+ scroll = cast(ScrollMock, vector._client.scroll)+ scroll.side_effect = [([common_first, common_second], None), ([rare_match], None)]++ docs = vector.search_by_full_text("common rare", top_k=2)++ assert [doc.metadata["doc_id"] for doc in docs] == ["1", "3"]+ assert scroll.call_count == 2+++def test_qdrant_full_text_search_deduplicates_overlapping_keyword_matches(qdrant_module: types.ModuleType) -> None:+ vector = qdrant_module.QdrantVector("collection_1", "group-1", _config(qdrant_module))+ first_match = qdrant_module.rest.PointStruct(+ id="p1", payload={"page_content": "common rare", "metadata": {"doc_id": "1"}}, vector=[0.1]+ )+ second_match = qdrant_module.rest.PointStruct(+ id="p2", payload={"page_content": "common", "metadata": {"doc_id": "2"}}, vector=[0.2]+ )+ later_match = qdrant_module.rest.PointStruct(+ id="p3", payload={"page_content": "rare", "metadata": {"doc_id": "3"}}, vector=[0.3]+ )+ cast(ScrollMock, vector._client.scroll).side_effect = [+ ([first_match, second_match], None),+ ([first_match, later_match], None),+ ]++ docs = vector.search_by_full_text("common rare", top_k=3)++ assert [doc.metadata["doc_id"] for doc in docs] == ["1", "3", "2"]+++def test_qdrant_full_text_search_stops_after_exhausting_duplicate_matches(qdrant_module: types.ModuleType) -> None:+ vector = qdrant_module.QdrantVector("collection_1", "group-1", _config(qdrant_module))+ first_match = qdrant_module.rest.PointStruct(+ id="p1", payload={"page_content": "common rare", "metadata": {"doc_id": "1"}}, vector=[0.1]+ )+ second_match = qdrant_module.rest.PointStruct(+ id="p2", payload={"page_content": "common", "metadata": {"doc_id": "2"}}, vector=[0.2]+ )+ scroll = cast(ScrollMock, vector._client.scroll)+ scroll.side_effect = [([first_match, second_match], None), ([first_match], None)]++ docs = vector.search_by_full_text("common rare", top_k=4)++ assert [doc.metadata["doc_id"] for doc in docs] == ["1", "2"]+ assert scroll.call_count == 2++[email protected]("top_k", [0, -1])+def test_qdrant_full_text_search_skips_nonpositive_top_k(qdrant_module: types.ModuleType, top_k: int) -> None:+ vector = qdrant_module.QdrantVector("collection_1", "group-1", _config(qdrant_module))++ assert vector.search_by_full_text("hello world", top_k=top_k) == []+ cast(ScrollMock, vector._client.scroll).assert_not_called()++@pytest.mark.parametrize("sqlite3_session", [(DatasetCollectionBinding,)], indirect=True)@pytest.mark.parametrize("use_caller_session", [True, False])def test_qdrant_factory_paths(diff --git a/api/providers/vdb/vdb-tidb-on-qdrant/src/dify_vdb_tidb_on_qdrant/tidb_on_qdrant_vector.py b/api/providers/vdb/vdb-tidb-on-qdrant/src/dify_vdb_tidb_on_qdrant/tidb_on_qdrant_vector.pyindex 150cf89af9b..6b464c67a88 100644--- a/api/providers/vdb/vdb-tidb-on-qdrant/src/dify_vdb_tidb_on_qdrant/tidb_on_qdrant_vector.py+++ b/api/providers/vdb/vdb-tidb-on-qdrant/src/dify_vdb_tidb_on_qdrant/tidb_on_qdrant_vector.py@@ -406,6 +406,7 @@ class TidbOnQdrantVector(BaseVector):seen_ids: set[str | int] = set()documents: list[Document] = []+ keyword_results: list[list[rest.Record]] = []# MatchText combines tokens with AND; separate searches provide OR between keywords.for keyword in keywords:scroll_filter = models.Filter(@@ -424,13 +425,30 @@ class TidbOnQdrantVector(BaseVector):with_payload=True,with_vectors=True,)- for result in results:- if result and result.id not in seen_ids:- seen_ids.add(result.id)- document = self._document_from_scored_point(result, Field.CONTENT_KEY, Field.METADATA_KEY)- documents.append(document)- if len(documents) >= top_k:- return documents+ keyword_results.append(results)++ # Select round-robin so a common first keyword cannot fill top_k before+ # later keywords contribute their matches.+ result_indexes = [0] * len(keyword_results)+ while True:+ found_result = False+ for index, results in enumerate(keyword_results):+ while result_indexes[index] < len(results):+ result = results[result_indexes[index]]+ result_indexes[index] += 1+ if result.id not in seen_ids:+ seen_ids.add(result.id)+ documents.append(+ self._document_from_scored_point(result, Field.CONTENT_KEY, Field.METADATA_KEY)+ )+ found_result = True+ break+ if len(documents) >= top_k:+ break+ if len(documents) >= top_k:+ break+ if not found_result:+ breakreturn documentsdiff --git a/api/providers/vdb/vdb-tidb-on-qdrant/tests/unit_tests/test_tidb_on_qdrant_vector.py b/api/providers/vdb/vdb-tidb-on-qdrant/tests/unit_tests/test_tidb_on_qdrant_vector.pyindex a93a3b61887..46302234c02 100644--- a/api/providers/vdb/vdb-tidb-on-qdrant/tests/unit_tests/test_tidb_on_qdrant_vector.py+++ b/api/providers/vdb/vdb-tidb-on-qdrant/tests/unit_tests/test_tidb_on_qdrant_vector.py@@ -1,3 +1,4 @@+from typing import Protocol, castfrom unittest.mock import patchimport httpx@@ -13,6 +14,11 @@ from core.rag.rerank.entity.weight import KeywordSetting, VectorSetting, Weightsfrom core.rag.rerank.weight_rerank import WeightRerankRunner+class ScrollMock(Protocol):+ side_effect: object+ call_count: int++class TestTidbOnQdrantVectorFullTextSearch:@pytest.fixturedef vector(self):@@ -67,7 +73,41 @@ class TestTidbOnQdrantVectorFullTextSearch:assert [doc.metadata["doc_id"] for doc in documents] == ["1", "2", "3"][:top_k]calls = vector._client.scroll.call_args_listkeywords = [call.kwargs["scroll_filter"].must[0].match.text for call in calls]- assert keywords == (["apple"] if top_k <= 2 else ["apple", "banana"])+ assert keywords == ["apple", "banana"]++ def test_later_keywords_are_not_starved_by_first_keyword(+ self, vector: TidbOnQdrantVector, points: list[rest.Record]+ ) -> None:+ scroll = cast(ScrollMock, vector._client.scroll)+ scroll.side_effect = [([points[0], points[1]], None), ([points[2]], None)]++ documents = vector.search_by_full_text("common rare", top_k=2)++ assert [doc.metadata["doc_id"] for doc in documents] == ["1", "3"]+ assert scroll.call_count == 2++ def test_overlapping_keyword_matches_are_deduplicated(+ self, vector: TidbOnQdrantVector, points: list[rest.Record]+ ) -> None:+ cast(ScrollMock, vector._client.scroll).side_effect = [+ ([points[0], points[1]], None),+ ([points[0], points[2]], None),+ ]++ documents = vector.search_by_full_text("common rare", top_k=3)++ assert [doc.metadata["doc_id"] for doc in documents] == ["1", "3", "2"]++ def test_stops_after_exhausting_duplicate_keyword_matches(+ self, vector: TidbOnQdrantVector, points: list[rest.Record]+ ) -> None:+ scroll = cast(ScrollMock, vector._client.scroll)+ scroll.side_effect = [([points[0], points[1]], None), ([points[0]], None)]++ documents = vector.search_by_full_text("common rare", top_k=4)++ assert [doc.metadata["doc_id"] for doc in documents] == ["1", "2"]+ assert scroll.call_count == 2def test_no_match_does_not_fall_back_to_unfiltered_scroll(self, vector):vector._client.scroll.return_value = ([], None)