Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

fix(vdb): balance multi-keyword Qdrant results (#44352)

mika8e26f026af91
diff --git a/api/providers/vdb/vdb-qdrant/src/dify_vdb_qdrant/qdrant_vector.py b/api/providers/vdb/vdb-qdrant/src/dify_vdb_qdrant/qdrant_vector.py
index 0be6534ee35..3c2c010d729 100644
--- a/api/providers/vdb/vdb-qdrant/src/dify_vdb_qdrant/qdrant_vector.py
+++ b/api/providers/vdb/vdb-qdrant/src/dify_vdb_qdrant/qdrant_vector.py
@@ -443,8 +443,12 @@ class QdrantVector(BaseVector):
return []
top_k = kwargs.get("top_k", 2)
+ if top_k <= 0:
+ return []
+
seen_ids: set[str | int] = set()
documents: list[Document] = []
+ keyword_results: list[list[rest.Record]] = []
# Search each keyword separately and merge results.
# This ensures each keyword gets its own search, preventing one keyword's
@@ -468,14 +472,29 @@ class QdrantVector(BaseVector):
with_vectors=True,
)
results = response[0]
-
- for result in results:
- if result and result.id not in seen_ids:
- seen_ids.add(result.id)
- document = self._document_from_scored_point(result, Field.CONTENT_KEY, Field.METADATA_KEY)
- documents.append(document)
- if len(documents) >= top_k:
- return documents
+ keyword_results.append(results)
+
+ # Select round-robin so a common first keyword cannot fill top_k before
+ # later keywords contribute their matches.
+ result_indexes = [0] * len(keyword_results)
+ while True:
+ found_result = False
+ for index, results in enumerate(keyword_results):
+ while result_indexes[index] < len(results):
+ result = results[result_indexes[index]]
+ result_indexes[index] += 1
+ if result.id not in seen_ids:
+ seen_ids.add(result.id)
+ document = self._document_from_scored_point(result, Field.CONTENT_KEY, Field.METADATA_KEY)
+ documents.append(document)
+ found_result = True
+ break
+ if len(documents) >= top_k:
+ break
+ if len(documents) >= top_k:
+ break
+ if not found_result:
+ break
return documents
diff --git a/api/providers/vdb/vdb-qdrant/tests/unit_tests/test_qdrant_vector.py b/api/providers/vdb/vdb-qdrant/tests/unit_tests/test_qdrant_vector.py
index e7b3c68b24b..2e5342bfc94 100644
--- a/api/providers/vdb/vdb-qdrant/tests/unit_tests/test_qdrant_vector.py
+++ b/api/providers/vdb/vdb-qdrant/tests/unit_tests/test_qdrant_vector.py
@@ -6,7 +6,7 @@ import types
from collections import UserDict
from pathlib import Path
from types import SimpleNamespace
-from typing import override
+from typing import Protocol, cast, override
from unittest.mock import MagicMock, patch
import pytest
@@ -19,6 +19,13 @@ from models.dataset import Dataset, DatasetCollectionBinding
from models.enums import CollectionBindingType
+class ScrollMock(Protocol):
+ side_effect: object
+ call_count: int
+
+ def assert_not_called(self) -> None: ...
+
+
def _build_fake_qdrant_modules():
qdrant_client = types.ModuleType("qdrant_client")
qdrant_http = types.ModuleType("qdrant_client.http")
@@ -279,7 +286,7 @@ def test_search_and_helper_methods(qdrant_module):
None,
),
]
- vector._client.scroll.side_effect = scroll_results
+ cast(ScrollMock, vector._client.scroll).side_effect = scroll_results
docs = vector.search_by_full_text("hello world", top_k=2, document_ids_filter=["d-1"])
assert len(docs) == 2
assert vector.search_by_full_text(" ", top_k=2) == []
@@ -297,6 +304,72 @@ def test_search_and_helper_methods(qdrant_module):
assert doc.page_content == "doc"
+def test_qdrant_full_text_search_does_not_starve_later_keywords(qdrant_module: types.ModuleType) -> None:
+ vector = qdrant_module.QdrantVector("collection_1", "group-1", _config(qdrant_module))
+ common_first = qdrant_module.rest.PointStruct(
+ id="p1", payload={"page_content": "common", "metadata": {"doc_id": "1"}}, vector=[0.1]
+ )
+ common_second = qdrant_module.rest.PointStruct(
+ id="p2", payload={"page_content": "common", "metadata": {"doc_id": "2"}}, vector=[0.2]
+ )
+ rare_match = qdrant_module.rest.PointStruct(
+ id="p3", payload={"page_content": "rare", "metadata": {"doc_id": "3"}}, vector=[0.3]
+ )
+ scroll = cast(ScrollMock, vector._client.scroll)
+ scroll.side_effect = [([common_first, common_second], None), ([rare_match], None)]
+
+ docs = vector.search_by_full_text("common rare", top_k=2)
+
+ assert [doc.metadata["doc_id"] for doc in docs] == ["1", "3"]
+ assert scroll.call_count == 2
+
+
+def test_qdrant_full_text_search_deduplicates_overlapping_keyword_matches(qdrant_module: types.ModuleType) -> None:
+ vector = qdrant_module.QdrantVector("collection_1", "group-1", _config(qdrant_module))
+ first_match = qdrant_module.rest.PointStruct(
+ id="p1", payload={"page_content": "common rare", "metadata": {"doc_id": "1"}}, vector=[0.1]
+ )
+ second_match = qdrant_module.rest.PointStruct(
+ id="p2", payload={"page_content": "common", "metadata": {"doc_id": "2"}}, vector=[0.2]
+ )
+ later_match = qdrant_module.rest.PointStruct(
+ id="p3", payload={"page_content": "rare", "metadata": {"doc_id": "3"}}, vector=[0.3]
+ )
+ cast(ScrollMock, vector._client.scroll).side_effect = [
+ ([first_match, second_match], None),
+ ([first_match, later_match], None),
+ ]
+
+ docs = vector.search_by_full_text("common rare", top_k=3)
+
+ assert [doc.metadata["doc_id"] for doc in docs] == ["1", "3", "2"]
+
+
+def test_qdrant_full_text_search_stops_after_exhausting_duplicate_matches(qdrant_module: types.ModuleType) -> None:
+ vector = qdrant_module.QdrantVector("collection_1", "group-1", _config(qdrant_module))
+ first_match = qdrant_module.rest.PointStruct(
+ id="p1", payload={"page_content": "common rare", "metadata": {"doc_id": "1"}}, vector=[0.1]
+ )
+ second_match = qdrant_module.rest.PointStruct(
+ id="p2", payload={"page_content": "common", "metadata": {"doc_id": "2"}}, vector=[0.2]
+ )
+ scroll = cast(ScrollMock, vector._client.scroll)
+ scroll.side_effect = [([first_match, second_match], None), ([first_match], None)]
+
+ docs = vector.search_by_full_text("common rare", top_k=4)
+
+ assert [doc.metadata["doc_id"] for doc in docs] == ["1", "2"]
+ assert scroll.call_count == 2
+
+
[email protected]("top_k", [0, -1])
+def test_qdrant_full_text_search_skips_nonpositive_top_k(qdrant_module: types.ModuleType, top_k: int) -> None:
+ vector = qdrant_module.QdrantVector("collection_1", "group-1", _config(qdrant_module))
+
+ assert vector.search_by_full_text("hello world", top_k=top_k) == []
+ cast(ScrollMock, vector._client.scroll).assert_not_called()
+
+
@pytest.mark.parametrize("sqlite3_session", [(DatasetCollectionBinding,)], indirect=True)
@pytest.mark.parametrize("use_caller_session", [True, False])
def test_qdrant_factory_paths(
diff --git a/api/providers/vdb/vdb-tidb-on-qdrant/src/dify_vdb_tidb_on_qdrant/tidb_on_qdrant_vector.py b/api/providers/vdb/vdb-tidb-on-qdrant/src/dify_vdb_tidb_on_qdrant/tidb_on_qdrant_vector.py
index 150cf89af9b..6b464c67a88 100644
--- a/api/providers/vdb/vdb-tidb-on-qdrant/src/dify_vdb_tidb_on_qdrant/tidb_on_qdrant_vector.py
+++ b/api/providers/vdb/vdb-tidb-on-qdrant/src/dify_vdb_tidb_on_qdrant/tidb_on_qdrant_vector.py
@@ -406,6 +406,7 @@ class TidbOnQdrantVector(BaseVector):
seen_ids: set[str | int] = set()
documents: list[Document] = []
+ keyword_results: list[list[rest.Record]] = []
# MatchText combines tokens with AND; separate searches provide OR between keywords.
for keyword in keywords:
scroll_filter = models.Filter(
@@ -424,13 +425,30 @@ class TidbOnQdrantVector(BaseVector):
with_payload=True,
with_vectors=True,
)
- for result in results:
- if result and result.id not in seen_ids:
- seen_ids.add(result.id)
- document = self._document_from_scored_point(result, Field.CONTENT_KEY, Field.METADATA_KEY)
- documents.append(document)
- if len(documents) >= top_k:
- return documents
+ keyword_results.append(results)
+
+ # Select round-robin so a common first keyword cannot fill top_k before
+ # later keywords contribute their matches.
+ result_indexes = [0] * len(keyword_results)
+ while True:
+ found_result = False
+ for index, results in enumerate(keyword_results):
+ while result_indexes[index] < len(results):
+ result = results[result_indexes[index]]
+ result_indexes[index] += 1
+ if result.id not in seen_ids:
+ seen_ids.add(result.id)
+ documents.append(
+ self._document_from_scored_point(result, Field.CONTENT_KEY, Field.METADATA_KEY)
+ )
+ found_result = True
+ break
+ if len(documents) >= top_k:
+ break
+ if len(documents) >= top_k:
+ break
+ if not found_result:
+ break
return documents
diff --git a/api/providers/vdb/vdb-tidb-on-qdrant/tests/unit_tests/test_tidb_on_qdrant_vector.py b/api/providers/vdb/vdb-tidb-on-qdrant/tests/unit_tests/test_tidb_on_qdrant_vector.py
index a93a3b61887..46302234c02 100644
--- a/api/providers/vdb/vdb-tidb-on-qdrant/tests/unit_tests/test_tidb_on_qdrant_vector.py
+++ b/api/providers/vdb/vdb-tidb-on-qdrant/tests/unit_tests/test_tidb_on_qdrant_vector.py
@@ -1,3 +1,4 @@
+from typing import Protocol, cast
from unittest.mock import patch
import httpx
@@ -13,6 +14,11 @@ from core.rag.rerank.entity.weight import KeywordSetting, VectorSetting, Weights
from core.rag.rerank.weight_rerank import WeightRerankRunner
+class ScrollMock(Protocol):
+ side_effect: object
+ call_count: int
+
+
class TestTidbOnQdrantVectorFullTextSearch:
@pytest.fixture
def vector(self):
@@ -67,7 +73,41 @@ class TestTidbOnQdrantVectorFullTextSearch:
assert [doc.metadata["doc_id"] for doc in documents] == ["1", "2", "3"][:top_k]
calls = vector._client.scroll.call_args_list
keywords = [call.kwargs["scroll_filter"].must[0].match.text for call in calls]
- assert keywords == (["apple"] if top_k <= 2 else ["apple", "banana"])
+ assert keywords == ["apple", "banana"]
+
+ def test_later_keywords_are_not_starved_by_first_keyword(
+ self, vector: TidbOnQdrantVector, points: list[rest.Record]
+ ) -> None:
+ scroll = cast(ScrollMock, vector._client.scroll)
+ scroll.side_effect = [([points[0], points[1]], None), ([points[2]], None)]
+
+ documents = vector.search_by_full_text("common rare", top_k=2)
+
+ assert [doc.metadata["doc_id"] for doc in documents] == ["1", "3"]
+ assert scroll.call_count == 2
+
+ def test_overlapping_keyword_matches_are_deduplicated(
+ self, vector: TidbOnQdrantVector, points: list[rest.Record]
+ ) -> None:
+ cast(ScrollMock, vector._client.scroll).side_effect = [
+ ([points[0], points[1]], None),
+ ([points[0], points[2]], None),
+ ]
+
+ documents = vector.search_by_full_text("common rare", top_k=3)
+
+ assert [doc.metadata["doc_id"] for doc in documents] == ["1", "3", "2"]
+
+ def test_stops_after_exhausting_duplicate_keyword_matches(
+ self, vector: TidbOnQdrantVector, points: list[rest.Record]
+ ) -> None:
+ scroll = cast(ScrollMock, vector._client.scroll)
+ scroll.side_effect = [([points[0], points[1]], None), ([points[0]], None)]
+
+ documents = vector.search_by_full_text("common rare", top_k=4)
+
+ assert [doc.metadata["doc_id"] for doc in documents] == ["1", "2"]
+ assert scroll.call_count == 2
def test_no_match_does_not_fall_back_to_unfiltered_scroll(self, vector):
vector._client.scroll.return_value = ([], None)