Build and run agents you can see, understand and trust.
Python agentchatbotlarge-language-modelsllmllm-agentmcpmulti-agentmulti-modalreact-agentrealtime-agentvoice-agentvoice-assistant
fix(rag): normalize distance scores so higher means more relevant (#2486)
hhbf8f004f4d64b9
diff --git a/src/agentscope/rag/_knowledge.py b/src/agentscope/rag/_knowledge.pyindex 14c862c39..83451331a 100644--- a/src/agentscope/rag/_knowledge.py+++ b/src/agentscope/rag/_knowledge.py@@ -214,9 +214,10 @@ class KnowledgeBase:Maximum number of results returned across all queries(after dedup).score_threshold (`float | None`, optional):- Minimum similarity score for a hit to be retained.- Only meaningful for similarity metrics where higher is- better (cosine / dot-product). ``None`` disables+ Minimum :attr:`VectorSearchResult.score` for a hit to+ be retained. The scale depends on the store's metric —+ a distance metric reports negated distances, so the+ threshold is negative there. ``None`` disablesfiltering.Returns:diff --git a/src/agentscope/rag/_vdb/_milvus_lite.py b/src/agentscope/rag/_vdb/_milvus_lite.pyindex 62aa8d1bc..fa4170e2c 100644--- a/src/agentscope/rag/_vdb/_milvus_lite.py+++ b/src/agentscope/rag/_vdb/_milvus_lite.py@@ -523,14 +523,17 @@ class MilvusLiteStore(VectorStoreBase):Since milvus-lite >= 3.1.0 the ``distance`` field for COSINEalready carries the cosine similarity (1.0 for identical- vectors), so no conversion is needed.+ vectors). For the ``L2`` distance metric the raw value is+ lower-is-better, so it is negated to keep ``score`` meaning+ "higher is more relevant" like the similarity metrics."""- # pylint: disable=unused-argumentif "distance" in hit:- return float(hit["distance"])- if "score" in hit:- return float(hit["score"])- return 0.0+ score = float(hit["distance"])+ elif "score" in hit:+ score = float(hit["score"])+ else:+ return 0.0+ return -score if metric_type == "L2" else score@staticmethoddef _is_local_db_uri(uri: str) -> bool:diff --git a/src/agentscope/rag/_vdb/_qdrant.py b/src/agentscope/rag/_vdb/_qdrant.pyindex a5f8b28cd..a54471705 100644--- a/src/agentscope/rag/_vdb/_qdrant.py+++ b/src/agentscope/rag/_vdb/_qdrant.py@@ -313,9 +313,12 @@ class QdrantStore(VectorStoreBase):with_payload=True,query_filter=self._build_metadata_filter(metadata_filter),)+ # ``Euclid`` / ``Manhattan`` are distances, so negate them to+ # keep ``score`` higher-is-better like ``Cosine`` / ``Dot``.+ sign = -1 if self._distance in ("Euclid", "Manhattan") else 1return [VectorSearchResult(- score=point.score,+ score=sign * point.score,document_id=point.payload["document_id"],chunk=Chunk.model_validate(point.payload["chunk"]),)diff --git a/src/agentscope/rag/_vdb/_vector_store.py b/src/agentscope/rag/_vdb/_vector_store.pyindex 8e2684b0e..060f9507b 100644--- a/src/agentscope/rag/_vdb/_vector_store.py+++ b/src/agentscope/rag/_vdb/_vector_store.py@@ -60,8 +60,10 @@ class VectorSearchResult(BaseModel):"""score: float- """Similarity score. Higher = more similar for cosine /- dot-product; lower = more similar for L2 distance."""+ """Relevance score, always higher-is-better. Backends whose+ metric is a distance (lower-is-better) negate it before returning,+ so the value is comparable within one store but not across stores+ using different metrics."""document_id: str"""The ID of the source document the matched chunk belongs to —@@ -253,7 +255,10 @@ class VectorStoreBase(ABC):Returns:`list[VectorSearchResult]`:- Results ordered by descending similarity score.+ Results ordered by descending+ :attr:`VectorSearchResult.score`. Implementations must+ normalize a distance metric into that higher-is-better+ score, e.g. by negating it."""# ------------------------------------------------------------------diff --git a/tests/rag_vdb_milvus_lite_test.py b/tests/rag_vdb_milvus_lite_test.pyindex af1a524c6..31d01278d 100644--- a/tests/rag_vdb_milvus_lite_test.py+++ b/tests/rag_vdb_milvus_lite_test.py@@ -157,6 +157,70 @@ class MilvusLiteStoreTest(IsolatedAsyncioTestCase):],)+ async def test_search_negates_l2_distance_scores(self) -> None:+ """The L2 metric yields higher-is-better scores too."""+ store = await self._exit_stack.enter_async_context(+ MilvusLiteStore(+ uri=str(self._tmpdir / "l2.db"),+ metric_type="L2",+ ),+ )+ await store.create_collection("kb_2", dimensions=3)+ await store.insert(+ "kb_2",+ [+ _make_record("Near!", [1.0, 0.0, 0.0], document_id="doc-1"),+ _make_record("Far!", [10.0, 0.0, 0.0], document_id="doc-2"),+ ],+ )++ results = await store.search(+ "kb_2",+ query_vector=[0.0, 0.0, 0.0],+ top_k=2,+ )++ # Negated squared distances, so the nearest record ranks first+ self.assertEqual(+ _dump_results(results),+ [+ {+ "score": -1.0,+ "document_id": "doc-1",+ "chunk": {+ "content": {+ "type": "text",+ "text": "Near!",+ "id": AnyString(),+ "created_at": AnyString(),+ "finished_at": None,+ },+ "source": "doc-1.txt",+ "chunk_index": 0,+ "total_chunks": 1,+ "metadata": {},+ },+ },+ {+ "score": -100.0,+ "document_id": "doc-2",+ "chunk": {+ "content": {+ "type": "text",+ "text": "Far!",+ "id": AnyString(),+ "created_at": AnyString(),+ "finished_at": None,+ },+ "source": "doc-2.txt",+ "chunk_index": 0,+ "total_chunks": 1,+ "metadata": {},+ },+ },+ ],+ )+async def test_search_top_k(self) -> None:"""top_k limits the number of returned results."""await self.store.create_collection("kb_1", dimensions=3)diff --git a/tests/rag_vdb_qdrant_test.py b/tests/rag_vdb_qdrant_test.pyindex fbbe68893..b78a35aad 100644--- a/tests/rag_vdb_qdrant_test.py+++ b/tests/rag_vdb_qdrant_test.py@@ -163,6 +163,67 @@ class QdrantStoreTest(IsolatedAsyncioTestCase):],)+ async def test_search_negates_distance_metric_scores(self) -> None:+ """A distance metric yields higher-is-better scores too."""+ store = await self._exit_stack.enter_async_context(+ QdrantStore(location=":memory:", distance="Euclid"),+ )+ await store.create_collection("kb-1", dimensions=3)+ await store.insert(+ "kb-1",+ [+ _make_record("Near!", [1.0, 0.0, 0.0], document_id="doc-1"),+ _make_record("Far!", [10.0, 0.0, 0.0], document_id="doc-2"),+ ],+ )++ results = await store.search(+ "kb-1",+ query_vector=[0.0, 0.0, 0.0],+ top_k=2,+ )++ # Negated distances, so the nearest record still ranks first+ self.assertEqual(+ _dump_results(results),+ [+ {+ "score": -1.0,+ "document_id": "doc-1",+ "chunk": {+ "content": {+ "type": "text",+ "text": "Near!",+ "id": AnyString(),+ "created_at": AnyString(),+ "finished_at": None,+ },+ "source": "doc-1.txt",+ "chunk_index": 0,+ "total_chunks": 1,+ "metadata": {},+ },+ },+ {+ "score": -10.0,+ "document_id": "doc-2",+ "chunk": {+ "content": {+ "type": "text",+ "text": "Far!",+ "id": AnyString(),+ "created_at": AnyString(),+ "finished_at": None,+ },+ "source": "doc-2.txt",+ "chunk_index": 0,+ "total_chunks": 1,+ "metadata": {},+ },+ },+ ],+ )+async def test_search_top_k(self) -> None:"""top_k limits the number of returned results."""await self.store.create_collection("kb-1", dimensions=3)