Skip to content

Build and run agents you can see, understand and trust.

fix(rag): normalize distance scores so higher means more relevant (#2486)

hhbf8f004f4d64b9
diff --git a/src/agentscope/rag/_knowledge.py b/src/agentscope/rag/_knowledge.py
index 14c862c39..83451331a 100644
--- a/src/agentscope/rag/_knowledge.py
+++ b/src/agentscope/rag/_knowledge.py
@@ -214,9 +214,10 @@ class KnowledgeBase:
Maximum number of results returned across all queries
(after dedup).
score_threshold (`float | None`, optional):
- Minimum similarity score for a hit to be retained.
- Only meaningful for similarity metrics where higher is
- better (cosine / dot-product). ``None`` disables
+ Minimum :attr:`VectorSearchResult.score` for a hit to
+ be retained. The scale depends on the store's metric —
+ a distance metric reports negated distances, so the
+ threshold is negative there. ``None`` disables
filtering.
Returns:
diff --git a/src/agentscope/rag/_vdb/_milvus_lite.py b/src/agentscope/rag/_vdb/_milvus_lite.py
index 62aa8d1bc..fa4170e2c 100644
--- a/src/agentscope/rag/_vdb/_milvus_lite.py
+++ b/src/agentscope/rag/_vdb/_milvus_lite.py
@@ -523,14 +523,17 @@ class MilvusLiteStore(VectorStoreBase):
Since milvus-lite >= 3.1.0 the ``distance`` field for COSINE
already carries the cosine similarity (1.0 for identical
- vectors), so no conversion is needed.
+ vectors). For the ``L2`` distance metric the raw value is
+ lower-is-better, so it is negated to keep ``score`` meaning
+ "higher is more relevant" like the similarity metrics.
"""
- # pylint: disable=unused-argument
if "distance" in hit:
- return float(hit["distance"])
- if "score" in hit:
- return float(hit["score"])
- return 0.0
+ score = float(hit["distance"])
+ elif "score" in hit:
+ score = float(hit["score"])
+ else:
+ return 0.0
+ return -score if metric_type == "L2" else score
@staticmethod
def _is_local_db_uri(uri: str) -> bool:
diff --git a/src/agentscope/rag/_vdb/_qdrant.py b/src/agentscope/rag/_vdb/_qdrant.py
index a5f8b28cd..a54471705 100644
--- a/src/agentscope/rag/_vdb/_qdrant.py
+++ b/src/agentscope/rag/_vdb/_qdrant.py
@@ -313,9 +313,12 @@ class QdrantStore(VectorStoreBase):
with_payload=True,
query_filter=self._build_metadata_filter(metadata_filter),
)
+ # ``Euclid`` / ``Manhattan`` are distances, so negate them to
+ # keep ``score`` higher-is-better like ``Cosine`` / ``Dot``.
+ sign = -1 if self._distance in ("Euclid", "Manhattan") else 1
return [
VectorSearchResult(
- score=point.score,
+ score=sign * point.score,
document_id=point.payload["document_id"],
chunk=Chunk.model_validate(point.payload["chunk"]),
)
diff --git a/src/agentscope/rag/_vdb/_vector_store.py b/src/agentscope/rag/_vdb/_vector_store.py
index 8e2684b0e..060f9507b 100644
--- a/src/agentscope/rag/_vdb/_vector_store.py
+++ b/src/agentscope/rag/_vdb/_vector_store.py
@@ -60,8 +60,10 @@ class VectorSearchResult(BaseModel):
"""
score: float
- """Similarity score. Higher = more similar for cosine /
- dot-product; lower = more similar for L2 distance."""
+ """Relevance score, always higher-is-better. Backends whose
+ metric is a distance (lower-is-better) negate it before returning,
+ so the value is comparable within one store but not across stores
+ using different metrics."""
document_id: str
"""The ID of the source document the matched chunk belongs to —
@@ -253,7 +255,10 @@ class VectorStoreBase(ABC):
Returns:
`list[VectorSearchResult]`:
- Results ordered by descending similarity score.
+ Results ordered by descending
+ :attr:`VectorSearchResult.score`. Implementations must
+ normalize a distance metric into that higher-is-better
+ score, e.g. by negating it.
"""
# ------------------------------------------------------------------
diff --git a/tests/rag_vdb_milvus_lite_test.py b/tests/rag_vdb_milvus_lite_test.py
index af1a524c6..31d01278d 100644
--- a/tests/rag_vdb_milvus_lite_test.py
+++ b/tests/rag_vdb_milvus_lite_test.py
@@ -157,6 +157,70 @@ class MilvusLiteStoreTest(IsolatedAsyncioTestCase):
],
)
+ async def test_search_negates_l2_distance_scores(self) -> None:
+ """The L2 metric yields higher-is-better scores too."""
+ store = await self._exit_stack.enter_async_context(
+ MilvusLiteStore(
+ uri=str(self._tmpdir / "l2.db"),
+ metric_type="L2",
+ ),
+ )
+ await store.create_collection("kb_2", dimensions=3)
+ await store.insert(
+ "kb_2",
+ [
+ _make_record("Near!", [1.0, 0.0, 0.0], document_id="doc-1"),
+ _make_record("Far!", [10.0, 0.0, 0.0], document_id="doc-2"),
+ ],
+ )
+
+ results = await store.search(
+ "kb_2",
+ query_vector=[0.0, 0.0, 0.0],
+ top_k=2,
+ )
+
+ # Negated squared distances, so the nearest record ranks first
+ self.assertEqual(
+ _dump_results(results),
+ [
+ {
+ "score": -1.0,
+ "document_id": "doc-1",
+ "chunk": {
+ "content": {
+ "type": "text",
+ "text": "Near!",
+ "id": AnyString(),
+ "created_at": AnyString(),
+ "finished_at": None,
+ },
+ "source": "doc-1.txt",
+ "chunk_index": 0,
+ "total_chunks": 1,
+ "metadata": {},
+ },
+ },
+ {
+ "score": -100.0,
+ "document_id": "doc-2",
+ "chunk": {
+ "content": {
+ "type": "text",
+ "text": "Far!",
+ "id": AnyString(),
+ "created_at": AnyString(),
+ "finished_at": None,
+ },
+ "source": "doc-2.txt",
+ "chunk_index": 0,
+ "total_chunks": 1,
+ "metadata": {},
+ },
+ },
+ ],
+ )
+
async def test_search_top_k(self) -> None:
"""top_k limits the number of returned results."""
await self.store.create_collection("kb_1", dimensions=3)
diff --git a/tests/rag_vdb_qdrant_test.py b/tests/rag_vdb_qdrant_test.py
index fbbe68893..b78a35aad 100644
--- a/tests/rag_vdb_qdrant_test.py
+++ b/tests/rag_vdb_qdrant_test.py
@@ -163,6 +163,67 @@ class QdrantStoreTest(IsolatedAsyncioTestCase):
],
)
+ async def test_search_negates_distance_metric_scores(self) -> None:
+ """A distance metric yields higher-is-better scores too."""
+ store = await self._exit_stack.enter_async_context(
+ QdrantStore(location=":memory:", distance="Euclid"),
+ )
+ await store.create_collection("kb-1", dimensions=3)
+ await store.insert(
+ "kb-1",
+ [
+ _make_record("Near!", [1.0, 0.0, 0.0], document_id="doc-1"),
+ _make_record("Far!", [10.0, 0.0, 0.0], document_id="doc-2"),
+ ],
+ )
+
+ results = await store.search(
+ "kb-1",
+ query_vector=[0.0, 0.0, 0.0],
+ top_k=2,
+ )
+
+ # Negated distances, so the nearest record still ranks first
+ self.assertEqual(
+ _dump_results(results),
+ [
+ {
+ "score": -1.0,
+ "document_id": "doc-1",
+ "chunk": {
+ "content": {
+ "type": "text",
+ "text": "Near!",
+ "id": AnyString(),
+ "created_at": AnyString(),
+ "finished_at": None,
+ },
+ "source": "doc-1.txt",
+ "chunk_index": 0,
+ "total_chunks": 1,
+ "metadata": {},
+ },
+ },
+ {
+ "score": -10.0,
+ "document_id": "doc-2",
+ "chunk": {
+ "content": {
+ "type": "text",
+ "text": "Far!",
+ "id": AnyString(),
+ "created_at": AnyString(),
+ "finished_at": None,
+ },
+ "source": "doc-2.txt",
+ "chunk_index": 0,
+ "total_chunks": 1,
+ "metadata": {},
+ },
+ },
+ ],
+ )
+
async def test_search_top_k(self) -> None:
"""top_k limits the number of returned results."""
await self.store.create_collection("kb-1", dimensions=3)