Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(milvus): preserve keyword retrieval scores

Jonesxq38fee1e1f721
diff --git a/internal/application/repository/retriever/milvus/repository.go b/internal/application/repository/retriever/milvus/repository.go
index 04f6a0fc7..ce39fe573 100644
--- a/internal/application/repository/retriever/milvus/repository.go
+++ b/internal/application/repository/retriever/milvus/repository.go
@@ -795,10 +795,10 @@ func (m *milvusRepository) VectorRetrieve(ctx context.Context,
log.Errorf("[Milvus] Failed to convert result set: %v", err)
return nil, fmt.Errorf("failed to convert result set: %w", err)
}
- var results []*types.IndexWithScore
- for i, set := range sets {
- set.Score = scores[i]
- results = append(results, fromMilvusVectorEmbedding(set.ID, set, types.MatchTypeEmbedding))
+ results, err := buildMilvusIndexResults(sets, scores, types.MatchTypeEmbedding)
+ if err != nil {
+ log.Errorf("[Milvus] Failed to attach vector scores: %v", err)
+ return nil, fmt.Errorf("failed to attach vector scores: %w", err)
}
if len(results) == 0 {
log.Warnf("[Milvus] No vector matches found that meet threshold %.4f", params.Threshold)
@@ -864,19 +864,41 @@ func (m *milvusRepository) KeywordsRetrieve(ctx context.Context,
log.Errorf("[Milvus] Keywords search failed: %v", err)
continue
}
- sets, _, err := convertResultSet(resultSet)
+ sets, scores, err := convertResultSet(resultSet)
if err != nil {
log.Errorf("[Milvus] Failed to convert result set: %v", err)
continue
}
- for _, set := range sets {
- set.Score = 1.0
- allResults = append(allResults, fromMilvusVectorEmbedding(set.ID, set, types.MatchTypeKeywords))
+ results, scoreErr := buildMilvusIndexResults(sets, scores, types.MatchTypeKeywords)
+ if scoreErr != nil {
+ log.Errorf("[Milvus] Failed to attach keyword scores: %v", scoreErr)
+ continue
}
+ allResults = append(allResults, results...)
}
- // Limit results to topK
- if len(allResults) > params.TopK {
+ // Searches across multiple collections return one score-sorted page per
+ // collection. Re-sort the combined list before applying the global TopK;
+ // otherwise the first collection can crowd out better matches from later
+ // collections.
+ slices.SortStableFunc(allResults, func(a, b *types.IndexWithScore) int {
+ if a.Score > b.Score {
+ return -1
+ }
+ if a.Score < b.Score {
+ return 1
+ }
+ if a.ChunkID < b.ChunkID {
+ return -1
+ }
+ if a.ChunkID > b.ChunkID {
+ return 1
+ }
+ return 0
+ })
+
+ // Limit results to topK after sorting the merged collection results.
+ if params.TopK > 0 && len(allResults) > params.TopK {
allResults = allResults[:params.TopK]
}
@@ -1011,6 +1033,35 @@ func buildRetrieveResult(results []*types.IndexWithScore, retrieverType types.Re
}
}
+// buildMilvusIndexResults attaches the score returned by Milvus to the
+// corresponding document. Search scores are meaningful for both vector and
+// BM25 searches; replacing keyword scores with a constant destroys the
+// ordering and makes retrieval observability misleading.
+func buildMilvusIndexResults(
+ documents []*MilvusVectorEmbeddingWithScore,
+ scores []float64,
+ matchType types.MatchType,
+) ([]*types.IndexWithScore, error) {
+ if len(documents) != len(scores) {
+ return nil, fmt.Errorf(
+ "result and score count mismatch: documents=%d scores=%d",
+ len(documents), len(scores),
+ )
+ }
+
+ results := make([]*types.IndexWithScore, 0, len(documents))
+ for i, document := range documents {
+ if document == nil {
+ return nil, fmt.Errorf("nil result at index %d", i)
+ }
+ document.Score = scores[i]
+ results = append(results,
+ fromMilvusVectorEmbedding(document.ID, document, matchType),
+ )
+ }
+ return results, nil
+}
+
func (m *milvusRepository) calculateStorageSize(embedding *MilvusVectorEmbedding) int64 {
// Payload fields
payloadSizeBytes := int64(0)
diff --git a/internal/application/repository/retriever/milvus/repository_test.go b/internal/application/repository/retriever/milvus/repository_test.go
index b8be6905e..b8d220479 100644
--- a/internal/application/repository/retriever/milvus/repository_test.go
+++ b/internal/application/repository/retriever/milvus/repository_test.go
@@ -5,9 +5,41 @@ import (
"errors"
"testing"
+ "github.com/Tencent/WeKnora/internal/types"
"github.com/stretchr/testify/require"
)
+func TestBuildMilvusIndexResultsPreservesSearchScores(t *testing.T) {
+ documents := []*MilvusVectorEmbeddingWithScore{
+ {MilvusVectorEmbedding: MilvusVectorEmbedding{ID: "id-1", ChunkID: "chunk-1"}},
+ {MilvusVectorEmbedding: MilvusVectorEmbedding{ID: "id-2", ChunkID: "chunk-2"}},
+ }
+
+ results, err := buildMilvusIndexResults(
+ documents,
+ []float64{3.25, 0.75},
+ types.MatchTypeKeywords,
+ )
+
+ require.NoError(t, err)
+ require.Len(t, results, 2)
+ require.Equal(t, types.MatchTypeKeywords, results[0].MatchType)
+ require.Equal(t, 3.25, results[0].Score)
+ require.Equal(t, 0.75, results[1].Score)
+}
+
+func TestBuildMilvusIndexResultsRejectsScoreCountMismatch(t *testing.T) {
+ _, err := buildMilvusIndexResults(
+ []*MilvusVectorEmbeddingWithScore{
+ {MilvusVectorEmbedding: MilvusVectorEmbedding{ID: "id-1"}},
+ },
+ nil,
+ types.MatchTypeKeywords,
+ )
+
+ require.Error(t, err)
+}
+
func TestUpdateChunkEnabledStatusInCollectionSkipsEmptyChunkIDs(t *testing.T) {
repo := &milvusRepository{}
diff --git a/internal/tracing/langfuse/retrieval_obs.go b/internal/tracing/langfuse/retrieval_obs.go
index 591560964..d103109ab 100644
--- a/internal/tracing/langfuse/retrieval_obs.go
+++ b/internal/tracing/langfuse/retrieval_obs.go
@@ -24,19 +24,20 @@ func TruncateRunes(s string, maxRunes int) string {
// SummarizeRetrieveOutput builds Langfuse output for the retrieve span.
func SummarizeRetrieveOutput(results []*types.RetrieveResult) map[string]interface{} {
out := map[string]interface{}{
- "total_hits": 0,
- "vector_hits": 0,
- "keyword_hits": 0,
- "group_count": len(results),
- "by_retriever": []map[string]interface{}{},
- "top_hits": []map[string]interface{}{},
+ "total_hits": 0,
+ "vector_hits": 0,
+ "keyword_hits": 0,
+ "group_count": len(results),
+ "by_retriever": []map[string]interface{}{},
+ "top_hits": []map[string]interface{}{},
+ "top_hits_by_retriever": []map[string]interface{}{},
+ "top_hits_strategy": "round_robin_by_retriever",
}
if len(results) == 0 {
return out
}
byRetriever := make([]map[string]interface{}, 0, len(results))
- var all []*types.IndexWithScore
for _, rr := range results {
if rr == nil {
continue
@@ -53,10 +54,11 @@ func SummarizeRetrieveOutput(results []*types.RetrieveResult) map[string]interfa
"retriever": string(rr.RetrieverType),
"count": count,
})
- all = append(all, rr.Results...)
}
out["by_retriever"] = byRetriever
- out["top_hits"] = summarizeIndexHits(all, defaultHitPreviewLimit)
+ topHits, topHitsByRetriever := summarizeRetrieveHits(results, defaultHitPreviewLimit)
+ out["top_hits"] = topHits
+ out["top_hits_by_retriever"] = topHitsByRetriever
return out
}
@@ -164,6 +166,101 @@ func summarizeIndexHits(hits []*types.IndexWithScore, limit int) []map[string]in
return out
}
+type retrieveHitGroup struct {
+ engine types.RetrieverEngineType
+ retriever types.RetrieverType
+ hits []*types.IndexWithScore
+}
+
+type retrieveHitRef struct {
+ engine types.RetrieverEngineType
+ retriever types.RetrieverType
+ hit *types.IndexWithScore
+}
+
+// summarizeRetrieveHits keeps the raw retrieve trace useful when score
+// scales differ between retrievers. Vector similarity and BM25 scores are
+// not directly comparable, so a global score sort can hide one retriever
+// completely. The flat top_hits list is therefore selected round-robin from
+// each retriever group, while top_hits_by_retriever preserves each group's
+// own score ordering for detailed inspection.
+func summarizeRetrieveHits(
+ results []*types.RetrieveResult,
+ limit int,
+) ([]map[string]interface{}, []map[string]interface{}) {
+ if limit <= 0 {
+ limit = defaultHitPreviewLimit
+ }
+
+ groups := make([]retrieveHitGroup, 0, len(results))
+ for _, rr := range results {
+ if rr == nil || len(rr.Results) == 0 {
+ continue
+ }
+ hits := append([]*types.IndexWithScore(nil), rr.Results...)
+ sort.SliceStable(hits, func(i, j int) bool {
+ if hits[i].Score != hits[j].Score {
+ return hits[i].Score > hits[j].Score
+ }
+ return hits[i].ChunkID < hits[j].ChunkID
+ })
+ groups = append(groups, retrieveHitGroup{
+ engine: rr.RetrieverEngineType,
+ retriever: rr.RetrieverType,
+ hits: hits,
+ })
+ }
+
+ byRetriever := make([]map[string]interface{}, 0, len(groups))
+ refs := make([]retrieveHitRef, 0, minInt(limit, len(results)))
+ for _, group := range groups {
+ byRetriever = append(byRetriever, map[string]interface{}{
+ "engine": string(group.engine),
+ "retriever": string(group.retriever),
+ "count": len(group.hits),
+ "top_hits": summarizeIndexHits(group.hits, limit),
+ })
+ }
+
+ for offset := 0; len(refs) < limit; offset++ {
+ added := false
+ for _, group := range groups {
+ if offset >= len(group.hits) {
+ continue
+ }
+ refs = append(refs, retrieveHitRef{
+ engine: group.engine,
+ retriever: group.retriever,
+ hit: group.hits[offset],
+ })
+ added = true
+ if len(refs) == limit {
+ break
+ }
+ }
+ if !added {
+ break
+ }
+ }
+
+ topHits := make([]map[string]interface{}, 0, len(refs))
+ for i, ref := range refs {
+ hit := ref.hit
+ topHits = append(topHits, map[string]interface{}{
+ "rank": i + 1,
+ "chunk_id": hit.ChunkID,
+ "knowledge_id": hit.KnowledgeID,
+ "knowledge_base_id": hit.KnowledgeBaseID,
+ "score": fmt.Sprintf("%.4f", hit.Score),
+ "match_type": hit.MatchType,
+ "engine": string(ref.engine),
+ "retriever": string(ref.retriever),
+ "preview": TruncateRunes(hit.Content, 160),
+ })
+ }
+ return topHits, byRetriever
+}
+
// SummarizePassagePreviews builds rerank passage previews aligned with candidates.
func SummarizePassagePreviews(
candidates []*types.SearchResult,
diff --git a/internal/tracing/langfuse/retrieval_obs_test.go b/internal/tracing/langfuse/retrieval_obs_test.go
index a9e2913f6..1d697b409 100644
--- a/internal/tracing/langfuse/retrieval_obs_test.go
+++ b/internal/tracing/langfuse/retrieval_obs_test.go
@@ -1,6 +1,7 @@
package langfuse
import (
+ "fmt"
"testing"
"github.com/Tencent/WeKnora/internal/types"
@@ -35,6 +36,48 @@ func TestSummarizeRetrieveOutput_countsAndPreviews(t *testing.T) {
if len(hits) != 3 || hits[0]["chunk_id"] != "c1" {
t.Fatalf("unexpected top_hits: %#v", hits)
}
+ if hits[0]["retriever"] != string(types.VectorRetrieverType) ||
+ hits[1]["retriever"] != string(types.KeywordsRetrieverType) {
+ t.Fatalf("top_hits should retain both retriever groups: %#v", hits)
+ }
+ if out["top_hits_strategy"] != "round_robin_by_retriever" {
+ t.Fatalf("unexpected top_hits strategy: %v", out["top_hits_strategy"])
+ }
+}
+
+func TestSummarizeRetrieveOutput_doesNotLetOneScoreScaleHideAnotherRetriever(t *testing.T) {
+ vector := make([]*types.IndexWithScore, 0, 5)
+ keyword := make([]*types.IndexWithScore, 0, 5)
+ for i := 0; i < 5; i++ {
+ vector = append(vector, &types.IndexWithScore{
+ ChunkID: fmt.Sprintf("vector-%d", i),
+ Score: 0.9 - float64(i)*0.1,
+ MatchType: types.MatchTypeEmbedding,
+ })
+ keyword = append(keyword, &types.IndexWithScore{
+ ChunkID: fmt.Sprintf("keyword-%d", i),
+ Score: 1.0,
+ MatchType: types.MatchTypeKeywords,
+ })
+ }
+
+ out := SummarizeRetrieveOutput([]*types.RetrieveResult{
+ {RetrieverType: types.VectorRetrieverType, Results: vector},
+ {RetrieverType: types.KeywordsRetrieverType, Results: keyword},
+ })
+ hits := out["top_hits"].([]map[string]interface{})
+ seen := map[string]bool{}
+ for _, hit := range hits {
+ seen[hit["retriever"].(string)] = true
+ }
+ if !seen[string(types.VectorRetrieverType)] || !seen[string(types.KeywordsRetrieverType)] {
+ t.Fatalf("top_hits lost a retriever group: %#v", hits)
+ }
+
+ groups := out["top_hits_by_retriever"].([]map[string]interface{})
+ if len(groups) != 2 {
+ t.Fatalf("top_hits_by_retriever groups = %d, want 2", len(groups))
+ }
}
func TestSummarizeSearchResults_sortsByScore(t *testing.T) {