Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(milvus): preserve keyword retrieval scores
Jonesxq38fee1e1f721
diff --git a/internal/application/repository/retriever/milvus/repository.go b/internal/application/repository/retriever/milvus/repository.goindex 04f6a0fc7..ce39fe573 100644--- a/internal/application/repository/retriever/milvus/repository.go+++ b/internal/application/repository/retriever/milvus/repository.go@@ -795,10 +795,10 @@ func (m *milvusRepository) VectorRetrieve(ctx context.Context,log.Errorf("[Milvus] Failed to convert result set: %v", err)return nil, fmt.Errorf("failed to convert result set: %w", err)}- var results []*types.IndexWithScore- for i, set := range sets {- set.Score = scores[i]- results = append(results, fromMilvusVectorEmbedding(set.ID, set, types.MatchTypeEmbedding))+ results, err := buildMilvusIndexResults(sets, scores, types.MatchTypeEmbedding)+ if err != nil {+ log.Errorf("[Milvus] Failed to attach vector scores: %v", err)+ return nil, fmt.Errorf("failed to attach vector scores: %w", err)}if len(results) == 0 {log.Warnf("[Milvus] No vector matches found that meet threshold %.4f", params.Threshold)@@ -864,19 +864,41 @@ func (m *milvusRepository) KeywordsRetrieve(ctx context.Context,log.Errorf("[Milvus] Keywords search failed: %v", err)continue}- sets, _, err := convertResultSet(resultSet)+ sets, scores, err := convertResultSet(resultSet)if err != nil {log.Errorf("[Milvus] Failed to convert result set: %v", err)continue}- for _, set := range sets {- set.Score = 1.0- allResults = append(allResults, fromMilvusVectorEmbedding(set.ID, set, types.MatchTypeKeywords))+ results, scoreErr := buildMilvusIndexResults(sets, scores, types.MatchTypeKeywords)+ if scoreErr != nil {+ log.Errorf("[Milvus] Failed to attach keyword scores: %v", scoreErr)+ continue}+ allResults = append(allResults, results...)}- // Limit results to topK- if len(allResults) > params.TopK {+ // Searches across multiple collections return one score-sorted page per+ // collection. Re-sort the combined list before applying the global TopK;+ // otherwise the first collection can crowd out better matches from later+ // collections.+ slices.SortStableFunc(allResults, func(a, b *types.IndexWithScore) int {+ if a.Score > b.Score {+ return -1+ }+ if a.Score < b.Score {+ return 1+ }+ if a.ChunkID < b.ChunkID {+ return -1+ }+ if a.ChunkID > b.ChunkID {+ return 1+ }+ return 0+ })++ // Limit results to topK after sorting the merged collection results.+ if params.TopK > 0 && len(allResults) > params.TopK {allResults = allResults[:params.TopK]}@@ -1011,6 +1033,35 @@ func buildRetrieveResult(results []*types.IndexWithScore, retrieverType types.Re}}+// buildMilvusIndexResults attaches the score returned by Milvus to the+// corresponding document. Search scores are meaningful for both vector and+// BM25 searches; replacing keyword scores with a constant destroys the+// ordering and makes retrieval observability misleading.+func buildMilvusIndexResults(+ documents []*MilvusVectorEmbeddingWithScore,+ scores []float64,+ matchType types.MatchType,+) ([]*types.IndexWithScore, error) {+ if len(documents) != len(scores) {+ return nil, fmt.Errorf(+ "result and score count mismatch: documents=%d scores=%d",+ len(documents), len(scores),+ )+ }++ results := make([]*types.IndexWithScore, 0, len(documents))+ for i, document := range documents {+ if document == nil {+ return nil, fmt.Errorf("nil result at index %d", i)+ }+ document.Score = scores[i]+ results = append(results,+ fromMilvusVectorEmbedding(document.ID, document, matchType),+ )+ }+ return results, nil+}+func (m *milvusRepository) calculateStorageSize(embedding *MilvusVectorEmbedding) int64 {// Payload fieldspayloadSizeBytes := int64(0)diff --git a/internal/application/repository/retriever/milvus/repository_test.go b/internal/application/repository/retriever/milvus/repository_test.goindex b8be6905e..b8d220479 100644--- a/internal/application/repository/retriever/milvus/repository_test.go+++ b/internal/application/repository/retriever/milvus/repository_test.go@@ -5,9 +5,41 @@ import ("errors""testing"+ "github.com/Tencent/WeKnora/internal/types""github.com/stretchr/testify/require")+func TestBuildMilvusIndexResultsPreservesSearchScores(t *testing.T) {+ documents := []*MilvusVectorEmbeddingWithScore{+ {MilvusVectorEmbedding: MilvusVectorEmbedding{ID: "id-1", ChunkID: "chunk-1"}},+ {MilvusVectorEmbedding: MilvusVectorEmbedding{ID: "id-2", ChunkID: "chunk-2"}},+ }++ results, err := buildMilvusIndexResults(+ documents,+ []float64{3.25, 0.75},+ types.MatchTypeKeywords,+ )++ require.NoError(t, err)+ require.Len(t, results, 2)+ require.Equal(t, types.MatchTypeKeywords, results[0].MatchType)+ require.Equal(t, 3.25, results[0].Score)+ require.Equal(t, 0.75, results[1].Score)+}++func TestBuildMilvusIndexResultsRejectsScoreCountMismatch(t *testing.T) {+ _, err := buildMilvusIndexResults(+ []*MilvusVectorEmbeddingWithScore{+ {MilvusVectorEmbedding: MilvusVectorEmbedding{ID: "id-1"}},+ },+ nil,+ types.MatchTypeKeywords,+ )++ require.Error(t, err)+}+func TestUpdateChunkEnabledStatusInCollectionSkipsEmptyChunkIDs(t *testing.T) {repo := &milvusRepository{}diff --git a/internal/tracing/langfuse/retrieval_obs.go b/internal/tracing/langfuse/retrieval_obs.goindex 591560964..d103109ab 100644--- a/internal/tracing/langfuse/retrieval_obs.go+++ b/internal/tracing/langfuse/retrieval_obs.go@@ -24,19 +24,20 @@ func TruncateRunes(s string, maxRunes int) string {// SummarizeRetrieveOutput builds Langfuse output for the retrieve span.func SummarizeRetrieveOutput(results []*types.RetrieveResult) map[string]interface{} {out := map[string]interface{}{- "total_hits": 0,- "vector_hits": 0,- "keyword_hits": 0,- "group_count": len(results),- "by_retriever": []map[string]interface{}{},- "top_hits": []map[string]interface{}{},+ "total_hits": 0,+ "vector_hits": 0,+ "keyword_hits": 0,+ "group_count": len(results),+ "by_retriever": []map[string]interface{}{},+ "top_hits": []map[string]interface{}{},+ "top_hits_by_retriever": []map[string]interface{}{},+ "top_hits_strategy": "round_robin_by_retriever",}if len(results) == 0 {return out}byRetriever := make([]map[string]interface{}, 0, len(results))- var all []*types.IndexWithScorefor _, rr := range results {if rr == nil {continue@@ -53,10 +54,11 @@ func SummarizeRetrieveOutput(results []*types.RetrieveResult) map[string]interfa"retriever": string(rr.RetrieverType),"count": count,})- all = append(all, rr.Results...)}out["by_retriever"] = byRetriever- out["top_hits"] = summarizeIndexHits(all, defaultHitPreviewLimit)+ topHits, topHitsByRetriever := summarizeRetrieveHits(results, defaultHitPreviewLimit)+ out["top_hits"] = topHits+ out["top_hits_by_retriever"] = topHitsByRetrieverreturn out}@@ -164,6 +166,101 @@ func summarizeIndexHits(hits []*types.IndexWithScore, limit int) []map[string]inreturn out}+type retrieveHitGroup struct {+ engine types.RetrieverEngineType+ retriever types.RetrieverType+ hits []*types.IndexWithScore+}++type retrieveHitRef struct {+ engine types.RetrieverEngineType+ retriever types.RetrieverType+ hit *types.IndexWithScore+}++// summarizeRetrieveHits keeps the raw retrieve trace useful when score+// scales differ between retrievers. Vector similarity and BM25 scores are+// not directly comparable, so a global score sort can hide one retriever+// completely. The flat top_hits list is therefore selected round-robin from+// each retriever group, while top_hits_by_retriever preserves each group's+// own score ordering for detailed inspection.+func summarizeRetrieveHits(+ results []*types.RetrieveResult,+ limit int,+) ([]map[string]interface{}, []map[string]interface{}) {+ if limit <= 0 {+ limit = defaultHitPreviewLimit+ }++ groups := make([]retrieveHitGroup, 0, len(results))+ for _, rr := range results {+ if rr == nil || len(rr.Results) == 0 {+ continue+ }+ hits := append([]*types.IndexWithScore(nil), rr.Results...)+ sort.SliceStable(hits, func(i, j int) bool {+ if hits[i].Score != hits[j].Score {+ return hits[i].Score > hits[j].Score+ }+ return hits[i].ChunkID < hits[j].ChunkID+ })+ groups = append(groups, retrieveHitGroup{+ engine: rr.RetrieverEngineType,+ retriever: rr.RetrieverType,+ hits: hits,+ })+ }++ byRetriever := make([]map[string]interface{}, 0, len(groups))+ refs := make([]retrieveHitRef, 0, minInt(limit, len(results)))+ for _, group := range groups {+ byRetriever = append(byRetriever, map[string]interface{}{+ "engine": string(group.engine),+ "retriever": string(group.retriever),+ "count": len(group.hits),+ "top_hits": summarizeIndexHits(group.hits, limit),+ })+ }++ for offset := 0; len(refs) < limit; offset++ {+ added := false+ for _, group := range groups {+ if offset >= len(group.hits) {+ continue+ }+ refs = append(refs, retrieveHitRef{+ engine: group.engine,+ retriever: group.retriever,+ hit: group.hits[offset],+ })+ added = true+ if len(refs) == limit {+ break+ }+ }+ if !added {+ break+ }+ }++ topHits := make([]map[string]interface{}, 0, len(refs))+ for i, ref := range refs {+ hit := ref.hit+ topHits = append(topHits, map[string]interface{}{+ "rank": i + 1,+ "chunk_id": hit.ChunkID,+ "knowledge_id": hit.KnowledgeID,+ "knowledge_base_id": hit.KnowledgeBaseID,+ "score": fmt.Sprintf("%.4f", hit.Score),+ "match_type": hit.MatchType,+ "engine": string(ref.engine),+ "retriever": string(ref.retriever),+ "preview": TruncateRunes(hit.Content, 160),+ })+ }+ return topHits, byRetriever+}+// SummarizePassagePreviews builds rerank passage previews aligned with candidates.func SummarizePassagePreviews(candidates []*types.SearchResult,diff --git a/internal/tracing/langfuse/retrieval_obs_test.go b/internal/tracing/langfuse/retrieval_obs_test.goindex a9e2913f6..1d697b409 100644--- a/internal/tracing/langfuse/retrieval_obs_test.go+++ b/internal/tracing/langfuse/retrieval_obs_test.go@@ -1,6 +1,7 @@package langfuseimport (+ "fmt""testing""github.com/Tencent/WeKnora/internal/types"@@ -35,6 +36,48 @@ func TestSummarizeRetrieveOutput_countsAndPreviews(t *testing.T) {if len(hits) != 3 || hits[0]["chunk_id"] != "c1" {t.Fatalf("unexpected top_hits: %#v", hits)}+ if hits[0]["retriever"] != string(types.VectorRetrieverType) ||+ hits[1]["retriever"] != string(types.KeywordsRetrieverType) {+ t.Fatalf("top_hits should retain both retriever groups: %#v", hits)+ }+ if out["top_hits_strategy"] != "round_robin_by_retriever" {+ t.Fatalf("unexpected top_hits strategy: %v", out["top_hits_strategy"])+ }+}++func TestSummarizeRetrieveOutput_doesNotLetOneScoreScaleHideAnotherRetriever(t *testing.T) {+ vector := make([]*types.IndexWithScore, 0, 5)+ keyword := make([]*types.IndexWithScore, 0, 5)+ for i := 0; i < 5; i++ {+ vector = append(vector, &types.IndexWithScore{+ ChunkID: fmt.Sprintf("vector-%d", i),+ Score: 0.9 - float64(i)*0.1,+ MatchType: types.MatchTypeEmbedding,+ })+ keyword = append(keyword, &types.IndexWithScore{+ ChunkID: fmt.Sprintf("keyword-%d", i),+ Score: 1.0,+ MatchType: types.MatchTypeKeywords,+ })+ }++ out := SummarizeRetrieveOutput([]*types.RetrieveResult{+ {RetrieverType: types.VectorRetrieverType, Results: vector},+ {RetrieverType: types.KeywordsRetrieverType, Results: keyword},+ })+ hits := out["top_hits"].([]map[string]interface{})+ seen := map[string]bool{}+ for _, hit := range hits {+ seen[hit["retriever"].(string)] = true+ }+ if !seen[string(types.VectorRetrieverType)] || !seen[string(types.KeywordsRetrieverType)] {+ t.Fatalf("top_hits lost a retriever group: %#v", hits)+ }++ groups := out["top_hits_by_retriever"].([]map[string]interface{})+ if len(groups) != 2 {+ t.Fatalf("top_hits_by_retriever groups = %d, want 2", len(groups))+ }}func TestSummarizeSearchResults_sortsByScore(t *testing.T) {