Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(retrieval): bound keyword-only BM25 scores before rerank (#3343)
wizardchend8be87d94082
diff --git a/internal/application/service/knowledgebase_search_fusion.go b/internal/application/service/knowledgebase_search_fusion.goindex 9c13eb8f5..b88702981 100644--- a/internal/application/service/knowledgebase_search_fusion.go+++ b/internal/application/service/knowledgebase_search_fusion.go@@ -2,7 +2,7 @@ package serviceimport ("context"-+ "math""slices""github.com/Tencent/WeKnora/internal/logger"@@ -38,8 +38,11 @@ func fuseOrDeduplicate(ctx context.Context, vectorResults, keywordResults []*typreturn result}if len(vectorResults) == 0 {- // Keyword-only: keep original scores (important for FAQ)+ // Keyword-only: keep relative BM25 order, but fold unbounded+ // scores into [0, 1] before they reach rerank/MMR. Raw BM25+ // (often >10) saturates compositeScore's 0.3*base term.result := deduplicateByScore(keywordResults)+ rescaleUnboundedScores(result)logger.Infof(ctx, "Result count after deduplication: %d", len(result))return result}@@ -77,6 +80,48 @@ func deduplicateByScore(results []*types.IndexWithScore) []*types.IndexWithScorereturn deduped}+// rescaleUnboundedScores maps a single-retriever candidate set onto [0, 1]+// by dividing through the maximum finite score. Keyword (BM25) scores are+// unbounded; leaving them in place saturates compositeScore (0.3*base with+// a [0, 1] clamp) so every candidate ties at 1.0 and MMR degenerates to+// diversity-only ordering.+//+// Scores already in [0, 1] are left unchanged so engines that stamp+// keyword hits at 1.0 (Qdrant, Doris) and already-normalized vector+// scores keep their current magnitude. Relative order is preserved+// either way. Retrieve/Langfuse sampling happens before fusion, so+// raw BM25 remains visible in the retrieve span.+func rescaleUnboundedScores(results []*types.IndexWithScore) {+ maxScore := 0.0+ for _, r := range results {+ if r == nil {+ continue+ }+ if math.IsNaN(r.Score) || math.IsInf(r.Score, 0) {+ continue+ }+ if r.Score > maxScore {+ maxScore = r.Score+ }+ }+ if maxScore <= 1 {+ return+ }+ for _, r := range results {+ if r == nil {+ continue+ }+ switch {+ case math.IsNaN(r.Score), math.IsInf(r.Score, -1), r.Score <= 0:+ r.Score = 0+ case math.IsInf(r.Score, 1):+ r.Score = 1+ default:+ r.Score = r.Score / maxScore+ }+ }+}+// fuseWithRRF merges vector and keyword retrieval results using Reciprocal Rank Fusion.// RRF score = vectorWeight/(k+vectorRank) + keywordWeight/(k+keywordRank).// k, vectorWeight and keywordWeight are sourced from retrievalCfg (with defaults).diff --git a/internal/application/service/knowledgebase_search_fusion_test.go b/internal/application/service/knowledgebase_search_fusion_test.gonew file mode 100644index 000000000..0bc352498--- /dev/null+++ b/internal/application/service/knowledgebase_search_fusion_test.go@@ -0,0 +1,95 @@+package service++import (+ "context"+ "math"+ "testing"++ "github.com/Tencent/WeKnora/internal/types"+ "github.com/stretchr/testify/require"+)++func TestFuseOrDeduplicate_KeywordOnlyRescalesUnboundedBM25(t *testing.T) {+ t.Parallel()++ got := fuseOrDeduplicate(context.Background(), nil, []*types.IndexWithScore{+ {ChunkID: "strong", Score: 16.1239},+ {ChunkID: "mid", Score: 8.06195},+ {ChunkID: "weak", Score: 4.030975},+ }, nil)++ require.Len(t, got, 3)+ require.Equal(t, "strong", got[0].ChunkID)+ require.InDelta(t, 1.0, got[0].Score, 1e-9)+ require.InDelta(t, 0.5, got[1].Score, 1e-9)+ require.InDelta(t, 0.25, got[2].Score, 1e-9)++ // 0.3*base stays below the composite clamp, so model score can still discriminate.+ require.Less(t, 0.3*got[0].Score, 1.0)+ require.Less(t, 0.3*got[1].Score, 0.3*got[0].Score)+}++func TestFuseOrDeduplicate_KeywordOnlyLeavesUnitIntervalScores(t *testing.T) {+ t.Parallel()++ flat := fuseOrDeduplicate(context.Background(), nil, []*types.IndexWithScore{+ {ChunkID: "a", Score: 1.0},+ {ChunkID: "b", Score: 1.0},+ }, nil)+ require.Len(t, flat, 2)+ require.InDelta(t, 1.0, flat[0].Score, 1e-9)+ require.InDelta(t, 1.0, flat[1].Score, 1e-9)++ bounded := fuseOrDeduplicate(context.Background(), nil, []*types.IndexWithScore{+ {ChunkID: "high", Score: 0.8},+ {ChunkID: "low", Score: 0.4},+ }, nil)+ require.Equal(t, "high", bounded[0].ChunkID)+ require.InDelta(t, 0.8, bounded[0].Score, 1e-9)+ require.InDelta(t, 0.4, bounded[1].Score, 1e-9)+}++func TestFuseOrDeduplicate_VectorOnlyKeepsEmbeddingScores(t *testing.T) {+ t.Parallel()++ got := fuseOrDeduplicate(context.Background(), []*types.IndexWithScore{+ {ChunkID: "near", Score: 0.91},+ {ChunkID: "far", Score: 0.22},+ }, nil, nil)++ require.Equal(t, "near", got[0].ChunkID)+ require.InDelta(t, 0.91, got[0].Score, 1e-9)+ require.InDelta(t, 0.22, got[1].Score, 1e-9)+}++func TestFuseOrDeduplicate_HybridUsesRRFNotRawBM25(t *testing.T) {+ t.Parallel()++ got := fuseOrDeduplicate(context.Background(),+ []*types.IndexWithScore{{ChunkID: "vec", Score: 0.9}},+ []*types.IndexWithScore{{ChunkID: "kw", Score: 16.1239}},+ nil,+ )++ require.Len(t, got, 2)+ for _, hit := range got {+ require.Greater(t, hit.Score, 0.0)+ require.Less(t, hit.Score, 1.0)+ require.NotEqual(t, 16.1239, hit.Score)+ }+}++func TestRescaleUnboundedScores_IgnoresNonFiniteWhenFindingMax(t *testing.T) {+ t.Parallel()++ hits := []*types.IndexWithScore{+ {ChunkID: "nan", Score: math.NaN()},+ {ChunkID: "top", Score: 10},+ {ChunkID: "low", Score: 5},+ nil,+ }+ rescaleUnboundedScores(hits)+ require.Equal(t, 0.0, hits[0].Score)+ require.InDelta(t, 1.0, hits[1].Score, 1e-9)+ require.InDelta(t, 0.5, hits[2].Score, 1e-9)+}