Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(retrieval): bound keyword-only BM25 scores before rerank (#3343)

wizardchend8be87d94082
diff --git a/internal/application/service/knowledgebase_search_fusion.go b/internal/application/service/knowledgebase_search_fusion.go
index 9c13eb8f5..b88702981 100644
--- a/internal/application/service/knowledgebase_search_fusion.go
+++ b/internal/application/service/knowledgebase_search_fusion.go
@@ -2,7 +2,7 @@ package service
import (
"context"
-
+ "math"
"slices"
"github.com/Tencent/WeKnora/internal/logger"
@@ -38,8 +38,11 @@ func fuseOrDeduplicate(ctx context.Context, vectorResults, keywordResults []*typ
return result
}
if len(vectorResults) == 0 {
- // Keyword-only: keep original scores (important for FAQ)
+ // Keyword-only: keep relative BM25 order, but fold unbounded
+ // scores into [0, 1] before they reach rerank/MMR. Raw BM25
+ // (often >10) saturates compositeScore's 0.3*base term.
result := deduplicateByScore(keywordResults)
+ rescaleUnboundedScores(result)
logger.Infof(ctx, "Result count after deduplication: %d", len(result))
return result
}
@@ -77,6 +80,48 @@ func deduplicateByScore(results []*types.IndexWithScore) []*types.IndexWithScore
return deduped
}
+// rescaleUnboundedScores maps a single-retriever candidate set onto [0, 1]
+// by dividing through the maximum finite score. Keyword (BM25) scores are
+// unbounded; leaving them in place saturates compositeScore (0.3*base with
+// a [0, 1] clamp) so every candidate ties at 1.0 and MMR degenerates to
+// diversity-only ordering.
+//
+// Scores already in [0, 1] are left unchanged so engines that stamp
+// keyword hits at 1.0 (Qdrant, Doris) and already-normalized vector
+// scores keep their current magnitude. Relative order is preserved
+// either way. Retrieve/Langfuse sampling happens before fusion, so
+// raw BM25 remains visible in the retrieve span.
+func rescaleUnboundedScores(results []*types.IndexWithScore) {
+ maxScore := 0.0
+ for _, r := range results {
+ if r == nil {
+ continue
+ }
+ if math.IsNaN(r.Score) || math.IsInf(r.Score, 0) {
+ continue
+ }
+ if r.Score > maxScore {
+ maxScore = r.Score
+ }
+ }
+ if maxScore <= 1 {
+ return
+ }
+ for _, r := range results {
+ if r == nil {
+ continue
+ }
+ switch {
+ case math.IsNaN(r.Score), math.IsInf(r.Score, -1), r.Score <= 0:
+ r.Score = 0
+ case math.IsInf(r.Score, 1):
+ r.Score = 1
+ default:
+ r.Score = r.Score / maxScore
+ }
+ }
+}
+
// fuseWithRRF merges vector and keyword retrieval results using Reciprocal Rank Fusion.
// RRF score = vectorWeight/(k+vectorRank) + keywordWeight/(k+keywordRank).
// k, vectorWeight and keywordWeight are sourced from retrievalCfg (with defaults).
diff --git a/internal/application/service/knowledgebase_search_fusion_test.go b/internal/application/service/knowledgebase_search_fusion_test.go
new file mode 100644
index 000000000..0bc352498
--- /dev/null
+++ b/internal/application/service/knowledgebase_search_fusion_test.go
@@ -0,0 +1,95 @@
+package service
+
+import (
+ "context"
+ "math"
+ "testing"
+
+ "github.com/Tencent/WeKnora/internal/types"
+ "github.com/stretchr/testify/require"
+)
+
+func TestFuseOrDeduplicate_KeywordOnlyRescalesUnboundedBM25(t *testing.T) {
+ t.Parallel()
+
+ got := fuseOrDeduplicate(context.Background(), nil, []*types.IndexWithScore{
+ {ChunkID: "strong", Score: 16.1239},
+ {ChunkID: "mid", Score: 8.06195},
+ {ChunkID: "weak", Score: 4.030975},
+ }, nil)
+
+ require.Len(t, got, 3)
+ require.Equal(t, "strong", got[0].ChunkID)
+ require.InDelta(t, 1.0, got[0].Score, 1e-9)
+ require.InDelta(t, 0.5, got[1].Score, 1e-9)
+ require.InDelta(t, 0.25, got[2].Score, 1e-9)
+
+ // 0.3*base stays below the composite clamp, so model score can still discriminate.
+ require.Less(t, 0.3*got[0].Score, 1.0)
+ require.Less(t, 0.3*got[1].Score, 0.3*got[0].Score)
+}
+
+func TestFuseOrDeduplicate_KeywordOnlyLeavesUnitIntervalScores(t *testing.T) {
+ t.Parallel()
+
+ flat := fuseOrDeduplicate(context.Background(), nil, []*types.IndexWithScore{
+ {ChunkID: "a", Score: 1.0},
+ {ChunkID: "b", Score: 1.0},
+ }, nil)
+ require.Len(t, flat, 2)
+ require.InDelta(t, 1.0, flat[0].Score, 1e-9)
+ require.InDelta(t, 1.0, flat[1].Score, 1e-9)
+
+ bounded := fuseOrDeduplicate(context.Background(), nil, []*types.IndexWithScore{
+ {ChunkID: "high", Score: 0.8},
+ {ChunkID: "low", Score: 0.4},
+ }, nil)
+ require.Equal(t, "high", bounded[0].ChunkID)
+ require.InDelta(t, 0.8, bounded[0].Score, 1e-9)
+ require.InDelta(t, 0.4, bounded[1].Score, 1e-9)
+}
+
+func TestFuseOrDeduplicate_VectorOnlyKeepsEmbeddingScores(t *testing.T) {
+ t.Parallel()
+
+ got := fuseOrDeduplicate(context.Background(), []*types.IndexWithScore{
+ {ChunkID: "near", Score: 0.91},
+ {ChunkID: "far", Score: 0.22},
+ }, nil, nil)
+
+ require.Equal(t, "near", got[0].ChunkID)
+ require.InDelta(t, 0.91, got[0].Score, 1e-9)
+ require.InDelta(t, 0.22, got[1].Score, 1e-9)
+}
+
+func TestFuseOrDeduplicate_HybridUsesRRFNotRawBM25(t *testing.T) {
+ t.Parallel()
+
+ got := fuseOrDeduplicate(context.Background(),
+ []*types.IndexWithScore{{ChunkID: "vec", Score: 0.9}},
+ []*types.IndexWithScore{{ChunkID: "kw", Score: 16.1239}},
+ nil,
+ )
+
+ require.Len(t, got, 2)
+ for _, hit := range got {
+ require.Greater(t, hit.Score, 0.0)
+ require.Less(t, hit.Score, 1.0)
+ require.NotEqual(t, 16.1239, hit.Score)
+ }
+}
+
+func TestRescaleUnboundedScores_IgnoresNonFiniteWhenFindingMax(t *testing.T) {
+ t.Parallel()
+
+ hits := []*types.IndexWithScore{
+ {ChunkID: "nan", Score: math.NaN()},
+ {ChunkID: "top", Score: 10},
+ {ChunkID: "low", Score: 5},
+ nil,
+ }
+ rescaleUnboundedScores(hits)
+ require.Equal(t, 0.0, hits[0].Score)
+ require.InDelta(t, 1.0, hits[1].Score, 1e-9)
+ require.InDelta(t, 0.5, hits[2].Score, 1e-9)
+}