Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(reranker): add direct tests for rerank scoring mathFix/reranker and Fix two defects found while testing (#21485)

ZZZZY82bd3b2a581e
diff --git a/internal/service/nlp/reranker.go b/internal/service/nlp/reranker.go
index abbbee19ca..76b32c574a 100644
--- a/internal/service/nlp/reranker.go
+++ b/internal/service/nlp/reranker.go
@@ -18,6 +18,7 @@ import (
"context"
"encoding/json"
"math"
+ "slices"
"sort"
"strconv"
"strings"
@@ -134,19 +135,10 @@ func RerankByModel(
}
}
- contentLtks := extractContentTokens(chunk, cfield)
- titleTks := extractTitleTokens(chunk)
- importantKwd := extractImportantKeywords(chunk)
- questionTks := extractQuestionTokens(chunk)
-
// Unlike RerankStandard/RerankWithKNN, the fields are not repeated here:
// these tokens are joined back into `docs` for a cross-encoder, where
// duplicating a field would distort the model's own scoring.
- tks := make([]string, 0, len(contentLtks)+len(titleTks)+len(importantKwd)+len(questionTks))
- tks = append(tks, contentLtks...)
- tks = append(tks, titleTks...)
- tks = append(tks, importantKwd...)
- tks = append(tks, questionTks...)
+ tks := buildRerankTokens(chunk, cfield, 1, 1, 1)
insTw = append(insTw, tks)
// Feed the reranker the natural chunk text (markup preserved), not the
@@ -309,25 +301,8 @@ func RerankStandard(
chunkVector := extractVector(chunk, vectorColumn, zeroVector)
insEmbd = append(insEmbd, chunkVector)
- // Extract tokens
- contentLtks := extractContentTokens(chunk, cfield)
- titleTks := extractTitleTokens(chunk)
- questionTks := extractQuestionTokens(chunk)
- importantKwd := extractImportantKeywords(chunk)
-
- // Combine tokens with weights: content + title*2 + important_kwd*5 + question_tks*6
- tks := make([]string, 0, len(contentLtks)+len(titleTks)*2+len(importantKwd)*5+len(questionTks)*6)
- tks = append(tks, contentLtks...)
- for i := 0; i < 2; i++ {
- tks = append(tks, titleTks...)
- }
- for i := 0; i < 5; i++ {
- tks = append(tks, importantKwd...)
- }
- for i := 0; i < 6; i++ {
- tks = append(tks, questionTks...)
- }
- insTw = append(insTw, tks)
+ insTw = append(insTw, buildRerankTokens(
+ chunk, cfield, titleRepeat, importantRepeat, questionRepeat))
}
if len(insEmbd) == 0 {
@@ -399,7 +374,7 @@ func HybridSimilarity(
}
if allZero {
- return tsim, tsim, vsim
+ return slices.Clone(tsim), tsim, vsim
}
// Combine similarities
@@ -627,6 +602,44 @@ func extractImportantKeywords(fields map[string]interface{}) []string {
}
}
+// Field repetition multipliers for the token-based rerank paths. RerankByModel
+// passes 1 for all three: it joins the tokens into natural text for a
+// cross-encoder, where duplication would distort the model's own score.
+const (
+ titleRepeat = 2
+ importantRepeat = 5
+ questionRepeat = 6
+)
+
+// buildRerankTokens assembles content + title*titleRepeat +
+// important*importantRepeat + question*questionRepeat, in that order.
+// The returned slice is never nil.
+func buildRerankTokens(
+ chunk map[string]interface{},
+ cfield string,
+ titleRepeat, importantRepeat, questionRepeat int,
+) []string {
+ content := extractContentTokens(chunk, cfield)
+ title := extractTitleTokens(chunk)
+ important := extractImportantKeywords(chunk)
+ question := extractQuestionTokens(chunk)
+
+ tks := make([]string, 0,
+ len(content)+len(title)*titleRepeat+
+ len(important)*importantRepeat+len(question)*questionRepeat)
+ tks = append(tks, content...)
+ for i := 0; i < titleRepeat; i++ {
+ tks = append(tks, title...)
+ }
+ for i := 0; i < importantRepeat; i++ {
+ tks = append(tks, important...)
+ }
+ for i := 0; i < questionRepeat; i++ {
+ tks = append(tks, question...)
+ }
+ return tks
+}
+
// cosineSimilarity calculates cosine similarity between two vectors.
// Three parallel pairwise sums (dot, ||a||², ||b||²) keep precision
// comparable to numpy's float64 reductions.
@@ -665,6 +678,17 @@ func applyRankFeatureScores(chunks []map[string]interface{}, sim []float64, rank
if len(chunks) == 0 || len(sim) == 0 {
return sim
}
+ // Only the overlapping prefix carries chunk-level rank features; scores
+ // beyond the chunk list keep their input value. A length mismatch is an
+ // upstream bug, so log it instead of silently scoring part of the list.
+ if len(sim) != len(chunks) {
+ common.Warn("rank feature: sim/chunks length mismatch, applying rank features to the overlapping prefix only",
+ zap.Int("simLen", len(sim)), zap.Int("chunkLen", len(chunks)))
+ }
+ if len(sim) > len(chunks) {
+ applyRankFeatureScores(chunks, sim[:len(chunks)], rankFeature)
+ return sim
+ }
// Collect pageranks from each chunk
pageranks := make([]float64, len(chunks))
@@ -768,6 +792,17 @@ func applyRankFeatureScoresForIDs(ids []string, field map[string]map[string]inte
if len(ids) == 0 || len(sim) == 0 {
return sim
}
+ // Only the overlapping prefix carries chunk-level rank features; scores
+ // beyond the id list keep their input value. A length mismatch is an
+ // upstream bug, so log it instead of silently scoring part of the list.
+ if len(sim) != len(ids) {
+ common.Warn("rank feature: sim/ids length mismatch, applying rank features to the overlapping prefix only",
+ zap.Int("simLen", len(sim)), zap.Int("idLen", len(ids)))
+ }
+ if len(sim) > len(ids) {
+ applyRankFeatureScoresForIDs(ids, field, sim[:len(ids)], rankFeature)
+ return sim
+ }
// Collect pageranks from each chunk via field map
pageranks := make([]float64, len(ids))
@@ -970,25 +1005,8 @@ func RerankWithKNN(
}
// Normalize text content - split, dedupe while preserving order (OrderedDict effect)
- contentLtks := extractContentTokens(chunk, cfield)
- titleTks := extractTitleTokens(chunk)
- questionTks := extractQuestionTokens(chunk)
- importantKwd := extractImportantKeywords(chunk)
-
- // Combine tokens with weights: content + title*2 + important_kwd*5 + question_tks*6
- // This matches Python: tks = content_ltks + title_tks * 2 + important_kwd * 5 + question_tks * 6
- tks := make([]string, 0, len(contentLtks)+len(titleTks)*2+len(importantKwd)*5+len(questionTks)*6)
- tks = append(tks, contentLtks...)
- for i := 0; i < 2; i++ {
- tks = append(tks, titleTks...)
- }
- for i := 0; i < 5; i++ {
- tks = append(tks, importantKwd...)
- }
- for i := 0; i < 6; i++ {
- tks = append(tks, questionTks...)
- }
- insTw = append(insTw, tks)
+ insTw = append(insTw, buildRerankTokens(
+ chunk, cfield, titleRepeat, importantRepeat, questionRepeat))
}
// Calculate token similarity
diff --git a/internal/service/nlp/reranker_bymodel_scoring_test.go b/internal/service/nlp/reranker_bymodel_scoring_test.go
new file mode 100644
index 0000000000..d362a976e2
--- /dev/null
+++ b/internal/service/nlp/reranker_bymodel_scoring_test.go
@@ -0,0 +1,187 @@
+// Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package nlp
+
+import (
+ "context"
+ "math"
+ "testing"
+
+ "ragflow/internal/common"
+ "ragflow/internal/entity/models"
+)
+
+// scoringDriver returns fixed relevance scores keyed by response index and
+// records the documents it was given. The embedded interface satisfies the rest
+// of models.ModelDriver; it is never called.
+type scoringDriver struct {
+ models.ModelDriver
+ results []models.RerankResult
+ documents []string
+}
+
+func (d *scoringDriver) Rerank(_ context.Context, _ *string, request models.RerankRequest, _ *models.APIConfig, _ *models.RerankConfig, _ *common.ModelUsage) (*models.RerankResponse, error) {
+ d.documents = request.Documents
+ return &models.RerankResponse{Data: d.results}, nil
+}
+
+func TestRerankByModel_Blend(t *testing.T) {
+ qb := NewQueryBuilder()
+ driver := &scoringDriver{results: []models.RerankResult{
+ {Index: 0, RelevanceScore: 0.9},
+ {Index: 1, RelevanceScore: 0.2},
+ }}
+ ids := []string{"a", "b"}
+ field := map[string]map[string]interface{}{
+ "a": {
+ "content_ltks": "alpha",
+ common.PAGERANK_FLD: 0.5,
+ },
+ "b": {"content_ltks": "beta"},
+ }
+ model := models.NewRerankModel(driver, nil, nil, 0)
+
+ sim, tsim, vsim := mustRerankByModel(t, model, chunksFor(ids), ids, field, "alpha", 0.3, 0.7, qb, nil)
+
+ // In-range provider scores pass through NormalizeRerankScores unchanged.
+ if !floatsClose(vsim, []float64{0.9, 0.2}, 1e-9) {
+ t.Fatalf("vsim = %v, want [0.9 0.2]", vsim)
+ }
+ _, keywords := qb.Question("alpha", "qa", 0.6, "")
+ wantTsim := TokenSimilarity(keywords, [][]string{
+ buildRerankTokens(field["a"], "content_ltks", 1, 1, 1),
+ buildRerankTokens(field["b"], "content_ltks", 1, 1, 1),
+ }, qb, "")
+ if !floatsClose(tsim, wantTsim, 1e-9) {
+ t.Fatalf("tsim = %v, want %v", tsim, wantTsim)
+ }
+ for i := range sim {
+ want := 0.3*tsim[i] + 0.7*vsim[i]
+ if i == 0 {
+ want += 0.5
+ }
+ if math.Abs(sim[i]-want) > 1e-9 {
+ t.Fatalf("sim[%d] = %v, want %v", i, sim[i], want)
+ }
+ }
+}
+
+func TestRerankByModel_NormalizationApplied(t *testing.T) {
+ qb := NewQueryBuilder()
+ // NVIDIA-style logits: out of range and mixed sign.
+ driver := &scoringDriver{results: []models.RerankResult{
+ {Index: 0, RelevanceScore: 10},
+ {Index: 1, RelevanceScore: -3},
+ }}
+ ids := []string{"a", "b"}
+ field := map[string]map[string]interface{}{
+ "a": {"content_ltks": "alpha"},
+ "b": {"content_ltks": "beta"},
+ }
+ model := models.NewRerankModel(driver, nil, nil, 0)
+
+ _, _, vsim := mustRerankByModel(t, model, chunksFor(ids), ids, field, "alpha", 0.3, 0.7, qb, nil)
+
+ want := NormalizeRerankScores([]float64{10, -3})
+ if !floatsClose(vsim, want, 1e-9) {
+ t.Fatalf("vsim = %v, want normalized %v", vsim, want)
+ }
+ for _, v := range vsim {
+ if v < 0 || v > 1 {
+ t.Fatalf("normalized score escaped [0,1]: %v", vsim)
+ }
+ }
+}
+
+func TestRerankByModel_IndexOutOfRangeIgnored(t *testing.T) {
+ qb := NewQueryBuilder()
+ driver := &scoringDriver{results: []models.RerankResult{
+ {Index: 5, RelevanceScore: 0.9}, // past the end
+ {Index: -1, RelevanceScore: 0.8}, // negative
+ {Index: 0, RelevanceScore: 0.3},
+ }}
+ ids := []string{"a"}
+ field := map[string]map[string]interface{}{
+ "a": {"content_ltks": "alpha"},
+ }
+ model := models.NewRerankModel(driver, nil, nil, 0)
+
+ sim, _, vsim := mustRerankByModel(t, model, chunksFor(ids), ids, field, "alpha", 0.3, 0.7, qb, nil)
+
+ if len(sim) != 1 || len(vsim) != 1 {
+ t.Fatalf("lengths = %d/%d, want 1/1", len(sim), len(vsim))
+ }
+ if math.Abs(vsim[0]-0.3) > 1e-9 {
+ t.Fatalf("vsim[0] = %v, want 0.3 (out-of-range indexes ignored)", vsim[0])
+ }
+}
+
+func TestRerankByModel_ChunksFallbackAndEmptyIDs(t *testing.T) {
+ qb := NewQueryBuilder()
+
+ t.Run("empty ids", func(t *testing.T) {
+ driver := &scoringDriver{}
+ model := models.NewRerankModel(driver, nil, nil, 0)
+ sim, tsim, vsim := mustRerankByModel(t, model, chunksFor([]string{"a"}), nil, nil, "alpha", 0.3, 0.7, qb, nil)
+ if len(sim) != 0 || len(tsim) != 0 || len(vsim) != 0 {
+ t.Fatalf("got %v/%v/%v, want all empty", sim, tsim, vsim)
+ }
+ })
+
+ t.Run("field miss falls back to chunks", func(t *testing.T) {
+ driver := &scoringDriver{results: []models.RerankResult{{Index: 0, RelevanceScore: 0.6}}}
+ model := models.NewRerankModel(driver, nil, nil, 0)
+ chunks := []map[string]interface{}{{"content_ltks": "alpha"}}
+ sim, _, vsim := mustRerankByModel(t, model, chunks, []string{"missing"}, nil, "alpha", 0.3, 0.7, qb, nil)
+ if len(sim) != 1 || math.Abs(vsim[0]-0.6) > 1e-9 {
+ t.Fatalf("sim=%v vsim=%v, want one score 0.6", sim, vsim)
+ }
+ })
+}
+
+// chunksFor builds a non-empty chunk list: RerankByModel returns early when
+// chunks is nil, even though it ultimately scores ids+field.
+func chunksFor(ids []string) []map[string]interface{} {
+ if len(ids) == 0 {
+ return []map[string]interface{}{{"content_ltks": ""}}
+ }
+ chunks := make([]map[string]interface{}, len(ids))
+ for i := range chunks {
+ chunks[i] = map[string]interface{}{"content_ltks": ""}
+ }
+ return chunks
+}
+
+func mustRerankByModel(
+ t *testing.T,
+ model *models.RerankModel,
+ chunks []map[string]interface{},
+ ids []string,
+ field map[string]map[string]interface{},
+ query string,
+ tkWeight, vtWeight float64,
+ qb *QueryBuilder,
+ rankFeature map[string]float64,
+) ([]float64, []float64, []float64) {
+ t.Helper()
+ sim, tsim, vsim, err := RerankByModel(
+ context.Background(), model, chunks, ids, field, query,
+ tkWeight, vtWeight, "content_ltks", qb, rankFeature, "",
+ )
+ if err != nil {
+ t.Fatalf("RerankByModel: %v", err)
+ }
+ return sim, tsim, vsim
+}
diff --git a/internal/service/nlp/reranker_rank_feature_test.go b/internal/service/nlp/reranker_rank_feature_test.go
new file mode 100644
index 0000000000..2b85150de1
--- /dev/null
+++ b/internal/service/nlp/reranker_rank_feature_test.go
@@ -0,0 +1,261 @@
+// Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package nlp
+
+import (
+ "math"
+ "testing"
+
+ "ragflow/internal/common"
+)
+
+// wantRankFeature computes the expected final score for one chunk independently
+// of applyRankFeatureScores. It mirrors the documented formula:
+//
+// qDenor = sqrt(sum(query_weight^2, excluding pagerank_fea))
+// tag_score = sum(qw*sc for known tags) / sqrt(sum(sc^2)) / qDenor
+// final = sim + tag_score*10 + pagerank
+func wantRankFeature(
+ sim float64,
+ tagFeas map[string]float64,
+ pagerank float64,
+ rankFeature map[string]float64,
+) float64 {
+ qDenor := 0.0
+ for k, w := range rankFeature {
+ if k == common.PAGERANK_FLD {
+ continue
+ }
+ qDenor += w * w
+ }
+ qDenor = math.Sqrt(qDenor)
+
+ tagScore := 0.0
+ if qDenor != 0 {
+ var nor, denor float64
+ for k, sc := range tagFeas {
+ if w, ok := rankFeature[k]; ok && k != common.PAGERANK_FLD {
+ nor += w * sc
+ }
+ denor += sc * sc
+ }
+ if denor != 0 {
+ tagScore = nor / math.Sqrt(denor) / qDenor
+ }
+ }
+ return sim + tagScore*10 + pagerank
+}
+
+func TestApplyRankFeatureScores_EmptyInputs(t *testing.T) {
+ if got := applyRankFeatureScores(nil, nil, map[string]float64{"a": 1}); len(got) != 0 {
+ t.Fatalf("nil chunks -> %v, want empty", got)
+ }
+ sim := []float64{1, 2}
+ if got := applyRankFeatureScores(nil, sim, nil); !floatsClose(got, sim, 0) {
+ t.Fatalf("nil chunks with scores -> %v, want unchanged %v", got, sim)
+ }
+ if got := applyRankFeatureScores([]map[string]interface{}{{}}, nil, nil); len(got) != 0 {
+ t.Fatalf("nil sim -> %v, want empty", got)
+ }
+}
+
+func TestApplyRankFeatureScores_PagerankOnly(t *testing.T) {
+ chunks := []map[string]interface{}{
+ {common.PAGERANK_FLD: 2.5},
+ {common.PAGERANK_FLD: 1},
+ {}, // missing pagerank counts as 0
+ }
+ sim := []float64{1, 1, 1}
+ got := applyRankFeatureScores(chunks, sim, nil)
+ want := []float64{3.5, 2, 1}
+ if !floatsClose(got, want, 1e-9) {
+ t.Fatalf("got %v, want %v", got, want)
+ }
+}
+
+func TestApplyRankFeatureScores_PagerankOnlyQueryNoDivZero(t *testing.T) {
+ chunks := []map[string]interface{}{{common.PAGERANK_FLD: 10}}
+ sim := []float64{0.5}
+ got := applyRankFeatureScores(chunks, sim, map[string]float64{common.PAGERANK_FLD: 10})
+ want := []float64{10.5}
+ if !floatsClose(got, want, 1e-9) {
+ t.Fatalf("got %v, want %v", got, want)
+ }
+ for _, v := range got {
+ if math.IsNaN(v) || math.IsInf(v, 0) {
+ t.Fatalf("non-finite score leaked: %v", got)
+ }
+ }
+}
+
+func TestApplyRankFeatureScores_Formula(t *testing.T) {
+ rankFeature := map[string]float64{"tag_a": 2, "tag_b": 1}
+ chunks := []map[string]interface{}{
+ {
+ common.TAG_FLD: map[string]interface{}{"tag_a": 0.5, "tag_b": 2.0, "unknown": 1.0},
+ common.PAGERANK_FLD: 0.5,
+ },
+ {common.TAG_FLD: map[string]float64{"tag_a": 1}},
+ }
+ sim := []float64{0.1, 0.2}
+ want := []float64{
+ wantRankFeature(0.1, map[string]float64{"tag_a": 0.5, "tag_b": 2.0, "unknown": 1.0}, 0.5, rankFeature),
+ wantRankFeature(0.2, map[string]float64{"tag_a": 1}, 0, rankFeature),
+ }
+ got := applyRankFeatureScores(chunks, sim, rankFeature)
+ if !floatsClose(got, want, 1e-9) {
+ t.Fatalf("got %v, want %v", got, want)
+ }
+}
+
+func TestApplyRankFeatureScores_UnknownTagOnlyDenor(t *testing.T) {
+ rankFeature := map[string]float64{"tag_a": 1}
+ // "unknown" must contribute to denor but not to nor.
+ withUnknown := applyRankFeatureScores(
+ []map[string]interface{}{{common.TAG_FLD: map[string]float64{"tag_a": 1, "unknown": 1}}},
+ []float64{0},
+ rankFeature,
+ )
+ knownOnly := applyRankFeatureScores(
+ []map[string]interface{}{{common.TAG_FLD: map[string]float64{"tag_a": 1}}},
+ []float64{0},
+ rankFeature,
+ )
+ if withUnknown[0] >= knownOnly[0] {
+ t.Fatalf("unknown tag raised the score: %v vs %v", withUnknown, knownOnly)
+ }
+}
+
+func TestApplyRankFeatureScores_PagerankExcludedFromQDenor(t *testing.T) {
+ chunks := []map[string]interface{}{{common.TAG_FLD: map[string]float64{"tag_a": 1}}}
+ sim := []float64{0}
+ for name, rf := range map[string]map[string]float64{
+ "tag only": {"tag_a": 1},
+ "tag plus page": {"tag_a": 1, common.PAGERANK_FLD: 1000},
+ } {
+ t.Run(name, func(t *testing.T) {
+ got := applyRankFeatureScores(chunks, append([]float64(nil), sim...), rf)
+ want := wantRankFeature(0, map[string]float64{"tag_a": 1}, 0, rf)
+ if math.Abs(got[0]-want) > 1e-9 {
+ t.Fatalf("got %v, want %v (pagerank must not enter qDenor)", got[0], want)
+ }
+ })
+ }
+}
+
+func TestApplyRankFeatureScores_TagFeasFormats(t *testing.T) {
+ rankFeature := map[string]float64{"tag_a": 2}
+ cases := []struct {
+ name string
+ form interface{}
+ feas map[string]float64
+ }{
+ {"map interface", map[string]interface{}{"tag_a": 0.5}, map[string]float64{"tag_a": 0.5}},
+ {"map float64", map[string]float64{"tag_a": 0.5}, map[string]float64{"tag_a": 0.5}},
+ {"map int", map[string]int{"tag_a": 1}, map[string]float64{"tag_a": 1}},
+ {"json string", `{"tag_a": 0.5}`, map[string]float64{"tag_a": 0.5}},
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ want := wantRankFeature(0, tc.feas, 0, rankFeature)
+ got := applyRankFeatureScores(
+ []map[string]interface{}{{common.TAG_FLD: tc.form}},
+ []float64{0},
+ rankFeature,
+ )
+ if math.Abs(got[0]-want) > 1e-9 {
+ t.Fatalf("got %v, want %v", got[0], want)
+ }
+ })
+ }
+}
+
+func TestApplyRankFeatureScores_ZeroDenominator(t *testing.T) {
+ cases := map[string]interface{}{
+ "empty map": map[string]float64{},
+ "all zero": map[string]float64{"tag_a": 0},
+ "absent": nil,
+ "empty json": "{}",
+ }
+ rankFeature := map[string]float64{"tag_a": 1}
+ for name, tagFeas := range cases {
+ t.Run(name, func(t *testing.T) {
+ chunk := map[string]interface{}{common.PAGERANK_FLD: 1.5}
+ if tagFeas != nil {
+ chunk[common.TAG_FLD] = tagFeas
+ }
+ got := applyRankFeatureScores([]map[string]interface{}{chunk}, []float64{0.5}, rankFeature)
+ if math.Abs(got[0]-2.0) > 1e-9 {
+ t.Fatalf("got %v, want pagerank-only 2.0", got[0])
+ }
+ })
+ }
+}
+
+func TestApplyRankFeatureScores_NoNaNOrInf(t *testing.T) {
+ rankFeature := map[string]float64{"tag_a": 2, common.PAGERANK_FLD: 10}
+ chunks := []map[string]interface{}{
+ {common.PAGERANK_FLD: 1},
+ {common.TAG_FLD: map[string]float64{"tag_a": 3}, common.PAGERANK_FLD: 0},
+ {common.TAG_FLD: map[string]float64{"tag_b": 3}},
+ }
+ got := applyRankFeatureScores(chunks, []float64{0.1, 0.2, 0.3}, rankFeature)
+ for i, v := range got {
+ if math.IsNaN(v) || math.IsInf(v, 0) {
+ t.Fatalf("score[%d] is non-finite: %v", i, got)
+ }
+ }
+}
+
+func TestApplyRankFeatureScores_LengthMismatch(t *testing.T) {
+ chunks := []map[string]interface{}{{common.PAGERANK_FLD: 1}}
+ sim := []float64{0, 5}
+ got := applyRankFeatureScores(chunks, sim, nil)
+ if !floatsClose(got, []float64{1, 5}, 1e-9) {
+ t.Fatalf("got %v, want [1 5] (trailing score preserved)", got)
+ }
+}
+
+func TestApplyRankFeatureScoresForIDs_MissingAndEquivalent(t *testing.T) {
+ rankFeature := map[string]float64{"tag_a": 2}
+ chunks := []map[string]interface{}{
+ {common.TAG_FLD: map[string]float64{"tag_a": 1}, common.PAGERANK_FLD: 0.5},
+ {common.PAGERANK_FLD: 1},
+ }
+ ids := []string{"a", "missing"}
+ field := map[string]map[string]interface{}{"a": chunks[0]}
+ sim := []float64{0.1, 0.2}
+
+ byIDs := applyRankFeatureScoresForIDs(ids, field, append([]float64(nil), sim...), rankFeature)
+ // The missing id is scored as pagerank 0, tag 0.
+ if math.Abs(byIDs[1]-0.2) > 1e-9 {
+ t.Fatalf("missing id score = %v, want 0.2", byIDs[1])
+ }
+
+ byChunks := applyRankFeatureScores(chunks, append([]float64(nil), sim...), rankFeature)
+ if math.Abs(byIDs[0]-byChunks[0]) > 1e-9 {
+ t.Fatalf("ids version = %v, chunks version = %v", byIDs[0], byChunks[0])
+ }
+}
+
+func TestApplyRankFeatureScoresForIDs_LengthMismatch(t *testing.T) {
+ ids := []string{"a"}
+ field := map[string]map[string]interface{}{"a": {common.PAGERANK_FLD: 2}}
+ sim := []float64{0, 7}
+ got := applyRankFeatureScoresForIDs(ids, field, sim, nil)
+ if !floatsClose(got, []float64{2, 7}, 1e-9) {
+ t.Fatalf("got %v, want [2 7] (trailing score preserved)", got)
+ }
+}
diff --git a/internal/service/nlp/reranker_scoring_test.go b/internal/service/nlp/reranker_scoring_test.go
new file mode 100644
index 0000000000..88fe48266d
--- /dev/null
+++ b/internal/service/nlp/reranker_scoring_test.go
@@ -0,0 +1,329 @@
+// Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package nlp
+
+import (
+ "context"
+ "math"
+ "testing"
+
+ "ragflow/internal/common"
+)
+
+func TestBuildRerankTokens_OrderAndMultipliers(t *testing.T) {
+ chunk := map[string]interface{}{
+ "content_ltks": "a b a",
+ "title_tks": "t1 t2",
+ "important_kwd": []string{"i1", "i2"},
+ "question_tks": "q1 q2",
+ }
+
+ t.Run("standard multipliers", func(t *testing.T) {
+ got := buildRerankTokens(chunk, "content_ltks", titleRepeat, importantRepeat, questionRepeat)
+ want := []string{
+ "a", "b", // content is deduplicated
+ "t1", "t2", "t1", "t2",
+ "i1", "i2", "i1", "i2", "i1", "i2", "i1", "i2", "i1", "i2",
+ "q1", "q2", "q1", "q2", "q1", "q2", "q1", "q2", "q1", "q2", "q1", "q2",
+ }
+ if len(got) != len(want) {
+ t.Fatalf("length = %d, want %d: %v", len(got), len(want), got)
+ }
+ for i := range want {
+ if got[i] != want[i] {
+ t.Fatalf("token[%d] = %q, want %q (full: %v)", i, got[i], want[i], got)
+ }
+ }
+ })
+
+ t.Run("no repetition", func(t *testing.T) {
+ got := buildRerankTokens(chunk, "content_ltks", 1, 1, 1)
+ want := []string{"a", "b", "t1", "t2", "i1", "i2", "q1", "q2"}
+ if len(got) != len(want) {
+ t.Fatalf("length = %d, want %d: %v", len(got), len(want), got)
+ }
+ for i := range want {
+ if got[i] != want[i] {
+ t.Fatalf("token[%d] = %q, want %q", i, got[i], want[i])
+ }
+ }
+ })
+}
+
+func TestBuildRerankTokens_MultiplierCounts(t *testing.T) {
+ chunk := map[string]interface{}{
+ "content_ltks": "c",
+ "title_tks": "t",
+ "important_kwd": []string{"i"},
+ "question_tks": "q",
+ }
+ got := buildRerankTokens(chunk, "content_ltks", titleRepeat, importantRepeat, questionRepeat)
+ counts := make(map[string]int)
+ for _, tk := range got {
+ counts[tk]++
+ }
+ want := map[string]int{"c": 1, "t": titleRepeat, "i": importantRepeat, "q": questionRepeat}
+ for tk, n := range want {
+ if counts[tk] != n {
+ t.Fatalf("token %q appears %d times, want %d", tk, counts[tk], n)
+ }
+ }
+ if titleRepeat != 2 || importantRepeat != 5 || questionRepeat != 6 {
+ t.Fatalf("multipliers changed: title=%d important=%d question=%d", titleRepeat, importantRepeat, questionRepeat)
+ }
+}
+
+func TestBuildRerankTokens_EmptyFieldsNonNil(t *testing.T) {
+ got := buildRerankTokens(map[string]interface{}{}, "content_ltks", titleRepeat, importantRepeat, questionRepeat)
+ if got == nil {
+ t.Fatal("expected non-nil empty slice")
+ }
+ if len(got) != 0 {
+ t.Fatalf("got %v, want empty", got)
+ }
+}
+
+func TestBuildRerankTokens_DedupSemantics(t *testing.T) {
+ chunk := map[string]interface{}{
+ "content_ltks": "x x y",
+ "title_tks": "x x",
+ "important_kwd": []string{"x", "x"},
+ "question_tks": "x x",
+ }
+ got := buildRerankTokens(chunk, "content_ltks", 1, 1, 1)
+ want := []string{"x", "y", "x", "x", "x", "x", "x", "x"}
+ if len(got) != len(want) {
+ t.Fatalf("length = %d, want %d: %v", len(got), len(want), got)
+ }
+ for i := range want {
+ if got[i] != want[i] {
+ t.Fatalf("token[%d] = %q, want %q (full: %v)", i, got[i], want[i], got)
+ }
+ }
+}
+
+func TestRerankStandard_EmptyChunks(t *testing.T) {
+ sim, tsim, vsim := RerankStandard(nil, nil, nil, "alpha", 0.3, 0.7, "content_ltks", nil, nil, "")
+ if len(sim) != 0 || len(tsim) != 0 || len(vsim) != 0 {
+ t.Fatalf("got sim=%v tsim=%v vsim=%v, want all empty", sim, tsim, vsim)
+ }
+}
+
+func TestRerankStandard_VectorAndPagerankWiring(t *testing.T) {
+ chunks := []map[string]interface{}{
+ {
+ "content_ltks": "alpha",
+ "q_2_vec": []float64{1, 0},
+ common.PAGERANK_FLD: 0.5,
+ },
+ {
+ "content_ltks": "beta",
+ "q_2_vec": []float64{0, 1},
+ },
+ }
+ sim, tsim, vsim := RerankStandard(
+ chunks, nil, []float64{1, 0}, "alpha",
+ 0.3, 0.7, "content_ltks", nil, nil, "",
+ )
+ wantVsim := []float64{1, 0}
+ if !floatsClose(vsim, wantVsim, 1e-9) {
+ t.Fatalf("vsim = %v, want %v", vsim, wantVsim)
+ }
+ // qb == nil -> no token score, so the blend is purely vector + pagerank.
+ wantSim := []float64{0.7 + 0.5, 0}
+ if !floatsClose(sim, wantSim, 1e-9) {
+ t.Fatalf("sim = %v, want %v", sim, wantSim)
+ }
+ if !floatsClose(tsim, []float64{0, 0}, 1e-9) {
+ t.Fatalf("tsim = %v, want zeros", tsim)
+ }
+}
+
+func TestRerankStandard_TokenWiring(t *testing.T) {
+ qb := NewQueryBuilder()
+ keywords := []string{"alpha", "gamma"}
+ chunks := []map[string]interface{}{
+ {
+ "content_ltks": "alpha beta",
+ "title_tks": "gamma",
+ "important_kwd": []string{"delta"},
+ "question_tks": "epsilon",
+ "q_2_vec": []float64{1, 0},
+ common.PAGERANK_FLD: 0.25,
+ },
+ }
+ sim, tsim, vsim := RerankStandard(
+ chunks, keywords, []float64{1, 0}, "alpha",
+ 0.3, 0.7, "content_ltks", qb, nil, "",
+ )
+
+ wantTsim := TokenSimilarity(keywords, [][]string{
+ buildRerankTokens(chunks[0], "content_ltks", titleRepeat, importantRepeat, questionRepeat),
+ }, qb, "")
+ if !floatsClose(tsim, wantTsim, 1e-9) {
+ t.Fatalf("tsim = %v, want %v", tsim, wantTsim)
+ }
+ wantSim := []float64{0.3*wantTsim[0] + 0.7*vsim[0] + 0.25}
+ if !floatsClose(sim, wantSim, 1e-9) {
+ t.Fatalf("sim = %v, want %v", sim, wantSim)
+ }
+}
+
+func TestRerankStandard_TSimNotPollutedByRankFeature(t *testing.T) {
+ qb := NewQueryBuilder()
+ chunks := []map[string]interface{}{
+ {
+ "content_ltks": "alpha",
+ common.PAGERANK_FLD: 2.5,
+ },
+ }
+ keywords := []string{"alpha"}
+ sim, tsim, vsim := RerankStandard(
+ chunks, keywords, nil, "alpha",
+ 0.3, 0.7, "content_ltks", qb, nil, "",
+ )
+ if len(vsim) != 1 || vsim[0] != 0 {
+ t.Fatalf("vsim = %v, want [0] for a nil query vector", vsim)
+ }
+ wantTsim := TokenSimilarity(keywords, [][]string{
+ buildRerankTokens(chunks[0], "content_ltks", titleRepeat, importantRepeat, questionRepeat),
+ }, qb, "")
+ if !floatsClose(tsim, wantTsim, 1e-9) {
+ t.Fatalf("tsim = %v, want %v (rank feature must not leak into term_similarity)", tsim, wantTsim)
+ }
+ // All-zero vectors fall back to tsim, then pagerank is added to sim only.
+ if math.Abs(sim[0]-(wantTsim[0]+2.5)) > 1e-9 {
+ t.Fatalf("sim = %v, want %v", sim[0], wantTsim[0]+2.5)
+ }
+}
+
+func TestRerankWithKNN_EmptyIDs(t *testing.T) {
+ sim, tsim, vsim := RerankWithKNN(
+ context.Background(), nil, nil, nil, nil, "alpha",
+ 0.3, 0.7, "content_ltks", nil, nil, "",
+ )
+ if len(sim) != 0 || len(tsim) != 0 || len(vsim) != 0 {
+ t.Fatalf("got sim=%v tsim=%v vsim=%v, want all empty", sim, tsim, vsim)
+ }
+}
+
+func TestRerankWithKNN_KNNMapping(t *testing.T) {
+ qb := NewQueryBuilder()
+ ids := []string{"a", "b", "c"}
+ field := map[string]map[string]interface{}{
+ "a": {"content_ltks": "alpha"},
+ "b": {"content_ltks": "beta"},
+ "c": {"content_ltks": "gamma"},
+ }
+ knnScores := map[string]float64{"a": 0.9, "c": 0.4}
+
+ _, _, vsim := RerankWithKNN(
+ context.Background(), nil, ids, field, knnScores, "alpha",
+ 0.3, 0.7, "content_ltks", qb, nil, "",
+ )
+ want := []float64{0.9, 0, 0.4}
+ if !floatsClose(vsim, want, 1e-9) {
+ t.Fatalf("vsim = %v, want %v (missing id must be 0)", vsim, want)
+ }
+}
+
+func TestRerankWithKNN_FallbackAndMissing(t *testing.T) {
+ qb := NewQueryBuilder()
+ // "a" is in field; "b" falls back to chunks[i]; "c" has neither.
+ ids := []string{"a", "b", "c"}
+ field := map[string]map[string]interface{}{
+ "a": {"content_ltks": "alpha"},
+ }
+ chunks := []map[string]interface{}{
+ {"content_ltks": "unused"},
+ {"content_ltks": "beta"},
+ }
+ sim, tsim, vsim := RerankWithKNN(
+ context.Background(), chunks, ids, field, nil, "alpha",
+ 0.3, 0.7, "content_ltks", qb, nil, "",
+ )
+ if len(sim) != 3 || len(tsim) != 3 || len(vsim) != 3 {
+ t.Fatalf("lengths = %d/%d/%d, want 3/3/3", len(sim), len(tsim), len(vsim))
+ }
+ for i, v := range vsim {
+ if v != 0 {
+ t.Fatalf("vsim[%d] = %v, want 0 with no knn scores", i, v)
+ }
+ }
+}
+
+func TestRerankWithKNN_ImportantKwdNormalization(t *testing.T) {
+ qb := NewQueryBuilder()
+
+ asString := map[string]map[string]interface{}{
+ "a": {"content_ltks": "alpha", "important_kwd": "delta"},
+ }
+ asSlice := map[string]map[string]interface{}{
+ "a": {"content_ltks": "alpha", "important_kwd": []string{"delta"}},
+ }
+
+ simString, tsimString, _ := RerankWithKNN(
+ context.Background(), nil, []string{"a"}, asString, nil, "alpha",
+ 0.3, 0.7, "content_ltks", qb, nil, "",
+ )
+ simSlice, tsimSlice, _ := RerankWithKNN(
+ context.Background(), nil, []string{"a"}, asSlice, nil, "alpha",
+ 0.3, 0.7, "content_ltks", qb, nil, "",
+ )
+ if !floatsClose(simString, simSlice, 1e-9) || !floatsClose(tsimString, tsimSlice, 1e-9) {
+ t.Fatalf("string vs []string differ: sim %v/%v tsim %v/%v",
+ simString, simSlice, tsimString, tsimSlice)
+ }
+ if _, ok := asString["a"]["important_kwd"].([]string); !ok {
+ t.Fatalf("important_kwd was not normalized in place: %T", asString["a"]["important_kwd"])
+ }
+}
+
+func TestRerankWithKNN_TokenWiring(t *testing.T) {
+ qb := NewQueryBuilder()
+ ids := []string{"a"}
+ field := map[string]map[string]interface{}{
+ "a": {
+ "content_ltks": "alpha beta",
+ "title_tks": "gamma",
+ "important_kwd": []string{"delta"},
+ "question_tks": "epsilon",
+ common.PAGERANK_FLD: 0.5,
+ },
+ }
+ knnScores := map[string]float64{"a": 0.8}
+ rankFeature := map[string]float64{"tag_a": 1}
+
+ sim, tsim, vsim := RerankWithKNN(
+ context.Background(), nil, ids, field, knnScores, "alpha",
+ 0.3, 0.7, "content_ltks", qb, rankFeature, "",
+ )
+ if !floatsClose(vsim, []float64{0.8}, 1e-9) {
+ t.Fatalf("vsim = %v, want [0.8]", vsim)
+ }
+
+ _, keywords := qb.Question("alpha", "qa", 0.6, "")
+ wantTsim := TokenSimilarity(keywords, [][]string{
+ buildRerankTokens(field["a"], "content_ltks", titleRepeat, importantRepeat, questionRepeat),
+ }, qb, "")
+ if !floatsClose(tsim, wantTsim, 1e-9) {
+ t.Fatalf("tsim = %v, want %v", tsim, wantTsim)
+ }
+ base := 0.3*wantTsim[0] + 0.7*0.8
+ want := base + 0.5 // pagerank only; no tag_feas means no tag score
+ if math.Abs(sim[0]-want) > 1e-9 {
+ t.Fatalf("sim = %v, want %v", sim[0], want)
+ }
+}
diff --git a/internal/service/nlp/reranker_similarity_test.go b/internal/service/nlp/reranker_similarity_test.go
new file mode 100644
index 0000000000..59af64b5a6
--- /dev/null
+++ b/internal/service/nlp/reranker_similarity_test.go
@@ -0,0 +1,341 @@
+// Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package nlp
+
+import (
+ "math"
+ "testing"
+)
+
+func TestCosineSimilarity_BasicCases(t *testing.T) {
+ cases := []struct {
+ name string
+ a, b []float64
+ want float64
+ }{
+ {"identical", []float64{1, 2, 3}, []float64{1, 2, 3}, 1},
+ {"orthogonal", []float64{1, 0}, []float64{0, 1}, 0},
+ {"opposite", []float64{1, 0}, []float64{-1, 0}, -1},
+ {"same direction different magnitude", []float64{1, 2}, []float64{2, 4}, 1},
+ {"negative scale", []float64{1, 2}, []float64{-2, -4}, -1},
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ got := cosineSimilarity(tc.a, tc.b)
+ if math.Abs(got-tc.want) > 1e-9 {
+ t.Fatalf("cosineSimilarity(%v, %v) = %v, want %v", tc.a, tc.b, got, tc.want)
+ }
+ })
+ }
+}
+
+func TestCosineSimilarity_Degenerate(t *testing.T) {
+ cases := []struct {
+ name string
+ a, b []float64
+ }{
+ {"length mismatch", []float64{1, 2}, []float64{1, 2, 3}},
+ {"left zero", []float64{0, 0}, []float64{1, 2}},
+ {"right zero", []float64{1, 2}, []float64{0, 0}},
+ {"both empty", nil, nil},
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ got := cosineSimilarity(tc.a, tc.b)
+ if got != 0 {
+ t.Fatalf("cosineSimilarity(%v, %v) = %v, want 0", tc.a, tc.b, got)
+ }
+ if math.IsNaN(got) || math.IsInf(got, 0) {
+ t.Fatalf("cosineSimilarity returned non-finite value %v", got)
+ }
+ })
+ }
+}
+
+func TestCosineSimilarity_NonFiniteInput(t *testing.T) {
+ // Non-finite components are not guarded by the implementation: the
+ // normalisation still divides by a non-finite norm, so the result is NaN.
+ // Pin the boundary so a future guard shows up as a deliberate change.
+ got := cosineSimilarity([]float64{math.Inf(1), 0}, []float64{1, 0})
+ if !math.IsNaN(got) {
+ t.Fatalf("cosineSimilarity with Inf = %v, want NaN (current boundary)", got)
+ }
+}
+
+func TestTokensToDict_NilQueryBuilderOrTermWeight(t *testing.T) {
+ cases := []struct {
+ name string
+ qb *QueryBuilder
+ }{
+ {"nil query builder", nil},
+ {"nil term weight", &QueryBuilder{}},
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ d, order := tokensToDict([]string{"alpha", "beta"}, tc.qb, "")
+ if len(d) != 0 {
+ t.Fatalf("dict = %v, want empty", d)
+ }
+ if len(order) != 0 {
+ t.Fatalf("key order = %v, want empty", order)
+ }
+ })
+ }
+}
+
+func TestTokensToDict_WeightsAndKeyOrder(t *testing.T) {
+ qb := NewQueryBuilder()
+ tks := []string{"alpha", "beta", "alpha", "gamma"}
+
+ // Use the dealer's own weights as the input oracle, then apply the 0.4/0.6
+ // formula independently. Tokenizer weights are deliberately not hardcoded.
+ wts := qb.termWeight.Weights(tks, false, "")
+ if len(wts) != len(tks) {
+ t.Fatalf("Weights returned %d entries, want %d", len(wts), len(tks))
+ }
+
+ want := make(map[string]float64)
+ var wantOrder []string
+ seen := make(map[string]bool)
+ for i, tw := range wts {
+ if !seen[tw.Term] {
+ seen[tw.Term] = true
+ wantOrder = append(wantOrder, tw.Term)
+ }
+ want[tw.Term] += tw.Weight * 0.4
+ if i+1 < len(wts) {
+ next := wts[i+1]
+ key := tw.Term + next.Term
+ if !seen[key] {
+ seen[key] = true
+ wantOrder = append(wantOrder, key)
+ }
+ want[key] += math.Max(tw.Weight, next.Weight) * 0.6
+ }
+ }
+
+ got, gotOrder := tokensToDict(tks, qb, "")
+ if len(got) != len(want) {
+ t.Fatalf("dict size = %d, want %d (%v vs %v)", len(got), len(want), got, want)
+ }
+ for k, v := range want {
+ if math.Abs(got[k]-v) > 1e-9 {
+ t.Fatalf("dict[%q] = %v, want %v", k, got[k], v)
+ }
+ }
+ if len(gotOrder) != len(wantOrder) {
+ t.Fatalf("key order = %v, want %v", gotOrder, wantOrder)
+ }
+ for i := range wantOrder {
+ if gotOrder[i] != wantOrder[i] {
+ t.Fatalf("key order[%d] = %q, want %q (full: %v vs %v)",
+ i, gotOrder[i], wantOrder[i], gotOrder, wantOrder)
+ }
+ }
+ // Repeated terms accumulate instead of being overwritten.
+ if got["alpha"] <= 0.4*wts[0].Weight {
+ t.Fatalf("repeated term alpha did not accumulate: %v", got["alpha"])
+ }
+}
+
+func TestTokenDictSimilarity_Cases(t *testing.T) {
+ t.Run("empty", func(t *testing.T) {
+ if got := tokenDictSimilarity(nil, map[string]float64{"a": 1}, nil); got != 0 {
+ t.Fatalf("empty query -> %v, want 0", got)
+ }
+ if got := tokenDictSimilarity(map[string]float64{"a": 1}, nil, nil); got != 0 {
+ t.Fatalf("empty doc -> %v, want 0", got)
+ }
+ })
+
+ t.Run("full overlap", func(t *testing.T) {
+ q := map[string]float64{"a": 1, "b": 1}
+ d := map[string]float64{"a": 5, "b": 5}
+ got := tokenDictSimilarity(q, d, []string{"a", "b"})
+ if math.Abs(got-1) > 1e-9 {
+ t.Fatalf("full overlap -> %v, want 1", got)
+ }
+ })
+
+ t.Run("partial overlap", func(t *testing.T) {
+ q := map[string]float64{"a": 1, "b": 3}
+ d := map[string]float64{"a": 9}
+ want := (1e-9 + 1) / (1e-9 + 4)
+ got := tokenDictSimilarity(q, d, []string{"a", "b"})
+ if math.Abs(got-want) > 1e-12 {
+ t.Fatalf("partial overlap -> %v, want %v", got, want)
+ }
+ })
+
+ t.Run("no overlap is near zero but not zero", func(t *testing.T) {
+ q := map[string]float64{"a": 1}
+ d := map[string]float64{"z": 1}
+ got := tokenDictSimilarity(q, d, []string{"a"})
+ if got == 0 {
+ t.Fatalf("no overlap returned exactly 0, smoothing missing")
+ }
+ if got > 1e-6 {
+ t.Fatalf("no overlap -> %v, want near zero", got)
+ }
+ })
+}
+
+func TestTokenDictSimilarity_NilKeyOrderFallback(t *testing.T) {
+ q := map[string]float64{"alpha": 1, "beta": 2, "gamma": 3}
+ d := map[string]float64{"beta": 1, "gamma": 1}
+ withOrder := tokenDictSimilarity(q, d, []string{"alpha", "beta", "gamma"})
+ withoutOrder := tokenDictSimilarity(q, d, nil)
+ if math.Abs(withOrder-withoutOrder) > 1e-12 {
+ t.Fatalf("nil order = %v, explicit sorted order = %v", withoutOrder, withOrder)
+ }
+}
+
+func TestTokenSimilarity_Cases(t *testing.T) {
+ t.Run("empty documents", func(t *testing.T) {
+ got := TokenSimilarity([]string{"a"}, nil, NewQueryBuilder(), "")
+ if len(got) != 0 {
+ t.Fatalf("nil docs -> %v, want empty", got)
+ }
+ })
+
+ t.Run("nil query builder returns zeros", func(t *testing.T) {
+ got := TokenSimilarity([]string{"alpha"}, [][]string{{"alpha"}, {"beta"}}, nil, "")
+ if len(got) != 2 {
+ t.Fatalf("len = %d, want 2", len(got))
+ }
+ for i, v := range got {
+ if v != 0 {
+ t.Fatalf("score[%d] = %v, want 0", i, v)
+ }
+ }
+ })
+
+ qb := NewQueryBuilder()
+ t.Run("identical", func(t *testing.T) {
+ got := TokenSimilarity([]string{"alpha", "beta"}, [][]string{{"alpha", "beta"}}, qb, "")
+ if len(got) != 1 || math.Abs(got[0]-1) > 1e-9 {
+ t.Fatalf("identical -> %v, want [1]", got)
+ }
+ })
+
+ t.Run("disjoint", func(t *testing.T) {
+ got := TokenSimilarity([]string{"alpha"}, [][]string{{"omega"}}, qb, "")
+ // The 1e-9 smoothing is in both numerator and denominator, so a
+ // disjoint pair is near zero but not exactly zero.
+ if len(got) != 1 || got[0] > 1e-6 {
+ t.Fatalf("disjoint -> %v, want near 0", got)
+ }
+ })
+
+ t.Run("partial", func(t *testing.T) {
+ got := TokenSimilarity([]string{"alpha", "beta"}, [][]string{{"alpha"}}, qb, "")
+ if len(got) != 1 || got[0] <= 0 || got[0] >= 1 {
+ t.Fatalf("partial -> %v, want 0 < x < 1", got)
+ }
+ })
+
+ t.Run("stable across calls", func(t *testing.T) {
+ atks := []string{"alpha", "beta", "gamma"}
+ btkss := [][]string{{"alpha"}, {"beta", "gamma"}, {}}
+ first := TokenSimilarity(atks, btkss, qb, "")
+ second := TokenSimilarity(atks, btkss, qb, "")
+ if !floatsClose(first, second, 0) {
+ t.Fatalf("unstable: %v vs %v", first, second)
+ }
+ })
+}
+
+func TestHybridSimilarity_VectorOnly(t *testing.T) {
+ avec := []float64{1, 0}
+ bvecs := [][]float64{{1, 0}, {0, 1}}
+ sim, tsim, vsim := HybridSimilarity(avec, bvecs, []string{"alpha"}, [][]string{{"alpha"}, {"beta"}}, 0.3, 0.7, nil, "")
+ if !floatsClose(vsim, []float64{1, 0}, 1e-9) {
+ t.Fatalf("vsim = %v, want [1 0]", vsim)
+ }
+ for i := range tsim {
+ if tsim[i] != 0 {
+ t.Fatalf("tsim[%d] = %v, want 0 with nil qb", i, tsim[i])
+ }
+ }
+ want := []float64{0.7, 0}
+ if !floatsClose(sim, want, 1e-9) {
+ t.Fatalf("sim = %v, want %v", sim, want)
+ }
+}
+
+func TestHybridSimilarity_BlendAndZero(t *testing.T) {
+ t.Run("blend", func(t *testing.T) {
+ qb := NewQueryBuilder()
+ avec := []float64{1, 0}
+ bvecs := [][]float64{{1, 0}, {1, 1}}
+ atks := []string{"alpha"}
+ btkss := [][]string{{"alpha"}, {"beta"}}
+ sim, tsim, vsim := HybridSimilarity(avec, bvecs, atks, btkss, 0.3, 0.7, qb, "")
+ for i := range sim {
+ want := 0.3*tsim[i] + 0.7*vsim[i]
+ if math.Abs(sim[i]-want) > 1e-9 {
+ t.Fatalf("sim[%d] = %v, want 0.3*%v + 0.7*%v = %v", i, sim[i], tsim[i], vsim[i], want)
+ }
+ }
+ })
+
+ t.Run("all zero vectors", func(t *testing.T) {
+ qb := NewQueryBuilder()
+ avec := []float64{0, 0}
+ bvecs := [][]float64{{0, 0}, {0, 0}}
+ sim, tsim, vsim := HybridSimilarity(avec, bvecs, []string{"alpha"}, [][]string{{"alpha"}, {"beta"}}, 0.3, 0.7, qb, "")
+ for i := range vsim {
+ if vsim[i] != 0 {
+ t.Fatalf("vsim[%d] = %v, want 0", i, vsim[i])
+ }
+ }
+ // Initial sim equals tsim; the slice must not be the same backing array.
+ if !floatsClose(sim, tsim, 1e-9) {
+ t.Fatalf("sim = %v, want tsim = %v", sim, tsim)
+ }
+ })
+}
+
+func TestHybridSimilarity_AllZeroDoesNotAlias(t *testing.T) {
+ avec := []float64{0, 0}
+ bvecs := [][]float64{{0, 0}}
+ sim, tsim, _ := HybridSimilarity(avec, bvecs, []string{"alpha"}, [][]string{{"alpha"}}, 0.3, 0.7, nil, "")
+ if len(sim) == 0 || len(tsim) == 0 {
+ t.Fatal("expected non-empty result")
+ }
+ if &sim[0] == &tsim[0] {
+ t.Fatal("sim and tsim share a backing array; rank feature scores would leak into term_similarity")
+ }
+ sim[0] += 42
+ if tsim[0] != 0 {
+ t.Fatalf("mutating sim changed tsim: %v", tsim)
+ }
+}
+
+func TestHybridSimilarity_LengthMismatch(t *testing.T) {
+ // Known limitation: fewer vectors than token lists indexes past vsim.
+ defer func() {
+ if r := recover(); r != nil {
+ t.Logf("HybridSimilarity panics on vector/token length mismatch (known limitation): %v", r)
+ }
+ }()
+ HybridSimilarity(
+ []float64{1, 0},
+ [][]float64{{1, 0}},
+ []string{"alpha"},
+ [][]string{{"alpha"}, {"beta"}},
+ 0.3, 0.7, nil, "",
+ )
+}