RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(reranker): add direct tests for rerank scoring mathFix/reranker and Fix two defects found while testing (#21485)
ZZZZY82bd3b2a581e
diff --git a/internal/service/nlp/reranker.go b/internal/service/nlp/reranker.goindex abbbee19ca..76b32c574a 100644--- a/internal/service/nlp/reranker.go+++ b/internal/service/nlp/reranker.go@@ -18,6 +18,7 @@ import ("context""encoding/json""math"+ "slices""sort""strconv""strings"@@ -134,19 +135,10 @@ func RerankByModel(}}- contentLtks := extractContentTokens(chunk, cfield)- titleTks := extractTitleTokens(chunk)- importantKwd := extractImportantKeywords(chunk)- questionTks := extractQuestionTokens(chunk)-// Unlike RerankStandard/RerankWithKNN, the fields are not repeated here:// these tokens are joined back into `docs` for a cross-encoder, where// duplicating a field would distort the model's own scoring.- tks := make([]string, 0, len(contentLtks)+len(titleTks)+len(importantKwd)+len(questionTks))- tks = append(tks, contentLtks...)- tks = append(tks, titleTks...)- tks = append(tks, importantKwd...)- tks = append(tks, questionTks...)+ tks := buildRerankTokens(chunk, cfield, 1, 1, 1)insTw = append(insTw, tks)// Feed the reranker the natural chunk text (markup preserved), not the@@ -309,25 +301,8 @@ func RerankStandard(chunkVector := extractVector(chunk, vectorColumn, zeroVector)insEmbd = append(insEmbd, chunkVector)- // Extract tokens- contentLtks := extractContentTokens(chunk, cfield)- titleTks := extractTitleTokens(chunk)- questionTks := extractQuestionTokens(chunk)- importantKwd := extractImportantKeywords(chunk)-- // Combine tokens with weights: content + title*2 + important_kwd*5 + question_tks*6- tks := make([]string, 0, len(contentLtks)+len(titleTks)*2+len(importantKwd)*5+len(questionTks)*6)- tks = append(tks, contentLtks...)- for i := 0; i < 2; i++ {- tks = append(tks, titleTks...)- }- for i := 0; i < 5; i++ {- tks = append(tks, importantKwd...)- }- for i := 0; i < 6; i++ {- tks = append(tks, questionTks...)- }- insTw = append(insTw, tks)+ insTw = append(insTw, buildRerankTokens(+ chunk, cfield, titleRepeat, importantRepeat, questionRepeat))}if len(insEmbd) == 0 {@@ -399,7 +374,7 @@ func HybridSimilarity(}if allZero {- return tsim, tsim, vsim+ return slices.Clone(tsim), tsim, vsim}// Combine similarities@@ -627,6 +602,44 @@ func extractImportantKeywords(fields map[string]interface{}) []string {}}+// Field repetition multipliers for the token-based rerank paths. RerankByModel+// passes 1 for all three: it joins the tokens into natural text for a+// cross-encoder, where duplication would distort the model's own score.+const (+ titleRepeat = 2+ importantRepeat = 5+ questionRepeat = 6+)++// buildRerankTokens assembles content + title*titleRepeat ++// important*importantRepeat + question*questionRepeat, in that order.+// The returned slice is never nil.+func buildRerankTokens(+ chunk map[string]interface{},+ cfield string,+ titleRepeat, importantRepeat, questionRepeat int,+) []string {+ content := extractContentTokens(chunk, cfield)+ title := extractTitleTokens(chunk)+ important := extractImportantKeywords(chunk)+ question := extractQuestionTokens(chunk)++ tks := make([]string, 0,+ len(content)+len(title)*titleRepeat++ len(important)*importantRepeat+len(question)*questionRepeat)+ tks = append(tks, content...)+ for i := 0; i < titleRepeat; i++ {+ tks = append(tks, title...)+ }+ for i := 0; i < importantRepeat; i++ {+ tks = append(tks, important...)+ }+ for i := 0; i < questionRepeat; i++ {+ tks = append(tks, question...)+ }+ return tks+}+// cosineSimilarity calculates cosine similarity between two vectors.// Three parallel pairwise sums (dot, ||a||², ||b||²) keep precision// comparable to numpy's float64 reductions.@@ -665,6 +678,17 @@ func applyRankFeatureScores(chunks []map[string]interface{}, sim []float64, rankif len(chunks) == 0 || len(sim) == 0 {return sim}+ // Only the overlapping prefix carries chunk-level rank features; scores+ // beyond the chunk list keep their input value. A length mismatch is an+ // upstream bug, so log it instead of silently scoring part of the list.+ if len(sim) != len(chunks) {+ common.Warn("rank feature: sim/chunks length mismatch, applying rank features to the overlapping prefix only",+ zap.Int("simLen", len(sim)), zap.Int("chunkLen", len(chunks)))+ }+ if len(sim) > len(chunks) {+ applyRankFeatureScores(chunks, sim[:len(chunks)], rankFeature)+ return sim+ }// Collect pageranks from each chunkpageranks := make([]float64, len(chunks))@@ -768,6 +792,17 @@ func applyRankFeatureScoresForIDs(ids []string, field map[string]map[string]inteif len(ids) == 0 || len(sim) == 0 {return sim}+ // Only the overlapping prefix carries chunk-level rank features; scores+ // beyond the id list keep their input value. A length mismatch is an+ // upstream bug, so log it instead of silently scoring part of the list.+ if len(sim) != len(ids) {+ common.Warn("rank feature: sim/ids length mismatch, applying rank features to the overlapping prefix only",+ zap.Int("simLen", len(sim)), zap.Int("idLen", len(ids)))+ }+ if len(sim) > len(ids) {+ applyRankFeatureScoresForIDs(ids, field, sim[:len(ids)], rankFeature)+ return sim+ }// Collect pageranks from each chunk via field mappageranks := make([]float64, len(ids))@@ -970,25 +1005,8 @@ func RerankWithKNN(}// Normalize text content - split, dedupe while preserving order (OrderedDict effect)- contentLtks := extractContentTokens(chunk, cfield)- titleTks := extractTitleTokens(chunk)- questionTks := extractQuestionTokens(chunk)- importantKwd := extractImportantKeywords(chunk)-- // Combine tokens with weights: content + title*2 + important_kwd*5 + question_tks*6- // This matches Python: tks = content_ltks + title_tks * 2 + important_kwd * 5 + question_tks * 6- tks := make([]string, 0, len(contentLtks)+len(titleTks)*2+len(importantKwd)*5+len(questionTks)*6)- tks = append(tks, contentLtks...)- for i := 0; i < 2; i++ {- tks = append(tks, titleTks...)- }- for i := 0; i < 5; i++ {- tks = append(tks, importantKwd...)- }- for i := 0; i < 6; i++ {- tks = append(tks, questionTks...)- }- insTw = append(insTw, tks)+ insTw = append(insTw, buildRerankTokens(+ chunk, cfield, titleRepeat, importantRepeat, questionRepeat))}// Calculate token similaritydiff --git a/internal/service/nlp/reranker_bymodel_scoring_test.go b/internal/service/nlp/reranker_bymodel_scoring_test.gonew file mode 100644index 0000000000..d362a976e2--- /dev/null+++ b/internal/service/nlp/reranker_bymodel_scoring_test.go@@ -0,0 +1,187 @@+// Copyright 2025 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package nlp++import (+ "context"+ "math"+ "testing"++ "ragflow/internal/common"+ "ragflow/internal/entity/models"+)++// scoringDriver returns fixed relevance scores keyed by response index and+// records the documents it was given. The embedded interface satisfies the rest+// of models.ModelDriver; it is never called.+type scoringDriver struct {+ models.ModelDriver+ results []models.RerankResult+ documents []string+}++func (d *scoringDriver) Rerank(_ context.Context, _ *string, request models.RerankRequest, _ *models.APIConfig, _ *models.RerankConfig, _ *common.ModelUsage) (*models.RerankResponse, error) {+ d.documents = request.Documents+ return &models.RerankResponse{Data: d.results}, nil+}++func TestRerankByModel_Blend(t *testing.T) {+ qb := NewQueryBuilder()+ driver := &scoringDriver{results: []models.RerankResult{+ {Index: 0, RelevanceScore: 0.9},+ {Index: 1, RelevanceScore: 0.2},+ }}+ ids := []string{"a", "b"}+ field := map[string]map[string]interface{}{+ "a": {+ "content_ltks": "alpha",+ common.PAGERANK_FLD: 0.5,+ },+ "b": {"content_ltks": "beta"},+ }+ model := models.NewRerankModel(driver, nil, nil, 0)++ sim, tsim, vsim := mustRerankByModel(t, model, chunksFor(ids), ids, field, "alpha", 0.3, 0.7, qb, nil)++ // In-range provider scores pass through NormalizeRerankScores unchanged.+ if !floatsClose(vsim, []float64{0.9, 0.2}, 1e-9) {+ t.Fatalf("vsim = %v, want [0.9 0.2]", vsim)+ }+ _, keywords := qb.Question("alpha", "qa", 0.6, "")+ wantTsim := TokenSimilarity(keywords, [][]string{+ buildRerankTokens(field["a"], "content_ltks", 1, 1, 1),+ buildRerankTokens(field["b"], "content_ltks", 1, 1, 1),+ }, qb, "")+ if !floatsClose(tsim, wantTsim, 1e-9) {+ t.Fatalf("tsim = %v, want %v", tsim, wantTsim)+ }+ for i := range sim {+ want := 0.3*tsim[i] + 0.7*vsim[i]+ if i == 0 {+ want += 0.5+ }+ if math.Abs(sim[i]-want) > 1e-9 {+ t.Fatalf("sim[%d] = %v, want %v", i, sim[i], want)+ }+ }+}++func TestRerankByModel_NormalizationApplied(t *testing.T) {+ qb := NewQueryBuilder()+ // NVIDIA-style logits: out of range and mixed sign.+ driver := &scoringDriver{results: []models.RerankResult{+ {Index: 0, RelevanceScore: 10},+ {Index: 1, RelevanceScore: -3},+ }}+ ids := []string{"a", "b"}+ field := map[string]map[string]interface{}{+ "a": {"content_ltks": "alpha"},+ "b": {"content_ltks": "beta"},+ }+ model := models.NewRerankModel(driver, nil, nil, 0)++ _, _, vsim := mustRerankByModel(t, model, chunksFor(ids), ids, field, "alpha", 0.3, 0.7, qb, nil)++ want := NormalizeRerankScores([]float64{10, -3})+ if !floatsClose(vsim, want, 1e-9) {+ t.Fatalf("vsim = %v, want normalized %v", vsim, want)+ }+ for _, v := range vsim {+ if v < 0 || v > 1 {+ t.Fatalf("normalized score escaped [0,1]: %v", vsim)+ }+ }+}++func TestRerankByModel_IndexOutOfRangeIgnored(t *testing.T) {+ qb := NewQueryBuilder()+ driver := &scoringDriver{results: []models.RerankResult{+ {Index: 5, RelevanceScore: 0.9}, // past the end+ {Index: -1, RelevanceScore: 0.8}, // negative+ {Index: 0, RelevanceScore: 0.3},+ }}+ ids := []string{"a"}+ field := map[string]map[string]interface{}{+ "a": {"content_ltks": "alpha"},+ }+ model := models.NewRerankModel(driver, nil, nil, 0)++ sim, _, vsim := mustRerankByModel(t, model, chunksFor(ids), ids, field, "alpha", 0.3, 0.7, qb, nil)++ if len(sim) != 1 || len(vsim) != 1 {+ t.Fatalf("lengths = %d/%d, want 1/1", len(sim), len(vsim))+ }+ if math.Abs(vsim[0]-0.3) > 1e-9 {+ t.Fatalf("vsim[0] = %v, want 0.3 (out-of-range indexes ignored)", vsim[0])+ }+}++func TestRerankByModel_ChunksFallbackAndEmptyIDs(t *testing.T) {+ qb := NewQueryBuilder()++ t.Run("empty ids", func(t *testing.T) {+ driver := &scoringDriver{}+ model := models.NewRerankModel(driver, nil, nil, 0)+ sim, tsim, vsim := mustRerankByModel(t, model, chunksFor([]string{"a"}), nil, nil, "alpha", 0.3, 0.7, qb, nil)+ if len(sim) != 0 || len(tsim) != 0 || len(vsim) != 0 {+ t.Fatalf("got %v/%v/%v, want all empty", sim, tsim, vsim)+ }+ })++ t.Run("field miss falls back to chunks", func(t *testing.T) {+ driver := &scoringDriver{results: []models.RerankResult{{Index: 0, RelevanceScore: 0.6}}}+ model := models.NewRerankModel(driver, nil, nil, 0)+ chunks := []map[string]interface{}{{"content_ltks": "alpha"}}+ sim, _, vsim := mustRerankByModel(t, model, chunks, []string{"missing"}, nil, "alpha", 0.3, 0.7, qb, nil)+ if len(sim) != 1 || math.Abs(vsim[0]-0.6) > 1e-9 {+ t.Fatalf("sim=%v vsim=%v, want one score 0.6", sim, vsim)+ }+ })+}++// chunksFor builds a non-empty chunk list: RerankByModel returns early when+// chunks is nil, even though it ultimately scores ids+field.+func chunksFor(ids []string) []map[string]interface{} {+ if len(ids) == 0 {+ return []map[string]interface{}{{"content_ltks": ""}}+ }+ chunks := make([]map[string]interface{}, len(ids))+ for i := range chunks {+ chunks[i] = map[string]interface{}{"content_ltks": ""}+ }+ return chunks+}++func mustRerankByModel(+ t *testing.T,+ model *models.RerankModel,+ chunks []map[string]interface{},+ ids []string,+ field map[string]map[string]interface{},+ query string,+ tkWeight, vtWeight float64,+ qb *QueryBuilder,+ rankFeature map[string]float64,+) ([]float64, []float64, []float64) {+ t.Helper()+ sim, tsim, vsim, err := RerankByModel(+ context.Background(), model, chunks, ids, field, query,+ tkWeight, vtWeight, "content_ltks", qb, rankFeature, "",+ )+ if err != nil {+ t.Fatalf("RerankByModel: %v", err)+ }+ return sim, tsim, vsim+}diff --git a/internal/service/nlp/reranker_rank_feature_test.go b/internal/service/nlp/reranker_rank_feature_test.gonew file mode 100644index 0000000000..2b85150de1--- /dev/null+++ b/internal/service/nlp/reranker_rank_feature_test.go@@ -0,0 +1,261 @@+// Copyright 2025 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package nlp++import (+ "math"+ "testing"++ "ragflow/internal/common"+)++// wantRankFeature computes the expected final score for one chunk independently+// of applyRankFeatureScores. It mirrors the documented formula:+//+// qDenor = sqrt(sum(query_weight^2, excluding pagerank_fea))+// tag_score = sum(qw*sc for known tags) / sqrt(sum(sc^2)) / qDenor+// final = sim + tag_score*10 + pagerank+func wantRankFeature(+ sim float64,+ tagFeas map[string]float64,+ pagerank float64,+ rankFeature map[string]float64,+) float64 {+ qDenor := 0.0+ for k, w := range rankFeature {+ if k == common.PAGERANK_FLD {+ continue+ }+ qDenor += w * w+ }+ qDenor = math.Sqrt(qDenor)++ tagScore := 0.0+ if qDenor != 0 {+ var nor, denor float64+ for k, sc := range tagFeas {+ if w, ok := rankFeature[k]; ok && k != common.PAGERANK_FLD {+ nor += w * sc+ }+ denor += sc * sc+ }+ if denor != 0 {+ tagScore = nor / math.Sqrt(denor) / qDenor+ }+ }+ return sim + tagScore*10 + pagerank+}++func TestApplyRankFeatureScores_EmptyInputs(t *testing.T) {+ if got := applyRankFeatureScores(nil, nil, map[string]float64{"a": 1}); len(got) != 0 {+ t.Fatalf("nil chunks -> %v, want empty", got)+ }+ sim := []float64{1, 2}+ if got := applyRankFeatureScores(nil, sim, nil); !floatsClose(got, sim, 0) {+ t.Fatalf("nil chunks with scores -> %v, want unchanged %v", got, sim)+ }+ if got := applyRankFeatureScores([]map[string]interface{}{{}}, nil, nil); len(got) != 0 {+ t.Fatalf("nil sim -> %v, want empty", got)+ }+}++func TestApplyRankFeatureScores_PagerankOnly(t *testing.T) {+ chunks := []map[string]interface{}{+ {common.PAGERANK_FLD: 2.5},+ {common.PAGERANK_FLD: 1},+ {}, // missing pagerank counts as 0+ }+ sim := []float64{1, 1, 1}+ got := applyRankFeatureScores(chunks, sim, nil)+ want := []float64{3.5, 2, 1}+ if !floatsClose(got, want, 1e-9) {+ t.Fatalf("got %v, want %v", got, want)+ }+}++func TestApplyRankFeatureScores_PagerankOnlyQueryNoDivZero(t *testing.T) {+ chunks := []map[string]interface{}{{common.PAGERANK_FLD: 10}}+ sim := []float64{0.5}+ got := applyRankFeatureScores(chunks, sim, map[string]float64{common.PAGERANK_FLD: 10})+ want := []float64{10.5}+ if !floatsClose(got, want, 1e-9) {+ t.Fatalf("got %v, want %v", got, want)+ }+ for _, v := range got {+ if math.IsNaN(v) || math.IsInf(v, 0) {+ t.Fatalf("non-finite score leaked: %v", got)+ }+ }+}++func TestApplyRankFeatureScores_Formula(t *testing.T) {+ rankFeature := map[string]float64{"tag_a": 2, "tag_b": 1}+ chunks := []map[string]interface{}{+ {+ common.TAG_FLD: map[string]interface{}{"tag_a": 0.5, "tag_b": 2.0, "unknown": 1.0},+ common.PAGERANK_FLD: 0.5,+ },+ {common.TAG_FLD: map[string]float64{"tag_a": 1}},+ }+ sim := []float64{0.1, 0.2}+ want := []float64{+ wantRankFeature(0.1, map[string]float64{"tag_a": 0.5, "tag_b": 2.0, "unknown": 1.0}, 0.5, rankFeature),+ wantRankFeature(0.2, map[string]float64{"tag_a": 1}, 0, rankFeature),+ }+ got := applyRankFeatureScores(chunks, sim, rankFeature)+ if !floatsClose(got, want, 1e-9) {+ t.Fatalf("got %v, want %v", got, want)+ }+}++func TestApplyRankFeatureScores_UnknownTagOnlyDenor(t *testing.T) {+ rankFeature := map[string]float64{"tag_a": 1}+ // "unknown" must contribute to denor but not to nor.+ withUnknown := applyRankFeatureScores(+ []map[string]interface{}{{common.TAG_FLD: map[string]float64{"tag_a": 1, "unknown": 1}}},+ []float64{0},+ rankFeature,+ )+ knownOnly := applyRankFeatureScores(+ []map[string]interface{}{{common.TAG_FLD: map[string]float64{"tag_a": 1}}},+ []float64{0},+ rankFeature,+ )+ if withUnknown[0] >= knownOnly[0] {+ t.Fatalf("unknown tag raised the score: %v vs %v", withUnknown, knownOnly)+ }+}++func TestApplyRankFeatureScores_PagerankExcludedFromQDenor(t *testing.T) {+ chunks := []map[string]interface{}{{common.TAG_FLD: map[string]float64{"tag_a": 1}}}+ sim := []float64{0}+ for name, rf := range map[string]map[string]float64{+ "tag only": {"tag_a": 1},+ "tag plus page": {"tag_a": 1, common.PAGERANK_FLD: 1000},+ } {+ t.Run(name, func(t *testing.T) {+ got := applyRankFeatureScores(chunks, append([]float64(nil), sim...), rf)+ want := wantRankFeature(0, map[string]float64{"tag_a": 1}, 0, rf)+ if math.Abs(got[0]-want) > 1e-9 {+ t.Fatalf("got %v, want %v (pagerank must not enter qDenor)", got[0], want)+ }+ })+ }+}++func TestApplyRankFeatureScores_TagFeasFormats(t *testing.T) {+ rankFeature := map[string]float64{"tag_a": 2}+ cases := []struct {+ name string+ form interface{}+ feas map[string]float64+ }{+ {"map interface", map[string]interface{}{"tag_a": 0.5}, map[string]float64{"tag_a": 0.5}},+ {"map float64", map[string]float64{"tag_a": 0.5}, map[string]float64{"tag_a": 0.5}},+ {"map int", map[string]int{"tag_a": 1}, map[string]float64{"tag_a": 1}},+ {"json string", `{"tag_a": 0.5}`, map[string]float64{"tag_a": 0.5}},+ }+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ want := wantRankFeature(0, tc.feas, 0, rankFeature)+ got := applyRankFeatureScores(+ []map[string]interface{}{{common.TAG_FLD: tc.form}},+ []float64{0},+ rankFeature,+ )+ if math.Abs(got[0]-want) > 1e-9 {+ t.Fatalf("got %v, want %v", got[0], want)+ }+ })+ }+}++func TestApplyRankFeatureScores_ZeroDenominator(t *testing.T) {+ cases := map[string]interface{}{+ "empty map": map[string]float64{},+ "all zero": map[string]float64{"tag_a": 0},+ "absent": nil,+ "empty json": "{}",+ }+ rankFeature := map[string]float64{"tag_a": 1}+ for name, tagFeas := range cases {+ t.Run(name, func(t *testing.T) {+ chunk := map[string]interface{}{common.PAGERANK_FLD: 1.5}+ if tagFeas != nil {+ chunk[common.TAG_FLD] = tagFeas+ }+ got := applyRankFeatureScores([]map[string]interface{}{chunk}, []float64{0.5}, rankFeature)+ if math.Abs(got[0]-2.0) > 1e-9 {+ t.Fatalf("got %v, want pagerank-only 2.0", got[0])+ }+ })+ }+}++func TestApplyRankFeatureScores_NoNaNOrInf(t *testing.T) {+ rankFeature := map[string]float64{"tag_a": 2, common.PAGERANK_FLD: 10}+ chunks := []map[string]interface{}{+ {common.PAGERANK_FLD: 1},+ {common.TAG_FLD: map[string]float64{"tag_a": 3}, common.PAGERANK_FLD: 0},+ {common.TAG_FLD: map[string]float64{"tag_b": 3}},+ }+ got := applyRankFeatureScores(chunks, []float64{0.1, 0.2, 0.3}, rankFeature)+ for i, v := range got {+ if math.IsNaN(v) || math.IsInf(v, 0) {+ t.Fatalf("score[%d] is non-finite: %v", i, got)+ }+ }+}++func TestApplyRankFeatureScores_LengthMismatch(t *testing.T) {+ chunks := []map[string]interface{}{{common.PAGERANK_FLD: 1}}+ sim := []float64{0, 5}+ got := applyRankFeatureScores(chunks, sim, nil)+ if !floatsClose(got, []float64{1, 5}, 1e-9) {+ t.Fatalf("got %v, want [1 5] (trailing score preserved)", got)+ }+}++func TestApplyRankFeatureScoresForIDs_MissingAndEquivalent(t *testing.T) {+ rankFeature := map[string]float64{"tag_a": 2}+ chunks := []map[string]interface{}{+ {common.TAG_FLD: map[string]float64{"tag_a": 1}, common.PAGERANK_FLD: 0.5},+ {common.PAGERANK_FLD: 1},+ }+ ids := []string{"a", "missing"}+ field := map[string]map[string]interface{}{"a": chunks[0]}+ sim := []float64{0.1, 0.2}++ byIDs := applyRankFeatureScoresForIDs(ids, field, append([]float64(nil), sim...), rankFeature)+ // The missing id is scored as pagerank 0, tag 0.+ if math.Abs(byIDs[1]-0.2) > 1e-9 {+ t.Fatalf("missing id score = %v, want 0.2", byIDs[1])+ }++ byChunks := applyRankFeatureScores(chunks, append([]float64(nil), sim...), rankFeature)+ if math.Abs(byIDs[0]-byChunks[0]) > 1e-9 {+ t.Fatalf("ids version = %v, chunks version = %v", byIDs[0], byChunks[0])+ }+}++func TestApplyRankFeatureScoresForIDs_LengthMismatch(t *testing.T) {+ ids := []string{"a"}+ field := map[string]map[string]interface{}{"a": {common.PAGERANK_FLD: 2}}+ sim := []float64{0, 7}+ got := applyRankFeatureScoresForIDs(ids, field, sim, nil)+ if !floatsClose(got, []float64{2, 7}, 1e-9) {+ t.Fatalf("got %v, want [2 7] (trailing score preserved)", got)+ }+}diff --git a/internal/service/nlp/reranker_scoring_test.go b/internal/service/nlp/reranker_scoring_test.gonew file mode 100644index 0000000000..88fe48266d--- /dev/null+++ b/internal/service/nlp/reranker_scoring_test.go@@ -0,0 +1,329 @@+// Copyright 2025 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package nlp++import (+ "context"+ "math"+ "testing"++ "ragflow/internal/common"+)++func TestBuildRerankTokens_OrderAndMultipliers(t *testing.T) {+ chunk := map[string]interface{}{+ "content_ltks": "a b a",+ "title_tks": "t1 t2",+ "important_kwd": []string{"i1", "i2"},+ "question_tks": "q1 q2",+ }++ t.Run("standard multipliers", func(t *testing.T) {+ got := buildRerankTokens(chunk, "content_ltks", titleRepeat, importantRepeat, questionRepeat)+ want := []string{+ "a", "b", // content is deduplicated+ "t1", "t2", "t1", "t2",+ "i1", "i2", "i1", "i2", "i1", "i2", "i1", "i2", "i1", "i2",+ "q1", "q2", "q1", "q2", "q1", "q2", "q1", "q2", "q1", "q2", "q1", "q2",+ }+ if len(got) != len(want) {+ t.Fatalf("length = %d, want %d: %v", len(got), len(want), got)+ }+ for i := range want {+ if got[i] != want[i] {+ t.Fatalf("token[%d] = %q, want %q (full: %v)", i, got[i], want[i], got)+ }+ }+ })++ t.Run("no repetition", func(t *testing.T) {+ got := buildRerankTokens(chunk, "content_ltks", 1, 1, 1)+ want := []string{"a", "b", "t1", "t2", "i1", "i2", "q1", "q2"}+ if len(got) != len(want) {+ t.Fatalf("length = %d, want %d: %v", len(got), len(want), got)+ }+ for i := range want {+ if got[i] != want[i] {+ t.Fatalf("token[%d] = %q, want %q", i, got[i], want[i])+ }+ }+ })+}++func TestBuildRerankTokens_MultiplierCounts(t *testing.T) {+ chunk := map[string]interface{}{+ "content_ltks": "c",+ "title_tks": "t",+ "important_kwd": []string{"i"},+ "question_tks": "q",+ }+ got := buildRerankTokens(chunk, "content_ltks", titleRepeat, importantRepeat, questionRepeat)+ counts := make(map[string]int)+ for _, tk := range got {+ counts[tk]+++ }+ want := map[string]int{"c": 1, "t": titleRepeat, "i": importantRepeat, "q": questionRepeat}+ for tk, n := range want {+ if counts[tk] != n {+ t.Fatalf("token %q appears %d times, want %d", tk, counts[tk], n)+ }+ }+ if titleRepeat != 2 || importantRepeat != 5 || questionRepeat != 6 {+ t.Fatalf("multipliers changed: title=%d important=%d question=%d", titleRepeat, importantRepeat, questionRepeat)+ }+}++func TestBuildRerankTokens_EmptyFieldsNonNil(t *testing.T) {+ got := buildRerankTokens(map[string]interface{}{}, "content_ltks", titleRepeat, importantRepeat, questionRepeat)+ if got == nil {+ t.Fatal("expected non-nil empty slice")+ }+ if len(got) != 0 {+ t.Fatalf("got %v, want empty", got)+ }+}++func TestBuildRerankTokens_DedupSemantics(t *testing.T) {+ chunk := map[string]interface{}{+ "content_ltks": "x x y",+ "title_tks": "x x",+ "important_kwd": []string{"x", "x"},+ "question_tks": "x x",+ }+ got := buildRerankTokens(chunk, "content_ltks", 1, 1, 1)+ want := []string{"x", "y", "x", "x", "x", "x", "x", "x"}+ if len(got) != len(want) {+ t.Fatalf("length = %d, want %d: %v", len(got), len(want), got)+ }+ for i := range want {+ if got[i] != want[i] {+ t.Fatalf("token[%d] = %q, want %q (full: %v)", i, got[i], want[i], got)+ }+ }+}++func TestRerankStandard_EmptyChunks(t *testing.T) {+ sim, tsim, vsim := RerankStandard(nil, nil, nil, "alpha", 0.3, 0.7, "content_ltks", nil, nil, "")+ if len(sim) != 0 || len(tsim) != 0 || len(vsim) != 0 {+ t.Fatalf("got sim=%v tsim=%v vsim=%v, want all empty", sim, tsim, vsim)+ }+}++func TestRerankStandard_VectorAndPagerankWiring(t *testing.T) {+ chunks := []map[string]interface{}{+ {+ "content_ltks": "alpha",+ "q_2_vec": []float64{1, 0},+ common.PAGERANK_FLD: 0.5,+ },+ {+ "content_ltks": "beta",+ "q_2_vec": []float64{0, 1},+ },+ }+ sim, tsim, vsim := RerankStandard(+ chunks, nil, []float64{1, 0}, "alpha",+ 0.3, 0.7, "content_ltks", nil, nil, "",+ )+ wantVsim := []float64{1, 0}+ if !floatsClose(vsim, wantVsim, 1e-9) {+ t.Fatalf("vsim = %v, want %v", vsim, wantVsim)+ }+ // qb == nil -> no token score, so the blend is purely vector + pagerank.+ wantSim := []float64{0.7 + 0.5, 0}+ if !floatsClose(sim, wantSim, 1e-9) {+ t.Fatalf("sim = %v, want %v", sim, wantSim)+ }+ if !floatsClose(tsim, []float64{0, 0}, 1e-9) {+ t.Fatalf("tsim = %v, want zeros", tsim)+ }+}++func TestRerankStandard_TokenWiring(t *testing.T) {+ qb := NewQueryBuilder()+ keywords := []string{"alpha", "gamma"}+ chunks := []map[string]interface{}{+ {+ "content_ltks": "alpha beta",+ "title_tks": "gamma",+ "important_kwd": []string{"delta"},+ "question_tks": "epsilon",+ "q_2_vec": []float64{1, 0},+ common.PAGERANK_FLD: 0.25,+ },+ }+ sim, tsim, vsim := RerankStandard(+ chunks, keywords, []float64{1, 0}, "alpha",+ 0.3, 0.7, "content_ltks", qb, nil, "",+ )++ wantTsim := TokenSimilarity(keywords, [][]string{+ buildRerankTokens(chunks[0], "content_ltks", titleRepeat, importantRepeat, questionRepeat),+ }, qb, "")+ if !floatsClose(tsim, wantTsim, 1e-9) {+ t.Fatalf("tsim = %v, want %v", tsim, wantTsim)+ }+ wantSim := []float64{0.3*wantTsim[0] + 0.7*vsim[0] + 0.25}+ if !floatsClose(sim, wantSim, 1e-9) {+ t.Fatalf("sim = %v, want %v", sim, wantSim)+ }+}++func TestRerankStandard_TSimNotPollutedByRankFeature(t *testing.T) {+ qb := NewQueryBuilder()+ chunks := []map[string]interface{}{+ {+ "content_ltks": "alpha",+ common.PAGERANK_FLD: 2.5,+ },+ }+ keywords := []string{"alpha"}+ sim, tsim, vsim := RerankStandard(+ chunks, keywords, nil, "alpha",+ 0.3, 0.7, "content_ltks", qb, nil, "",+ )+ if len(vsim) != 1 || vsim[0] != 0 {+ t.Fatalf("vsim = %v, want [0] for a nil query vector", vsim)+ }+ wantTsim := TokenSimilarity(keywords, [][]string{+ buildRerankTokens(chunks[0], "content_ltks", titleRepeat, importantRepeat, questionRepeat),+ }, qb, "")+ if !floatsClose(tsim, wantTsim, 1e-9) {+ t.Fatalf("tsim = %v, want %v (rank feature must not leak into term_similarity)", tsim, wantTsim)+ }+ // All-zero vectors fall back to tsim, then pagerank is added to sim only.+ if math.Abs(sim[0]-(wantTsim[0]+2.5)) > 1e-9 {+ t.Fatalf("sim = %v, want %v", sim[0], wantTsim[0]+2.5)+ }+}++func TestRerankWithKNN_EmptyIDs(t *testing.T) {+ sim, tsim, vsim := RerankWithKNN(+ context.Background(), nil, nil, nil, nil, "alpha",+ 0.3, 0.7, "content_ltks", nil, nil, "",+ )+ if len(sim) != 0 || len(tsim) != 0 || len(vsim) != 0 {+ t.Fatalf("got sim=%v tsim=%v vsim=%v, want all empty", sim, tsim, vsim)+ }+}++func TestRerankWithKNN_KNNMapping(t *testing.T) {+ qb := NewQueryBuilder()+ ids := []string{"a", "b", "c"}+ field := map[string]map[string]interface{}{+ "a": {"content_ltks": "alpha"},+ "b": {"content_ltks": "beta"},+ "c": {"content_ltks": "gamma"},+ }+ knnScores := map[string]float64{"a": 0.9, "c": 0.4}++ _, _, vsim := RerankWithKNN(+ context.Background(), nil, ids, field, knnScores, "alpha",+ 0.3, 0.7, "content_ltks", qb, nil, "",+ )+ want := []float64{0.9, 0, 0.4}+ if !floatsClose(vsim, want, 1e-9) {+ t.Fatalf("vsim = %v, want %v (missing id must be 0)", vsim, want)+ }+}++func TestRerankWithKNN_FallbackAndMissing(t *testing.T) {+ qb := NewQueryBuilder()+ // "a" is in field; "b" falls back to chunks[i]; "c" has neither.+ ids := []string{"a", "b", "c"}+ field := map[string]map[string]interface{}{+ "a": {"content_ltks": "alpha"},+ }+ chunks := []map[string]interface{}{+ {"content_ltks": "unused"},+ {"content_ltks": "beta"},+ }+ sim, tsim, vsim := RerankWithKNN(+ context.Background(), chunks, ids, field, nil, "alpha",+ 0.3, 0.7, "content_ltks", qb, nil, "",+ )+ if len(sim) != 3 || len(tsim) != 3 || len(vsim) != 3 {+ t.Fatalf("lengths = %d/%d/%d, want 3/3/3", len(sim), len(tsim), len(vsim))+ }+ for i, v := range vsim {+ if v != 0 {+ t.Fatalf("vsim[%d] = %v, want 0 with no knn scores", i, v)+ }+ }+}++func TestRerankWithKNN_ImportantKwdNormalization(t *testing.T) {+ qb := NewQueryBuilder()++ asString := map[string]map[string]interface{}{+ "a": {"content_ltks": "alpha", "important_kwd": "delta"},+ }+ asSlice := map[string]map[string]interface{}{+ "a": {"content_ltks": "alpha", "important_kwd": []string{"delta"}},+ }++ simString, tsimString, _ := RerankWithKNN(+ context.Background(), nil, []string{"a"}, asString, nil, "alpha",+ 0.3, 0.7, "content_ltks", qb, nil, "",+ )+ simSlice, tsimSlice, _ := RerankWithKNN(+ context.Background(), nil, []string{"a"}, asSlice, nil, "alpha",+ 0.3, 0.7, "content_ltks", qb, nil, "",+ )+ if !floatsClose(simString, simSlice, 1e-9) || !floatsClose(tsimString, tsimSlice, 1e-9) {+ t.Fatalf("string vs []string differ: sim %v/%v tsim %v/%v",+ simString, simSlice, tsimString, tsimSlice)+ }+ if _, ok := asString["a"]["important_kwd"].([]string); !ok {+ t.Fatalf("important_kwd was not normalized in place: %T", asString["a"]["important_kwd"])+ }+}++func TestRerankWithKNN_TokenWiring(t *testing.T) {+ qb := NewQueryBuilder()+ ids := []string{"a"}+ field := map[string]map[string]interface{}{+ "a": {+ "content_ltks": "alpha beta",+ "title_tks": "gamma",+ "important_kwd": []string{"delta"},+ "question_tks": "epsilon",+ common.PAGERANK_FLD: 0.5,+ },+ }+ knnScores := map[string]float64{"a": 0.8}+ rankFeature := map[string]float64{"tag_a": 1}++ sim, tsim, vsim := RerankWithKNN(+ context.Background(), nil, ids, field, knnScores, "alpha",+ 0.3, 0.7, "content_ltks", qb, rankFeature, "",+ )+ if !floatsClose(vsim, []float64{0.8}, 1e-9) {+ t.Fatalf("vsim = %v, want [0.8]", vsim)+ }++ _, keywords := qb.Question("alpha", "qa", 0.6, "")+ wantTsim := TokenSimilarity(keywords, [][]string{+ buildRerankTokens(field["a"], "content_ltks", titleRepeat, importantRepeat, questionRepeat),+ }, qb, "")+ if !floatsClose(tsim, wantTsim, 1e-9) {+ t.Fatalf("tsim = %v, want %v", tsim, wantTsim)+ }+ base := 0.3*wantTsim[0] + 0.7*0.8+ want := base + 0.5 // pagerank only; no tag_feas means no tag score+ if math.Abs(sim[0]-want) > 1e-9 {+ t.Fatalf("sim = %v, want %v", sim[0], want)+ }+}diff --git a/internal/service/nlp/reranker_similarity_test.go b/internal/service/nlp/reranker_similarity_test.gonew file mode 100644index 0000000000..59af64b5a6--- /dev/null+++ b/internal/service/nlp/reranker_similarity_test.go@@ -0,0 +1,341 @@+// Copyright 2025 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package nlp++import (+ "math"+ "testing"+)++func TestCosineSimilarity_BasicCases(t *testing.T) {+ cases := []struct {+ name string+ a, b []float64+ want float64+ }{+ {"identical", []float64{1, 2, 3}, []float64{1, 2, 3}, 1},+ {"orthogonal", []float64{1, 0}, []float64{0, 1}, 0},+ {"opposite", []float64{1, 0}, []float64{-1, 0}, -1},+ {"same direction different magnitude", []float64{1, 2}, []float64{2, 4}, 1},+ {"negative scale", []float64{1, 2}, []float64{-2, -4}, -1},+ }+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ got := cosineSimilarity(tc.a, tc.b)+ if math.Abs(got-tc.want) > 1e-9 {+ t.Fatalf("cosineSimilarity(%v, %v) = %v, want %v", tc.a, tc.b, got, tc.want)+ }+ })+ }+}++func TestCosineSimilarity_Degenerate(t *testing.T) {+ cases := []struct {+ name string+ a, b []float64+ }{+ {"length mismatch", []float64{1, 2}, []float64{1, 2, 3}},+ {"left zero", []float64{0, 0}, []float64{1, 2}},+ {"right zero", []float64{1, 2}, []float64{0, 0}},+ {"both empty", nil, nil},+ }+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ got := cosineSimilarity(tc.a, tc.b)+ if got != 0 {+ t.Fatalf("cosineSimilarity(%v, %v) = %v, want 0", tc.a, tc.b, got)+ }+ if math.IsNaN(got) || math.IsInf(got, 0) {+ t.Fatalf("cosineSimilarity returned non-finite value %v", got)+ }+ })+ }+}++func TestCosineSimilarity_NonFiniteInput(t *testing.T) {+ // Non-finite components are not guarded by the implementation: the+ // normalisation still divides by a non-finite norm, so the result is NaN.+ // Pin the boundary so a future guard shows up as a deliberate change.+ got := cosineSimilarity([]float64{math.Inf(1), 0}, []float64{1, 0})+ if !math.IsNaN(got) {+ t.Fatalf("cosineSimilarity with Inf = %v, want NaN (current boundary)", got)+ }+}++func TestTokensToDict_NilQueryBuilderOrTermWeight(t *testing.T) {+ cases := []struct {+ name string+ qb *QueryBuilder+ }{+ {"nil query builder", nil},+ {"nil term weight", &QueryBuilder{}},+ }+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ d, order := tokensToDict([]string{"alpha", "beta"}, tc.qb, "")+ if len(d) != 0 {+ t.Fatalf("dict = %v, want empty", d)+ }+ if len(order) != 0 {+ t.Fatalf("key order = %v, want empty", order)+ }+ })+ }+}++func TestTokensToDict_WeightsAndKeyOrder(t *testing.T) {+ qb := NewQueryBuilder()+ tks := []string{"alpha", "beta", "alpha", "gamma"}++ // Use the dealer's own weights as the input oracle, then apply the 0.4/0.6+ // formula independently. Tokenizer weights are deliberately not hardcoded.+ wts := qb.termWeight.Weights(tks, false, "")+ if len(wts) != len(tks) {+ t.Fatalf("Weights returned %d entries, want %d", len(wts), len(tks))+ }++ want := make(map[string]float64)+ var wantOrder []string+ seen := make(map[string]bool)+ for i, tw := range wts {+ if !seen[tw.Term] {+ seen[tw.Term] = true+ wantOrder = append(wantOrder, tw.Term)+ }+ want[tw.Term] += tw.Weight * 0.4+ if i+1 < len(wts) {+ next := wts[i+1]+ key := tw.Term + next.Term+ if !seen[key] {+ seen[key] = true+ wantOrder = append(wantOrder, key)+ }+ want[key] += math.Max(tw.Weight, next.Weight) * 0.6+ }+ }++ got, gotOrder := tokensToDict(tks, qb, "")+ if len(got) != len(want) {+ t.Fatalf("dict size = %d, want %d (%v vs %v)", len(got), len(want), got, want)+ }+ for k, v := range want {+ if math.Abs(got[k]-v) > 1e-9 {+ t.Fatalf("dict[%q] = %v, want %v", k, got[k], v)+ }+ }+ if len(gotOrder) != len(wantOrder) {+ t.Fatalf("key order = %v, want %v", gotOrder, wantOrder)+ }+ for i := range wantOrder {+ if gotOrder[i] != wantOrder[i] {+ t.Fatalf("key order[%d] = %q, want %q (full: %v vs %v)",+ i, gotOrder[i], wantOrder[i], gotOrder, wantOrder)+ }+ }+ // Repeated terms accumulate instead of being overwritten.+ if got["alpha"] <= 0.4*wts[0].Weight {+ t.Fatalf("repeated term alpha did not accumulate: %v", got["alpha"])+ }+}++func TestTokenDictSimilarity_Cases(t *testing.T) {+ t.Run("empty", func(t *testing.T) {+ if got := tokenDictSimilarity(nil, map[string]float64{"a": 1}, nil); got != 0 {+ t.Fatalf("empty query -> %v, want 0", got)+ }+ if got := tokenDictSimilarity(map[string]float64{"a": 1}, nil, nil); got != 0 {+ t.Fatalf("empty doc -> %v, want 0", got)+ }+ })++ t.Run("full overlap", func(t *testing.T) {+ q := map[string]float64{"a": 1, "b": 1}+ d := map[string]float64{"a": 5, "b": 5}+ got := tokenDictSimilarity(q, d, []string{"a", "b"})+ if math.Abs(got-1) > 1e-9 {+ t.Fatalf("full overlap -> %v, want 1", got)+ }+ })++ t.Run("partial overlap", func(t *testing.T) {+ q := map[string]float64{"a": 1, "b": 3}+ d := map[string]float64{"a": 9}+ want := (1e-9 + 1) / (1e-9 + 4)+ got := tokenDictSimilarity(q, d, []string{"a", "b"})+ if math.Abs(got-want) > 1e-12 {+ t.Fatalf("partial overlap -> %v, want %v", got, want)+ }+ })++ t.Run("no overlap is near zero but not zero", func(t *testing.T) {+ q := map[string]float64{"a": 1}+ d := map[string]float64{"z": 1}+ got := tokenDictSimilarity(q, d, []string{"a"})+ if got == 0 {+ t.Fatalf("no overlap returned exactly 0, smoothing missing")+ }+ if got > 1e-6 {+ t.Fatalf("no overlap -> %v, want near zero", got)+ }+ })+}++func TestTokenDictSimilarity_NilKeyOrderFallback(t *testing.T) {+ q := map[string]float64{"alpha": 1, "beta": 2, "gamma": 3}+ d := map[string]float64{"beta": 1, "gamma": 1}+ withOrder := tokenDictSimilarity(q, d, []string{"alpha", "beta", "gamma"})+ withoutOrder := tokenDictSimilarity(q, d, nil)+ if math.Abs(withOrder-withoutOrder) > 1e-12 {+ t.Fatalf("nil order = %v, explicit sorted order = %v", withoutOrder, withOrder)+ }+}++func TestTokenSimilarity_Cases(t *testing.T) {+ t.Run("empty documents", func(t *testing.T) {+ got := TokenSimilarity([]string{"a"}, nil, NewQueryBuilder(), "")+ if len(got) != 0 {+ t.Fatalf("nil docs -> %v, want empty", got)+ }+ })++ t.Run("nil query builder returns zeros", func(t *testing.T) {+ got := TokenSimilarity([]string{"alpha"}, [][]string{{"alpha"}, {"beta"}}, nil, "")+ if len(got) != 2 {+ t.Fatalf("len = %d, want 2", len(got))+ }+ for i, v := range got {+ if v != 0 {+ t.Fatalf("score[%d] = %v, want 0", i, v)+ }+ }+ })++ qb := NewQueryBuilder()+ t.Run("identical", func(t *testing.T) {+ got := TokenSimilarity([]string{"alpha", "beta"}, [][]string{{"alpha", "beta"}}, qb, "")+ if len(got) != 1 || math.Abs(got[0]-1) > 1e-9 {+ t.Fatalf("identical -> %v, want [1]", got)+ }+ })++ t.Run("disjoint", func(t *testing.T) {+ got := TokenSimilarity([]string{"alpha"}, [][]string{{"omega"}}, qb, "")+ // The 1e-9 smoothing is in both numerator and denominator, so a+ // disjoint pair is near zero but not exactly zero.+ if len(got) != 1 || got[0] > 1e-6 {+ t.Fatalf("disjoint -> %v, want near 0", got)+ }+ })++ t.Run("partial", func(t *testing.T) {+ got := TokenSimilarity([]string{"alpha", "beta"}, [][]string{{"alpha"}}, qb, "")+ if len(got) != 1 || got[0] <= 0 || got[0] >= 1 {+ t.Fatalf("partial -> %v, want 0 < x < 1", got)+ }+ })++ t.Run("stable across calls", func(t *testing.T) {+ atks := []string{"alpha", "beta", "gamma"}+ btkss := [][]string{{"alpha"}, {"beta", "gamma"}, {}}+ first := TokenSimilarity(atks, btkss, qb, "")+ second := TokenSimilarity(atks, btkss, qb, "")+ if !floatsClose(first, second, 0) {+ t.Fatalf("unstable: %v vs %v", first, second)+ }+ })+}++func TestHybridSimilarity_VectorOnly(t *testing.T) {+ avec := []float64{1, 0}+ bvecs := [][]float64{{1, 0}, {0, 1}}+ sim, tsim, vsim := HybridSimilarity(avec, bvecs, []string{"alpha"}, [][]string{{"alpha"}, {"beta"}}, 0.3, 0.7, nil, "")+ if !floatsClose(vsim, []float64{1, 0}, 1e-9) {+ t.Fatalf("vsim = %v, want [1 0]", vsim)+ }+ for i := range tsim {+ if tsim[i] != 0 {+ t.Fatalf("tsim[%d] = %v, want 0 with nil qb", i, tsim[i])+ }+ }+ want := []float64{0.7, 0}+ if !floatsClose(sim, want, 1e-9) {+ t.Fatalf("sim = %v, want %v", sim, want)+ }+}++func TestHybridSimilarity_BlendAndZero(t *testing.T) {+ t.Run("blend", func(t *testing.T) {+ qb := NewQueryBuilder()+ avec := []float64{1, 0}+ bvecs := [][]float64{{1, 0}, {1, 1}}+ atks := []string{"alpha"}+ btkss := [][]string{{"alpha"}, {"beta"}}+ sim, tsim, vsim := HybridSimilarity(avec, bvecs, atks, btkss, 0.3, 0.7, qb, "")+ for i := range sim {+ want := 0.3*tsim[i] + 0.7*vsim[i]+ if math.Abs(sim[i]-want) > 1e-9 {+ t.Fatalf("sim[%d] = %v, want 0.3*%v + 0.7*%v = %v", i, sim[i], tsim[i], vsim[i], want)+ }+ }+ })++ t.Run("all zero vectors", func(t *testing.T) {+ qb := NewQueryBuilder()+ avec := []float64{0, 0}+ bvecs := [][]float64{{0, 0}, {0, 0}}+ sim, tsim, vsim := HybridSimilarity(avec, bvecs, []string{"alpha"}, [][]string{{"alpha"}, {"beta"}}, 0.3, 0.7, qb, "")+ for i := range vsim {+ if vsim[i] != 0 {+ t.Fatalf("vsim[%d] = %v, want 0", i, vsim[i])+ }+ }+ // Initial sim equals tsim; the slice must not be the same backing array.+ if !floatsClose(sim, tsim, 1e-9) {+ t.Fatalf("sim = %v, want tsim = %v", sim, tsim)+ }+ })+}++func TestHybridSimilarity_AllZeroDoesNotAlias(t *testing.T) {+ avec := []float64{0, 0}+ bvecs := [][]float64{{0, 0}}+ sim, tsim, _ := HybridSimilarity(avec, bvecs, []string{"alpha"}, [][]string{{"alpha"}}, 0.3, 0.7, nil, "")+ if len(sim) == 0 || len(tsim) == 0 {+ t.Fatal("expected non-empty result")+ }+ if &sim[0] == &tsim[0] {+ t.Fatal("sim and tsim share a backing array; rank feature scores would leak into term_similarity")+ }+ sim[0] += 42+ if tsim[0] != 0 {+ t.Fatalf("mutating sim changed tsim: %v", tsim)+ }+}++func TestHybridSimilarity_LengthMismatch(t *testing.T) {+ // Known limitation: fewer vectors than token lists indexes past vsim.+ defer func() {+ if r := recover(); r != nil {+ t.Logf("HybridSimilarity panics on vector/token length mismatch (known limitation): %v", r)+ }+ }()+ HybridSimilarity(+ []float64{1, 0},+ [][]float64{{1, 0}},+ []string{"alpha"},+ [][]string{{"alpha"}, {"beta"}},+ 0.3, 0.7, nil, "",+ )+}