RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(rag/nlp): include question_tks when reranking with a rerank model (#18473)
Muhammad Furqan8f24fc2acd92
diff --git a/internal/service/nlp/reranker.go b/internal/service/nlp/reranker.goindex d0d1109562..9c2f338c97 100644--- a/internal/service/nlp/reranker.go+++ b/internal/service/nlp/reranker.go@@ -134,12 +134,16 @@ func RerankByModel(contentLtks := extractContentTokens(chunk, cfield)titleTks := extractTitleTokens(chunk)importantKwd := extractImportantKeywords(chunk)+ questionTks := extractQuestionTokens(chunk)- // Combine tokens without repetition (simpler version for model reranking)- tks := make([]string, 0, len(contentLtks)+len(titleTks)+len(importantKwd))+ // Unlike RerankStandard/RerankWithKNN, the fields are not repeated here:+ // these tokens are joined back into `docs` for a cross-encoder, where+ // duplicating a field would distort the model's own scoring.+ tks := make([]string, 0, len(contentLtks)+len(titleTks)+len(importantKwd)+len(questionTks))tks = append(tks, contentLtks...)tks = append(tks, titleTks...)tks = append(tks, importantKwd...)+ tks = append(tks, questionTks...)insTw = append(insTw, tks)// Build document text for model rerankingdiff --git a/internal/service/nlp/reranker_question_tks_test.go b/internal/service/nlp/reranker_question_tks_test.gonew file mode 100644index 0000000000..a0650981b2--- /dev/null+++ b/internal/service/nlp/reranker_question_tks_test.go@@ -0,0 +1,117 @@+// Copyright 2025 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package nlp++import (+ "context"+ "strings"+ "testing"++ "ragflow/internal/common"+ "ragflow/internal/entity/models"+)++// captureDriver records the documents RerankByModel hands to the cross-encoder,+// which is the only observable the token assembly produces. The embedded+// interface satisfies the rest of models.ModelDriver; it is never called.+type captureDriver struct {+ models.ModelDriver+ documents []string+}++func (d *captureDriver) Rerank(_ context.Context, _ *string, request models.RerankRequest, _ *models.APIConfig, _ *models.RerankConfig, _ *common.ModelUsage) (*models.RerankResponse, error) {+ d.documents = request.Documents+ return &models.RerankResponse{}, nil+}++// rerankOneChunk runs RerankByModel over a single chunk and returns the+// document text the reranker model received for it.+func rerankOneChunk(t *testing.T, chunk map[string]interface{}) string {+ t.Helper()++ driver := &captureDriver{}+ ids := []string{"c1"}+ field := map[string]map[string]interface{}{"c1": chunk}++ RerankByModel(+ context.Background(),+ &models.RerankModel{ModelDriver: driver},+ []map[string]interface{}{chunk},+ ids,+ field,+ "epsilon",+ 0.3,+ 0.7,+ "content_ltks",+ nil,+ nil,+ )++ if len(driver.documents) != 1 {+ t.Fatalf("expected 1 document to reach the reranker model, got %d", len(driver.documents))+ }+ return driver.documents[0]+}++func fullChunk() map[string]interface{} {+ return map[string]interface{}{+ "content_ltks": "alpha beta",+ "title_tks": "gamma",+ "important_kwd": []string{"delta"},+ "question_tks": "epsilon zeta",+ }+}++// TestRerankByModelIncludesQuestionTks pins the fix: RerankStandard and+// RerankWithKNN both fold question_tks into their token lists, so a chunk+// whose generated questions carry the match was scored on those tokens on+// every path except this one, where the cross-encoder never saw them.+func TestRerankByModelIncludesQuestionTks(t *testing.T) {+ doc := rerankOneChunk(t, fullChunk())++ for _, tk := range []string{"alpha", "beta", "gamma", "delta", "epsilon", "zeta"} {+ if !strings.Contains(doc, tk) {+ t.Errorf("document %q is missing token %q", doc, tk)+ }+ }+}++// TestRerankByModelDoesNotRepeatFields guards the deliberate difference from+// RerankStandard/RerankWithKNN, which weight fields by repeating them: these+// tokens are joined back into a document for a cross-encoder, so a repeated+// field would distort the model's own scoring.+func TestRerankByModelDoesNotRepeatFields(t *testing.T) {+ doc := rerankOneChunk(t, fullChunk())++ for _, tk := range []string{"gamma", "delta", "epsilon", "zeta"} {+ if got := strings.Count(doc, tk); got != 1 {+ t.Errorf("token %q appears %d times in %q, want 1", tk, got, doc)+ }+ }+}++// TestRerankByModelWithoutQuestionTks covers chunks that carry no generated+// questions, which is the common case: the field is absent rather than empty,+// so extractQuestionTokens must contribute nothing at all.+func TestRerankByModelWithoutQuestionTks(t *testing.T) {+ chunk := fullChunk()+ delete(chunk, "question_tks")++ doc := rerankOneChunk(t, chunk)++ if want := "alpha beta gamma delta"; doc != want {+ t.Errorf("got %q, want %q", doc, want)+ }+}diff --git a/rag/nlp/search.py b/rag/nlp/search.pyindex c211a17e83..445b598880 100644--- a/rag/nlp/search.py+++ b/rag/nlp/search.py@@ -570,8 +570,12 @@ class Dealer:# content_ltks = list(OrderedDict.fromkeys(sres.field[i][cfield].split()))content_ltks = sres.field[i][cfield].split()title_tks = [t for t in sres.field[i].get("title_tks", "").split() if t]+ question_tks = [t for t in sres.field[i].get("question_tks", "").split() if t]important_kwd = sres.field[i].get("important_kwd", [])- tks = content_ltks + title_tks + important_kwd+ # Unlike rerank()/rerank_with_knn(), the fields are not repeated here:+ # these tokens are joined back into `docs` for a cross-encoder, where+ # duplicating a field would distort the model's own scoring.+ tks = content_ltks + title_tks + important_kwd + question_tksins_tw.append(tks)docs = [remove_redundant_spaces(" ".join(tks)) for tks in ins_tw]diff --git a/test/unit_test/rag/nlp/test_search_rerank.py b/test/unit_test/rag/nlp/test_search_rerank.pynew file mode 100644index 0000000000..6f9d53e78a--- /dev/null+++ b/test/unit_test/rag/nlp/test_search_rerank.py@@ -0,0 +1,142 @@+#+# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+#++"""Rerank token-assembly tests for ``rag.nlp.search.Dealer``.++All three rerank paths build a per-chunk token list (``ins_tw``) that feeds the+term-similarity score, and ``rerank_by_model`` additionally joins it back into+the ``docs`` strings handed to the reranker model. These tests pin which chunk+fields end up in that list.+"""++from unittest.mock import MagicMock++import numpy as np+import pytest++# `rag.nlp.query` imports `rag.utils.redis_conn`, which imports `common.settings`,+# which imports `rag.utils.redis_conn` back. That cycle only resolves when+# `common.settings` is initialised first -- which is what the API server and task+# executor do at startup, so importing it here reproduces the runtime order.+import common.settings # noqa: F401++from rag.nlp.search import Dealer+++def _dealer(captured):+ """Dealer with ``__init__`` bypassed (it builds a real FulltextQueryer,+ which loads the tokenizer) and ``qryr`` stubbed so we can capture ins_tw."""+ dealer = Dealer.__new__(Dealer)+ qryr = MagicMock()+ qryr.question.return_value = (None, ["apple"])++ def _token_similarity(keywords, ins_tw):+ captured["ins_tw"] = ins_tw+ return [0.5] * len(ins_tw)++ def _hybrid_similarity(ans_embd, ins_embd, ans, inst, tkweight, vtweight):+ captured["ins_tw"] = inst+ n = len(ins_embd)+ return np.array([0.5] * n), np.array([0.5] * n), np.array([0.5] * n)++ qryr.token_similarity.side_effect = _token_similarity+ qryr.hybrid_similarity.side_effect = _hybrid_similarity+ dealer.qryr = qryr+ return dealer+++def _sres(**field_overrides):+ field = {+ "content_ltks": "apple sells phones",+ "title_tks": "apple report",+ "question_tks": "what does apple sell",+ "important_kwd": ["revenue"],+ "q_3_vec": [0.1, 0.2, 0.3],+ }+ field.update(field_overrides)+ return Dealer.SearchResult(total=1, ids=["c1"], query_vector=[0.1, 0.2, 0.3], field={"c1": field})+++def test_rerank_by_model_includes_question_tks():+ # Regression: rerank_by_model omitted question_tks entirely, so the field+ # reached neither token_similarity nor the docs sent to the reranker --+ # even though rerank()/rerank_with_knn() both weight it highest and the+ # ES query boosts question_tks^20.+ captured = {}+ dealer = _dealer(captured)+ rerank_mdl = MagicMock()+ rerank_mdl.similarity.return_value = (np.array([0.5]), 0)++ dealer.rerank_by_model(rerank_mdl, _sres(), "apple")++ tokens = captured["ins_tw"][0]+ assert "what" in tokens and "sell" in tokens++ # The reranker model must see the questions too: ins_tw is joined into docs.+ docs = rerank_mdl.similarity.call_args[0][1]+ assert "what does apple sell" in docs[0]+++def test_rerank_by_model_does_not_repeat_fields():+ # rerank_by_model deliberately drops the *2/*5/*6 multipliers used by the+ # other two paths: its tokens are joined back into a string for a+ # cross-encoder, where duplicating a field would distort the model's score.+ captured = {}+ dealer = _dealer(captured)+ rerank_mdl = MagicMock()+ rerank_mdl.similarity.return_value = (np.array([0.5]), 0)++ dealer.rerank_by_model(rerank_mdl, _sres(), "apple")++ tokens = captured["ins_tw"][0]+ assert tokens.count("revenue") == 1+ assert tokens.count("what") == 1++[email protected]("path", ["rerank", "rerank_with_knn", "rerank_by_model"])+def test_all_rerank_paths_include_question_tks(path):+ # Parity guard: whichever rerank path a deployment takes, the chunk's+ # generated questions must contribute to the score.+ captured = {}+ dealer = _dealer(captured)++ if path == "rerank":+ dealer.rerank(_sres(), "apple")+ elif path == "rerank_with_knn":+ dealer.rerank_with_knn(_sres(), "apple", {"c1": 0.5})+ else:+ rerank_mdl = MagicMock()+ rerank_mdl.similarity.return_value = (np.array([0.5]), 0)+ dealer.rerank_by_model(rerank_mdl, _sres(), "apple")++ assert "what" in captured["ins_tw"][0], f"{path} dropped question_tks"+++def test_rerank_by_model_handles_missing_question_tks():+ # Chunks indexed before question generation have no question_tks field.+ captured = {}+ dealer = _dealer(captured)+ rerank_mdl = MagicMock()+ rerank_mdl.similarity.return_value = (np.array([0.5]), 0)++ sres = _sres()+ del sres.field["c1"]["question_tks"]+ dealer.rerank_by_model(rerank_mdl, sres, "apple")++ tokens = captured["ins_tw"][0]+ assert "apple" in tokens+ assert "what" not in tokens