Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(rag/nlp): include question_tks when reranking with a rerank model (#18473)

Muhammad Furqan8f24fc2acd92
diff --git a/internal/service/nlp/reranker.go b/internal/service/nlp/reranker.go
index d0d1109562..9c2f338c97 100644
--- a/internal/service/nlp/reranker.go
+++ b/internal/service/nlp/reranker.go
@@ -134,12 +134,16 @@ func RerankByModel(
contentLtks := extractContentTokens(chunk, cfield)
titleTks := extractTitleTokens(chunk)
importantKwd := extractImportantKeywords(chunk)
+ questionTks := extractQuestionTokens(chunk)
- // Combine tokens without repetition (simpler version for model reranking)
- tks := make([]string, 0, len(contentLtks)+len(titleTks)+len(importantKwd))
+ // Unlike RerankStandard/RerankWithKNN, the fields are not repeated here:
+ // these tokens are joined back into `docs` for a cross-encoder, where
+ // duplicating a field would distort the model's own scoring.
+ tks := make([]string, 0, len(contentLtks)+len(titleTks)+len(importantKwd)+len(questionTks))
tks = append(tks, contentLtks...)
tks = append(tks, titleTks...)
tks = append(tks, importantKwd...)
+ tks = append(tks, questionTks...)
insTw = append(insTw, tks)
// Build document text for model reranking
diff --git a/internal/service/nlp/reranker_question_tks_test.go b/internal/service/nlp/reranker_question_tks_test.go
new file mode 100644
index 0000000000..a0650981b2
--- /dev/null
+++ b/internal/service/nlp/reranker_question_tks_test.go
@@ -0,0 +1,117 @@
+// Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package nlp
+
+import (
+ "context"
+ "strings"
+ "testing"
+
+ "ragflow/internal/common"
+ "ragflow/internal/entity/models"
+)
+
+// captureDriver records the documents RerankByModel hands to the cross-encoder,
+// which is the only observable the token assembly produces. The embedded
+// interface satisfies the rest of models.ModelDriver; it is never called.
+type captureDriver struct {
+ models.ModelDriver
+ documents []string
+}
+
+func (d *captureDriver) Rerank(_ context.Context, _ *string, request models.RerankRequest, _ *models.APIConfig, _ *models.RerankConfig, _ *common.ModelUsage) (*models.RerankResponse, error) {
+ d.documents = request.Documents
+ return &models.RerankResponse{}, nil
+}
+
+// rerankOneChunk runs RerankByModel over a single chunk and returns the
+// document text the reranker model received for it.
+func rerankOneChunk(t *testing.T, chunk map[string]interface{}) string {
+ t.Helper()
+
+ driver := &captureDriver{}
+ ids := []string{"c1"}
+ field := map[string]map[string]interface{}{"c1": chunk}
+
+ RerankByModel(
+ context.Background(),
+ &models.RerankModel{ModelDriver: driver},
+ []map[string]interface{}{chunk},
+ ids,
+ field,
+ "epsilon",
+ 0.3,
+ 0.7,
+ "content_ltks",
+ nil,
+ nil,
+ )
+
+ if len(driver.documents) != 1 {
+ t.Fatalf("expected 1 document to reach the reranker model, got %d", len(driver.documents))
+ }
+ return driver.documents[0]
+}
+
+func fullChunk() map[string]interface{} {
+ return map[string]interface{}{
+ "content_ltks": "alpha beta",
+ "title_tks": "gamma",
+ "important_kwd": []string{"delta"},
+ "question_tks": "epsilon zeta",
+ }
+}
+
+// TestRerankByModelIncludesQuestionTks pins the fix: RerankStandard and
+// RerankWithKNN both fold question_tks into their token lists, so a chunk
+// whose generated questions carry the match was scored on those tokens on
+// every path except this one, where the cross-encoder never saw them.
+func TestRerankByModelIncludesQuestionTks(t *testing.T) {
+ doc := rerankOneChunk(t, fullChunk())
+
+ for _, tk := range []string{"alpha", "beta", "gamma", "delta", "epsilon", "zeta"} {
+ if !strings.Contains(doc, tk) {
+ t.Errorf("document %q is missing token %q", doc, tk)
+ }
+ }
+}
+
+// TestRerankByModelDoesNotRepeatFields guards the deliberate difference from
+// RerankStandard/RerankWithKNN, which weight fields by repeating them: these
+// tokens are joined back into a document for a cross-encoder, so a repeated
+// field would distort the model's own scoring.
+func TestRerankByModelDoesNotRepeatFields(t *testing.T) {
+ doc := rerankOneChunk(t, fullChunk())
+
+ for _, tk := range []string{"gamma", "delta", "epsilon", "zeta"} {
+ if got := strings.Count(doc, tk); got != 1 {
+ t.Errorf("token %q appears %d times in %q, want 1", tk, got, doc)
+ }
+ }
+}
+
+// TestRerankByModelWithoutQuestionTks covers chunks that carry no generated
+// questions, which is the common case: the field is absent rather than empty,
+// so extractQuestionTokens must contribute nothing at all.
+func TestRerankByModelWithoutQuestionTks(t *testing.T) {
+ chunk := fullChunk()
+ delete(chunk, "question_tks")
+
+ doc := rerankOneChunk(t, chunk)
+
+ if want := "alpha beta gamma delta"; doc != want {
+ t.Errorf("got %q, want %q", doc, want)
+ }
+}
diff --git a/rag/nlp/search.py b/rag/nlp/search.py
index c211a17e83..445b598880 100644
--- a/rag/nlp/search.py
+++ b/rag/nlp/search.py
@@ -570,8 +570,12 @@ class Dealer:
# content_ltks = list(OrderedDict.fromkeys(sres.field[i][cfield].split()))
content_ltks = sres.field[i][cfield].split()
title_tks = [t for t in sres.field[i].get("title_tks", "").split() if t]
+ question_tks = [t for t in sres.field[i].get("question_tks", "").split() if t]
important_kwd = sres.field[i].get("important_kwd", [])
- tks = content_ltks + title_tks + important_kwd
+ # Unlike rerank()/rerank_with_knn(), the fields are not repeated here:
+ # these tokens are joined back into `docs` for a cross-encoder, where
+ # duplicating a field would distort the model's own scoring.
+ tks = content_ltks + title_tks + important_kwd + question_tks
ins_tw.append(tks)
docs = [remove_redundant_spaces(" ".join(tks)) for tks in ins_tw]
diff --git a/test/unit_test/rag/nlp/test_search_rerank.py b/test/unit_test/rag/nlp/test_search_rerank.py
new file mode 100644
index 0000000000..6f9d53e78a
--- /dev/null
+++ b/test/unit_test/rag/nlp/test_search_rerank.py
@@ -0,0 +1,142 @@
+#
+# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""Rerank token-assembly tests for ``rag.nlp.search.Dealer``.
+
+All three rerank paths build a per-chunk token list (``ins_tw``) that feeds the
+term-similarity score, and ``rerank_by_model`` additionally joins it back into
+the ``docs`` strings handed to the reranker model. These tests pin which chunk
+fields end up in that list.
+"""
+
+from unittest.mock import MagicMock
+
+import numpy as np
+import pytest
+
+# `rag.nlp.query` imports `rag.utils.redis_conn`, which imports `common.settings`,
+# which imports `rag.utils.redis_conn` back. That cycle only resolves when
+# `common.settings` is initialised first -- which is what the API server and task
+# executor do at startup, so importing it here reproduces the runtime order.
+import common.settings # noqa: F401
+
+from rag.nlp.search import Dealer
+
+
+def _dealer(captured):
+ """Dealer with ``__init__`` bypassed (it builds a real FulltextQueryer,
+ which loads the tokenizer) and ``qryr`` stubbed so we can capture ins_tw."""
+ dealer = Dealer.__new__(Dealer)
+ qryr = MagicMock()
+ qryr.question.return_value = (None, ["apple"])
+
+ def _token_similarity(keywords, ins_tw):
+ captured["ins_tw"] = ins_tw
+ return [0.5] * len(ins_tw)
+
+ def _hybrid_similarity(ans_embd, ins_embd, ans, inst, tkweight, vtweight):
+ captured["ins_tw"] = inst
+ n = len(ins_embd)
+ return np.array([0.5] * n), np.array([0.5] * n), np.array([0.5] * n)
+
+ qryr.token_similarity.side_effect = _token_similarity
+ qryr.hybrid_similarity.side_effect = _hybrid_similarity
+ dealer.qryr = qryr
+ return dealer
+
+
+def _sres(**field_overrides):
+ field = {
+ "content_ltks": "apple sells phones",
+ "title_tks": "apple report",
+ "question_tks": "what does apple sell",
+ "important_kwd": ["revenue"],
+ "q_3_vec": [0.1, 0.2, 0.3],
+ }
+ field.update(field_overrides)
+ return Dealer.SearchResult(total=1, ids=["c1"], query_vector=[0.1, 0.2, 0.3], field={"c1": field})
+
+
+def test_rerank_by_model_includes_question_tks():
+ # Regression: rerank_by_model omitted question_tks entirely, so the field
+ # reached neither token_similarity nor the docs sent to the reranker --
+ # even though rerank()/rerank_with_knn() both weight it highest and the
+ # ES query boosts question_tks^20.
+ captured = {}
+ dealer = _dealer(captured)
+ rerank_mdl = MagicMock()
+ rerank_mdl.similarity.return_value = (np.array([0.5]), 0)
+
+ dealer.rerank_by_model(rerank_mdl, _sres(), "apple")
+
+ tokens = captured["ins_tw"][0]
+ assert "what" in tokens and "sell" in tokens
+
+ # The reranker model must see the questions too: ins_tw is joined into docs.
+ docs = rerank_mdl.similarity.call_args[0][1]
+ assert "what does apple sell" in docs[0]
+
+
+def test_rerank_by_model_does_not_repeat_fields():
+ # rerank_by_model deliberately drops the *2/*5/*6 multipliers used by the
+ # other two paths: its tokens are joined back into a string for a
+ # cross-encoder, where duplicating a field would distort the model's score.
+ captured = {}
+ dealer = _dealer(captured)
+ rerank_mdl = MagicMock()
+ rerank_mdl.similarity.return_value = (np.array([0.5]), 0)
+
+ dealer.rerank_by_model(rerank_mdl, _sres(), "apple")
+
+ tokens = captured["ins_tw"][0]
+ assert tokens.count("revenue") == 1
+ assert tokens.count("what") == 1
+
+
[email protected]("path", ["rerank", "rerank_with_knn", "rerank_by_model"])
+def test_all_rerank_paths_include_question_tks(path):
+ # Parity guard: whichever rerank path a deployment takes, the chunk's
+ # generated questions must contribute to the score.
+ captured = {}
+ dealer = _dealer(captured)
+
+ if path == "rerank":
+ dealer.rerank(_sres(), "apple")
+ elif path == "rerank_with_knn":
+ dealer.rerank_with_knn(_sres(), "apple", {"c1": 0.5})
+ else:
+ rerank_mdl = MagicMock()
+ rerank_mdl.similarity.return_value = (np.array([0.5]), 0)
+ dealer.rerank_by_model(rerank_mdl, _sres(), "apple")
+
+ assert "what" in captured["ins_tw"][0], f"{path} dropped question_tks"
+
+
+def test_rerank_by_model_handles_missing_question_tks():
+ # Chunks indexed before question generation have no question_tks field.
+ captured = {}
+ dealer = _dealer(captured)
+ rerank_mdl = MagicMock()
+ rerank_mdl.similarity.return_value = (np.array([0.5]), 0)
+
+ sres = _sres()
+ del sres.field["c1"]["question_tks"]
+ dealer.rerank_by_model(rerank_mdl, sres, "apple")
+
+ tokens = captured["ins_tw"][0]
+ assert "apple" in tokens
+ assert "what" not in tokens