Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

feat(nlp): rerank on natural chunk text, not tokenized content_ltks (#16961)

VincentLambert941edd12696a
diff --git a/internal/service/nlp/reranker.go b/internal/service/nlp/reranker.go
index 01c3b22063..36c6ca0bae 100644
--- a/internal/service/nlp/reranker.go
+++ b/internal/service/nlp/reranker.go
@@ -149,8 +149,18 @@ func RerankByModel(
tks = append(tks, questionTks...)
insTw = append(insTw, tks)
- // Build document text for model reranking
- docText := RemoveRedundantSpaces(strings.Join(tks, " "))
+ // Feed the reranker the natural chunk text (markup preserved), not the
+ // tokenized content_ltks. Neural rerankers score stemmed / accent-split
+ // tokens far lower, which collapses relevance scores and forces an
+ // artificially low similarity_threshold. The natural text is passed
+ // as-is: RemoveRedundantSpaces is ASCII-oriented and mangles
+ // multilingual text ("sécurité des données" -> "sécuritédes données"),
+ // so it is only applied to the tokenized fallback used when
+ // content_with_weight is absent. Mirrors rag/nlp/search.py.
+ docText := extractNaturalText(chunk)
+ if docText == "" {
+ docText = RemoveRedundantSpaces(strings.Join(tks, " "))
+ }
docs = append(docs, docText)
}
@@ -555,6 +565,16 @@ func extractContentTokens(fields map[string]interface{}, cfield string) []string
return result
}
+// extractNaturalText returns the natural chunk text (content_with_weight),
+// or "" when the field is absent or not a string.
+func extractNaturalText(fields map[string]interface{}) string {
+ v, ok := fields["content_with_weight"].(string)
+ if !ok {
+ return ""
+ }
+ return v
+}
+
// extractTitleTokens extracts title tokens from chunk fields
func extractTitleTokens(fields map[string]interface{}) []string {
v, ok := fields["title_tks"].(string)
diff --git a/internal/service/nlp/reranker_question_tks_test.go b/internal/service/nlp/reranker_question_tks_test.go
index a0650981b2..a6a2204396 100644
--- a/internal/service/nlp/reranker_question_tks_test.go
+++ b/internal/service/nlp/reranker_question_tks_test.go
@@ -115,3 +115,50 @@ func TestRerankByModelWithoutQuestionTks(t *testing.T) {
t.Errorf("got %q, want %q", doc, want)
}
}
+
+// TestRerankByModelPrefersNaturalText pins the reranker input contract: the
+// cross-encoder is scored on content_with_weight (natural text, markup
+// preserved, passed through untouched), never on the tokenized fields.
+func TestRerankByModelPrefersNaturalText(t *testing.T) {
+ chunk := fullChunk()
+ chunk["content_with_weight"] = "<p>La sécurité des données : voir l'annexe A.</p>"
+
+ doc := rerankOneChunk(t, chunk)
+
+ if want := chunk["content_with_weight"].(string); doc != want {
+ t.Errorf("got %q, want %q", doc, want)
+ }
+}
+
+// TestRerankByModelNaturalTextKeepsAccentedSpacing guards against
+// RemoveRedundantSpaces being applied to natural text: it treats every
+// non-ASCII letter as punctuation and would collapse "sécurité des" into
+// "sécuritédes".
+func TestRerankByModelNaturalTextKeepsAccentedSpacing(t *testing.T) {
+ chunk := fullChunk()
+ chunk["content_with_weight"] = "sécurité des données"
+
+ doc := rerankOneChunk(t, chunk)
+
+ if doc != "sécurité des données" {
+ t.Errorf("natural text was altered: got %q", doc)
+ }
+ if got := RemoveRedundantSpaces("sécurité des données"); got == "sécurité des données" {
+ t.Errorf("test premise broken: RemoveRedundantSpaces no longer mangles accented text (%q)", got)
+ }
+}
+
+// TestRerankByModelFallsBackToTokensWithoutNaturalText covers chunks with
+// no content_with_weight (or an empty one): the tokenized fields are joined
+// and cleaned exactly as before.
+func TestRerankByModelFallsBackToTokensWithoutNaturalText(t *testing.T) {
+ for name, chunk := range map[string]map[string]interface{}{
+ "absent": fullChunk(),
+ "empty": func() map[string]interface{} { c := fullChunk(); c["content_with_weight"] = ""; return c }(),
+ } {
+ doc := rerankOneChunk(t, chunk)
+ if want := "alpha beta gamma delta epsilon zeta"; doc != want {
+ t.Errorf("%s: got %q, want %q", name, doc, want)
+ }
+ }
+}
diff --git a/rag/nlp/search.py b/rag/nlp/search.py
index 22baac4160..ad02320827 100644
--- a/rag/nlp/search.py
+++ b/rag/nlp/search.py
@@ -562,6 +562,7 @@ class Dealer:
if isinstance(sres.field[i].get("important_kwd", []), str):
sres.field[i]["important_kwd"] = [sres.field[i]["important_kwd"]]
ins_tw = []
+ rerank_docs = []
for i in sres.ids:
# content_ltks = list(OrderedDict.fromkeys(sres.field[i][cfield].split()))
content_ltks = sres.field[i].get(cfield, "").split()
@@ -573,15 +574,23 @@ class Dealer:
# duplicating a field would distort the model's own scoring.
tks = content_ltks + title_tks + important_kwd + question_tks
ins_tw.append(tks)
-
- # if no content_ltks, use content_with_weight instead to avoid empty docs that might cause the reranker to fail with 400 error
- docs = [remove_redundant_spaces(" ".join(tks)) or str(sres.field[i].get("content_with_weight") or "") for i, tks in zip(sres.ids, ins_tw)]
+ # Feed the reranker the natural chunk text (markup preserved), not the
+ # tokenized content_ltks. Neural rerankers score stemmed / accent-split
+ # tokens far lower, which collapses relevance scores and forces an
+ # artificially low similarity_threshold. The natural text is passed
+ # as-is: remove_redundant_spaces() is ASCII-oriented and mangles
+ # multilingual text ("sécurité des données" -> "sécuritédes données"),
+ # so it is only applied to the tokenized fallback used when
+ # content_with_weight is absent. Per-provider truncation to the model
+ # window stays the reranker connector's responsibility.
+ natural = str(sres.field[i].get("content_with_weight") or "")
+ rerank_docs.append(natural or remove_redundant_spaces(" ".join(tks)))
tksim = self.qryr.token_similarity(keywords, ins_tw)
# rerank_mdl.similarity() returns scores normalized to [0, 1] for every
# provider (see RerankModel.Base.similarity), so the blend below stays
# on a single scale regardless of the configured reranker.
- vtsim, _ = rerank_mdl.similarity(query, docs)
+ vtsim, _ = rerank_mdl.similarity(query, rerank_docs)
## For rank feature(tag_fea) scores.
rank_fea = self._rank_feature_scores(rank_feature, sres)
diff --git a/test/unit_test/rag/test_rerank_by_model_input.py b/test/unit_test/rag/test_rerank_by_model_input.py
new file mode 100644
index 0000000000..7592ffec88
--- /dev/null
+++ b/test/unit_test/rag/test_rerank_by_model_input.py
@@ -0,0 +1,131 @@
+#
+# Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""Unit tests for the reranker input built by Dealer.rerank_by_model.
+
+External rerankers must be scored on the natural chunk text
+(``content_with_weight``, markup preserved), not the tokenized
+``content_ltks``: neural rerankers score stemmed / accent-split tokens far
+lower, which collapses relevance scores. The tokenized ``ins_tw`` used for the
+keyword-similarity part of the blend must stay unchanged.
+"""
+
+import sys
+import types
+
+import numpy as np
+import pytest
+
+# Stub the heavy / circular-importing dependencies before importing search,
+# mirroring test_search_pagination.py so the module imports in isolation.
+_fake_query = types.ModuleType("rag.nlp.query")
+
+
+class _DummyFulltextQueryer:
+ pass
+
+
+_fake_query.FulltextQueryer = _DummyFulltextQueryer
+sys.modules.setdefault("rag.nlp.query", _fake_query)
+sys.modules.setdefault("common.settings", types.ModuleType("common.settings"))
+
+from rag.nlp.search import Dealer # noqa: E402
+
+pytestmark = pytest.mark.p1
+
+
+class _CapturingRerank:
+ """Reranker double that records the documents it is asked to score."""
+
+ def __init__(self):
+ self.docs = None
+
+ def similarity(self, query, docs):
+ self.docs = list(docs)
+ return np.zeros(len(docs)), 0
+
+
+def _dealer(token_similarity_spy=None):
+ dealer = Dealer.__new__(Dealer)
+
+ class _Queryer:
+ def question(self, query):
+ return None, ["kw"]
+
+ def token_similarity(self, keywords, ins_tw):
+ if token_similarity_spy is not None:
+ token_similarity_spy["ins_tw"] = ins_tw
+ return [0.0] * len(ins_tw)
+
+ dealer.qryr = _Queryer()
+ # Tag/pagerank feature scoring is out of scope here.
+ dealer._rank_feature_scores = lambda rank_feature, sres: np.zeros(len(sres.ids))
+ return dealer
+
+
+def _sres(field_by_id):
+ return Dealer.SearchResult(total=len(field_by_id), ids=list(field_by_id), field=field_by_id)
+
+
+def test_reranker_receives_natural_text():
+ rr = _CapturingRerank()
+ sres = _sres({"c1": {"content_with_weight": "Paris is the capital of France.", "content_ltks": "pari capit franc"}})
+ _dealer().rerank_by_model(rr, sres, "q")
+ assert rr.docs == ["Paris is the capital of France."]
+
+
+def test_natural_text_is_passed_as_is_without_space_cleanup():
+ # remove_redundant_spaces() is ASCII-oriented: it treats accented letters as
+ # punctuation and collapses "sécurité des" into "sécuritédes". Natural text
+ # must reach the reranker untouched.
+ rr = _CapturingRerank()
+ text = "La sécurité des données : voir l'annexe A."
+ sres = _sres({"c1": {"content_with_weight": text, "content_ltks": "secur donne annex"}})
+ _dealer().rerank_by_model(rr, sres, "q")
+ assert rr.docs == [text]
+
+
[email protected]("field", [{}, {"content_with_weight": ""}, {"content_with_weight": None}])
+def test_falls_back_to_tokenized_when_natural_absent(field):
+ rr = _CapturingRerank()
+ chunk = {"content_ltks": "pari capit franc", "title_tks": "titre", "important_kwd": ["kw1"], "question_tks": "quest"}
+ chunk.update(field)
+ sres = _sres({"c1": chunk})
+ _dealer().rerank_by_model(rr, sres, "q")
+ assert rr.docs == ["pari capit franc titre kw1 quest"]
+
+
+def test_fallback_still_cleans_redundant_spaces():
+ rr = _CapturingRerank()
+ sres = _sres({"c1": {"content_ltks": "( pari ) capit !"}})
+ _dealer().rerank_by_model(rr, sres, "q")
+ assert rr.docs == ["(pari) capit!"]
+
+
+def test_markup_is_preserved_not_stripped():
+ rr = _CapturingRerank()
+ html = "<table> <thead> <tr> <th>Contact</th> </tr> </thead> </table>"
+ sres = _sres({"c1": {"content_with_weight": html, "content_ltks": "contact"}})
+ _dealer().rerank_by_model(rr, sres, "q")
+ assert "<table>" in rr.docs[0] and "<th>Contact</th>" in rr.docs[0]
+
+
+def test_keyword_similarity_still_uses_tokenized_tokens():
+ spy = {}
+ rr = _CapturingRerank()
+ sres = _sres({"c1": {"content_with_weight": "Natural prose.", "content_ltks": "natur pros", "important_kwd": ["kw1"]}})
+ _dealer(token_similarity_spy=spy).rerank_by_model(rr, sres, "q")
+ # The keyword blend input stays the tokenized tokens, not the natural text.
+ assert spy["ins_tw"] == [["natur", "pros", "kw1"]]