RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
feat(nlp): rerank on natural chunk text, not tokenized content_ltks (#16961)
VincentLambert941edd12696a
diff --git a/internal/service/nlp/reranker.go b/internal/service/nlp/reranker.goindex 01c3b22063..36c6ca0bae 100644--- a/internal/service/nlp/reranker.go+++ b/internal/service/nlp/reranker.go@@ -149,8 +149,18 @@ func RerankByModel(tks = append(tks, questionTks...)insTw = append(insTw, tks)- // Build document text for model reranking- docText := RemoveRedundantSpaces(strings.Join(tks, " "))+ // Feed the reranker the natural chunk text (markup preserved), not the+ // tokenized content_ltks. Neural rerankers score stemmed / accent-split+ // tokens far lower, which collapses relevance scores and forces an+ // artificially low similarity_threshold. The natural text is passed+ // as-is: RemoveRedundantSpaces is ASCII-oriented and mangles+ // multilingual text ("sécurité des données" -> "sécuritédes données"),+ // so it is only applied to the tokenized fallback used when+ // content_with_weight is absent. Mirrors rag/nlp/search.py.+ docText := extractNaturalText(chunk)+ if docText == "" {+ docText = RemoveRedundantSpaces(strings.Join(tks, " "))+ }docs = append(docs, docText)}@@ -555,6 +565,16 @@ func extractContentTokens(fields map[string]interface{}, cfield string) []stringreturn result}+// extractNaturalText returns the natural chunk text (content_with_weight),+// or "" when the field is absent or not a string.+func extractNaturalText(fields map[string]interface{}) string {+ v, ok := fields["content_with_weight"].(string)+ if !ok {+ return ""+ }+ return v+}+// extractTitleTokens extracts title tokens from chunk fieldsfunc extractTitleTokens(fields map[string]interface{}) []string {v, ok := fields["title_tks"].(string)diff --git a/internal/service/nlp/reranker_question_tks_test.go b/internal/service/nlp/reranker_question_tks_test.goindex a0650981b2..a6a2204396 100644--- a/internal/service/nlp/reranker_question_tks_test.go+++ b/internal/service/nlp/reranker_question_tks_test.go@@ -115,3 +115,50 @@ func TestRerankByModelWithoutQuestionTks(t *testing.T) {t.Errorf("got %q, want %q", doc, want)}}++// TestRerankByModelPrefersNaturalText pins the reranker input contract: the+// cross-encoder is scored on content_with_weight (natural text, markup+// preserved, passed through untouched), never on the tokenized fields.+func TestRerankByModelPrefersNaturalText(t *testing.T) {+ chunk := fullChunk()+ chunk["content_with_weight"] = "<p>La sécurité des données : voir l'annexe A.</p>"++ doc := rerankOneChunk(t, chunk)++ if want := chunk["content_with_weight"].(string); doc != want {+ t.Errorf("got %q, want %q", doc, want)+ }+}++// TestRerankByModelNaturalTextKeepsAccentedSpacing guards against+// RemoveRedundantSpaces being applied to natural text: it treats every+// non-ASCII letter as punctuation and would collapse "sécurité des" into+// "sécuritédes".+func TestRerankByModelNaturalTextKeepsAccentedSpacing(t *testing.T) {+ chunk := fullChunk()+ chunk["content_with_weight"] = "sécurité des données"++ doc := rerankOneChunk(t, chunk)++ if doc != "sécurité des données" {+ t.Errorf("natural text was altered: got %q", doc)+ }+ if got := RemoveRedundantSpaces("sécurité des données"); got == "sécurité des données" {+ t.Errorf("test premise broken: RemoveRedundantSpaces no longer mangles accented text (%q)", got)+ }+}++// TestRerankByModelFallsBackToTokensWithoutNaturalText covers chunks with+// no content_with_weight (or an empty one): the tokenized fields are joined+// and cleaned exactly as before.+func TestRerankByModelFallsBackToTokensWithoutNaturalText(t *testing.T) {+ for name, chunk := range map[string]map[string]interface{}{+ "absent": fullChunk(),+ "empty": func() map[string]interface{} { c := fullChunk(); c["content_with_weight"] = ""; return c }(),+ } {+ doc := rerankOneChunk(t, chunk)+ if want := "alpha beta gamma delta epsilon zeta"; doc != want {+ t.Errorf("%s: got %q, want %q", name, doc, want)+ }+ }+}diff --git a/rag/nlp/search.py b/rag/nlp/search.pyindex 22baac4160..ad02320827 100644--- a/rag/nlp/search.py+++ b/rag/nlp/search.py@@ -562,6 +562,7 @@ class Dealer:if isinstance(sres.field[i].get("important_kwd", []), str):sres.field[i]["important_kwd"] = [sres.field[i]["important_kwd"]]ins_tw = []+ rerank_docs = []for i in sres.ids:# content_ltks = list(OrderedDict.fromkeys(sres.field[i][cfield].split()))content_ltks = sres.field[i].get(cfield, "").split()@@ -573,15 +574,23 @@ class Dealer:# duplicating a field would distort the model's own scoring.tks = content_ltks + title_tks + important_kwd + question_tksins_tw.append(tks)-- # if no content_ltks, use content_with_weight instead to avoid empty docs that might cause the reranker to fail with 400 error- docs = [remove_redundant_spaces(" ".join(tks)) or str(sres.field[i].get("content_with_weight") or "") for i, tks in zip(sres.ids, ins_tw)]+ # Feed the reranker the natural chunk text (markup preserved), not the+ # tokenized content_ltks. Neural rerankers score stemmed / accent-split+ # tokens far lower, which collapses relevance scores and forces an+ # artificially low similarity_threshold. The natural text is passed+ # as-is: remove_redundant_spaces() is ASCII-oriented and mangles+ # multilingual text ("sécurité des données" -> "sécuritédes données"),+ # so it is only applied to the tokenized fallback used when+ # content_with_weight is absent. Per-provider truncation to the model+ # window stays the reranker connector's responsibility.+ natural = str(sres.field[i].get("content_with_weight") or "")+ rerank_docs.append(natural or remove_redundant_spaces(" ".join(tks)))tksim = self.qryr.token_similarity(keywords, ins_tw)# rerank_mdl.similarity() returns scores normalized to [0, 1] for every# provider (see RerankModel.Base.similarity), so the blend below stays# on a single scale regardless of the configured reranker.- vtsim, _ = rerank_mdl.similarity(query, docs)+ vtsim, _ = rerank_mdl.similarity(query, rerank_docs)## For rank feature(tag_fea) scores.rank_fea = self._rank_feature_scores(rank_feature, sres)diff --git a/test/unit_test/rag/test_rerank_by_model_input.py b/test/unit_test/rag/test_rerank_by_model_input.pynew file mode 100644index 0000000000..7592ffec88--- /dev/null+++ b/test/unit_test/rag/test_rerank_by_model_input.py@@ -0,0 +1,131 @@+#+# Copyright 2025 The InfiniFlow Authors. All Rights Reserved.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+#+"""Unit tests for the reranker input built by Dealer.rerank_by_model.++External rerankers must be scored on the natural chunk text+(``content_with_weight``, markup preserved), not the tokenized+``content_ltks``: neural rerankers score stemmed / accent-split tokens far+lower, which collapses relevance scores. The tokenized ``ins_tw`` used for the+keyword-similarity part of the blend must stay unchanged.+"""++import sys+import types++import numpy as np+import pytest++# Stub the heavy / circular-importing dependencies before importing search,+# mirroring test_search_pagination.py so the module imports in isolation.+_fake_query = types.ModuleType("rag.nlp.query")+++class _DummyFulltextQueryer:+ pass+++_fake_query.FulltextQueryer = _DummyFulltextQueryer+sys.modules.setdefault("rag.nlp.query", _fake_query)+sys.modules.setdefault("common.settings", types.ModuleType("common.settings"))++from rag.nlp.search import Dealer # noqa: E402++pytestmark = pytest.mark.p1+++class _CapturingRerank:+ """Reranker double that records the documents it is asked to score."""++ def __init__(self):+ self.docs = None++ def similarity(self, query, docs):+ self.docs = list(docs)+ return np.zeros(len(docs)), 0+++def _dealer(token_similarity_spy=None):+ dealer = Dealer.__new__(Dealer)++ class _Queryer:+ def question(self, query):+ return None, ["kw"]++ def token_similarity(self, keywords, ins_tw):+ if token_similarity_spy is not None:+ token_similarity_spy["ins_tw"] = ins_tw+ return [0.0] * len(ins_tw)++ dealer.qryr = _Queryer()+ # Tag/pagerank feature scoring is out of scope here.+ dealer._rank_feature_scores = lambda rank_feature, sres: np.zeros(len(sres.ids))+ return dealer+++def _sres(field_by_id):+ return Dealer.SearchResult(total=len(field_by_id), ids=list(field_by_id), field=field_by_id)+++def test_reranker_receives_natural_text():+ rr = _CapturingRerank()+ sres = _sres({"c1": {"content_with_weight": "Paris is the capital of France.", "content_ltks": "pari capit franc"}})+ _dealer().rerank_by_model(rr, sres, "q")+ assert rr.docs == ["Paris is the capital of France."]+++def test_natural_text_is_passed_as_is_without_space_cleanup():+ # remove_redundant_spaces() is ASCII-oriented: it treats accented letters as+ # punctuation and collapses "sécurité des" into "sécuritédes". Natural text+ # must reach the reranker untouched.+ rr = _CapturingRerank()+ text = "La sécurité des données : voir l'annexe A."+ sres = _sres({"c1": {"content_with_weight": text, "content_ltks": "secur donne annex"}})+ _dealer().rerank_by_model(rr, sres, "q")+ assert rr.docs == [text]++[email protected]("field", [{}, {"content_with_weight": ""}, {"content_with_weight": None}])+def test_falls_back_to_tokenized_when_natural_absent(field):+ rr = _CapturingRerank()+ chunk = {"content_ltks": "pari capit franc", "title_tks": "titre", "important_kwd": ["kw1"], "question_tks": "quest"}+ chunk.update(field)+ sres = _sres({"c1": chunk})+ _dealer().rerank_by_model(rr, sres, "q")+ assert rr.docs == ["pari capit franc titre kw1 quest"]+++def test_fallback_still_cleans_redundant_spaces():+ rr = _CapturingRerank()+ sres = _sres({"c1": {"content_ltks": "( pari ) capit !"}})+ _dealer().rerank_by_model(rr, sres, "q")+ assert rr.docs == ["(pari) capit!"]+++def test_markup_is_preserved_not_stripped():+ rr = _CapturingRerank()+ html = "<table> <thead> <tr> <th>Contact</th> </tr> </thead> </table>"+ sres = _sres({"c1": {"content_with_weight": html, "content_ltks": "contact"}})+ _dealer().rerank_by_model(rr, sres, "q")+ assert "<table>" in rr.docs[0] and "<th>Contact</th>" in rr.docs[0]+++def test_keyword_similarity_still_uses_tokenized_tokens():+ spy = {}+ rr = _CapturingRerank()+ sres = _sres({"c1": {"content_with_weight": "Natural prose.", "content_ltks": "natur pros", "important_kwd": ["kw1"]}})+ _dealer(token_similarity_spy=spy).rerank_by_model(rr, sres, "q")+ # The keyword blend input stays the tokenized tokens, not the natural text.+ assert spy["ins_tw"] == [["natur", "pros", "kw1"]]