RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Fix missing content_ltks when rerank (#19230)
Wang Qib48cfc131ebe
diff --git a/rag/nlp/search.py b/rag/nlp/search.pyindex 51d5b28f79..cd07f6b918 100644--- a/rag/nlp/search.py+++ b/rag/nlp/search.py@@ -513,7 +513,7 @@ class Dealer:sres.field[i]["important_kwd"] = [sres.field[i]["important_kwd"]]ins_tw = []for i in sres.ids:- content_ltks = list(OrderedDict.fromkeys(sres.field[i][cfield].split()))+ content_ltks = list(OrderedDict.fromkeys(sres.field[i].get(cfield, "").split()))title_tks = [t for t in sres.field[i].get("title_tks", "").split() if t]question_tks = [t for t in sres.field[i].get("question_tks", "").split() if t]important_kwd = sres.field[i].get("important_kwd", [])@@ -545,7 +545,7 @@ class Dealer:sres.field[i]["important_kwd"] = [sres.field[i]["important_kwd"]]ins_tw = []for i in sres.ids:- content_ltks = list(OrderedDict.fromkeys(sres.field[i][cfield].split()))+ content_ltks = list(OrderedDict.fromkeys(sres.field[i].get(cfield, "").split()))title_tks = [t for t in sres.field[i].get("title_tks", "").split() if t]question_tks = [t for t in sres.field[i].get("question_tks", "").split() if t]important_kwd = sres.field[i].get("important_kwd", [])@@ -568,7 +568,7 @@ class Dealer:ins_tw = []for i in sres.ids:# content_ltks = list(OrderedDict.fromkeys(sres.field[i][cfield].split()))- content_ltks = sres.field[i][cfield].split()+ content_ltks = sres.field[i].get(cfield, "").split()title_tks = [t for t in sres.field[i].get("title_tks", "").split() if t]question_tks = [t for t in sres.field[i].get("question_tks", "").split() if t]important_kwd = sres.field[i].get("important_kwd", [])@@ -578,7 +578,8 @@ class Dealer:tks = content_ltks + title_tks + important_kwd + question_tksins_tw.append(tks)- docs = [remove_redundant_spaces(" ".join(tks)) for tks in ins_tw]+ # if no content_ltks, use content_with_weight instead to avoid empty docs that might cause the reranker to fail with 400 error+ docs = [remove_redundant_spaces(" ".join(tks)) or str(sres.field[i].get("content_with_weight") or "") for i, tks in zip(sres.ids, ins_tw)]tksim = self.qryr.token_similarity(keywords, ins_tw)# rerank_mdl.similarity() returns scores normalized to [0, 1] for every@@ -766,7 +767,7 @@ class Dealer:# Dealer.fetch_chunk_vectors when needed.d = {"chunk_id": id,- "content_ltks": chunk["content_ltks"],+ "content_ltks": chunk.get("content_ltks", ""),"content_with_weight": chunk.get("content_with_weight", ""),"doc_id": did,"docnm_kwd": dnm,