Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

Fix missing content_ltks when rerank (#19230)

Wang Qib48cfc131ebe
diff --git a/rag/nlp/search.py b/rag/nlp/search.py
index 51d5b28f79..cd07f6b918 100644
--- a/rag/nlp/search.py
+++ b/rag/nlp/search.py
@@ -513,7 +513,7 @@ class Dealer:
sres.field[i]["important_kwd"] = [sres.field[i]["important_kwd"]]
ins_tw = []
for i in sres.ids:
- content_ltks = list(OrderedDict.fromkeys(sres.field[i][cfield].split()))
+ content_ltks = list(OrderedDict.fromkeys(sres.field[i].get(cfield, "").split()))
title_tks = [t for t in sres.field[i].get("title_tks", "").split() if t]
question_tks = [t for t in sres.field[i].get("question_tks", "").split() if t]
important_kwd = sres.field[i].get("important_kwd", [])
@@ -545,7 +545,7 @@ class Dealer:
sres.field[i]["important_kwd"] = [sres.field[i]["important_kwd"]]
ins_tw = []
for i in sres.ids:
- content_ltks = list(OrderedDict.fromkeys(sres.field[i][cfield].split()))
+ content_ltks = list(OrderedDict.fromkeys(sres.field[i].get(cfield, "").split()))
title_tks = [t for t in sres.field[i].get("title_tks", "").split() if t]
question_tks = [t for t in sres.field[i].get("question_tks", "").split() if t]
important_kwd = sres.field[i].get("important_kwd", [])
@@ -568,7 +568,7 @@ class Dealer:
ins_tw = []
for i in sres.ids:
# content_ltks = list(OrderedDict.fromkeys(sres.field[i][cfield].split()))
- content_ltks = sres.field[i][cfield].split()
+ content_ltks = sres.field[i].get(cfield, "").split()
title_tks = [t for t in sres.field[i].get("title_tks", "").split() if t]
question_tks = [t for t in sres.field[i].get("question_tks", "").split() if t]
important_kwd = sres.field[i].get("important_kwd", [])
@@ -578,7 +578,8 @@ class Dealer:
tks = content_ltks + title_tks + important_kwd + question_tks
ins_tw.append(tks)
- docs = [remove_redundant_spaces(" ".join(tks)) for tks in ins_tw]
+ # if no content_ltks, use content_with_weight instead to avoid empty docs that might cause the reranker to fail with 400 error
+ docs = [remove_redundant_spaces(" ".join(tks)) or str(sres.field[i].get("content_with_weight") or "") for i, tks in zip(sres.ids, ins_tw)]
tksim = self.qryr.token_similarity(keywords, ins_tw)
# rerank_mdl.similarity() returns scores normalized to [0, 1] for every
@@ -766,7 +767,7 @@ class Dealer:
# Dealer.fetch_chunk_vectors when needed.
d = {
"chunk_id": id,
- "content_ltks": chunk["content_ltks"],
+ "content_ltks": chunk.get("content_ltks", ""),
"content_with_weight": chunk.get("content_with_weight", ""),
"doc_id": did,
"docnm_kwd": dnm,