Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

[Refactor] Use regex to do hightlight (#19488)

Wang Qia1261c810608
diff --git a/common/doc_store/es_conn_base.py b/common/doc_store/es_conn_base.py
index e745a697b3..6bdb3bdea8 100644
--- a/common/doc_store/es_conn_base.py
+++ b/common/doc_store/es_conn_base.py
@@ -28,7 +28,7 @@ from elasticsearch.client import IndicesClient
from common.file_utils import get_project_base_directory
from common.misc_utils import convert_bytes
from common.doc_store.doc_store_base import DocStoreConnection, OrderByExpr, MatchExpr
-from rag.nlp import is_english, rag_tokenizer
+from rag.nlp import rag_tokenizer
from common import settings
ATTEMPT_TIME = 2
@@ -323,25 +323,10 @@ class ESConnectionBase(DocStoreConnection):
def get_highlight(self, res, keywords: list[str], field_name: str):
ans = {}
+ pattern = re.compile("|".join(re.escape(w) for w in sorted(filter(None, keywords), key=len, reverse=True)), re.IGNORECASE) if any(keywords) else None
for d in res["hits"]["hits"]:
- highlights = d.get("highlight")
- if not highlights:
- continue
- txt = "...".join([a for a in list(highlights.items())[0][1]])
- if not is_english(txt.split()):
- ans[d["_id"]] = txt
- continue
-
txt = d["_source"][field_name]
- txt = re.sub(r"[\r\n]", " ", txt, flags=re.IGNORECASE | re.MULTILINE)
- txt_list = []
- for t in re.split(r"[.?!;\n]", txt):
- for w in keywords:
- t = re.sub(r"(^|[ .?/'\"\(\)!,:;-])(%s)([ .?/'\"\(\)!,:;-])" % re.escape(w), r"\1<em>\2</em>\3", t, flags=re.IGNORECASE | re.MULTILINE)
- if not re.search(r"<em>[^<>]+</em>", t, flags=re.IGNORECASE | re.MULTILINE):
- continue
- txt_list.append(t)
- ans[d["_id"]] = "...".join(txt_list) if txt_list else "...".join([a for a in list(highlights.items())[0][1]])
+ ans[d["_id"]] = pattern.sub(lambda match: f"<em>{match.group(0)}</em>", txt) if pattern else txt
return ans
diff --git a/common/doc_store/infinity_conn_base.py b/common/doc_store/infinity_conn_base.py
index 59e5051aec..07c9ce3cb0 100644
--- a/common/doc_store/infinity_conn_base.py
+++ b/common/doc_store/infinity_conn_base.py
@@ -32,7 +32,6 @@ from infinity.index import IndexInfo, IndexType
from common import settings
from common.doc_store.doc_store_base import DocStoreConnection, MatchExpr, OrderByExpr
from common.file_utils import get_project_base_directory
-from rag.nlp import is_english
# Concurrent CREATE/DROP TABLE on the same Infinity instance can race on
# Infinity's RocksDB-backed catalog counters (e.g. ``db|1|next_table_id``).
@@ -788,55 +787,23 @@ class InfinityConnectionBase(DocStoreConnection):
raise NotImplementedError("Not implemented")
def get_highlight(self, res: tuple[pd.DataFrame, int] | pd.DataFrame, keywords: list[str], field_name: str):
- # Extract DataFrame from result
if isinstance(res, tuple):
df, _ = res
else:
df = res
- if df.empty or field_name not in df.columns:
- return {}
-
- ans = {}
- num_rows = len(res)
- column_id = res["id"]
- if field_name not in res:
- if field_name == "content_with_weight" and "content" in res:
+ if field_name not in df:
+ if field_name == "content_with_weight" and "content" in df:
field_name = "content"
else:
return {}
- for i in range(num_rows):
- id = column_id[i]
- txt = res[field_name][i]
- if re.search(r"<em>[^<>]+</em>", txt, flags=re.IGNORECASE | re.MULTILINE):
- ans[id] = txt
- continue
- txt = re.sub(r"[\r\n]", " ", txt, flags=re.IGNORECASE | re.MULTILINE)
- txt_list = []
- for t in re.split(r"[.?!;\n]", txt):
- if is_english([t]):
- for w in keywords:
- t = re.sub(
- r"(^|[ .?/'\"\(\)!,:;-])(%s)([ .?/'\"\(\)!,:;-])" % re.escape(w),
- r"\1<em>\2</em>\3",
- t,
- flags=re.IGNORECASE | re.MULTILINE,
- )
- else:
- for w in sorted(keywords, key=len, reverse=True):
- t = re.sub(
- re.escape(w),
- f"<em>{w}</em>",
- t,
- flags=re.IGNORECASE | re.MULTILINE,
- )
- if not re.search(r"<em>[^<>]+</em>", t, flags=re.IGNORECASE | re.MULTILINE):
- continue
- txt_list.append(t)
- if txt_list:
- ans[id] = "...".join(txt_list)
- else:
- ans[id] = txt
+ if df.empty:
+ return {}
+
+ ans = {}
+ pattern = re.compile("|".join(re.escape(w) for w in sorted(filter(None, keywords), key=len, reverse=True)), re.IGNORECASE) if any(keywords) else None
+ for id, txt in zip(df["id"], df[field_name]):
+ ans[id] = pattern.sub(lambda match: f"<em>{match.group(0)}</em>", txt) if pattern else txt
return ans
def get_aggregation(self, res: tuple[pd.DataFrame, int] | pd.DataFrame, field_name: str):
diff --git a/rag/nlp/search.py b/rag/nlp/search.py
index cd07f6b918..22baac4160 100644
--- a/rag/nlp/search.py
+++ b/rag/nlp/search.py
@@ -231,11 +231,7 @@ class Dealer:
total = self.dataStore.get_total(res)
logging.debug("Dealer.search TOTAL: {}".format(total))
else:
- highlightFields = ["content_ltks", "title_tks"]
- if not highlight:
- highlightFields = []
- elif isinstance(highlight, list):
- highlightFields = highlight
+ highlightFields = []
matchText, keywords = self.qryr.question(qst, min_match=(0.3 if min_match else 0))
if emb_mdl is None:
matchExprs = [matchText] if matchText else []
@@ -309,9 +305,9 @@ class Dealer:
logging.debug(f"TOTAL: {total}")
ids = self.dataStore.get_doc_ids(res)
keywords = list(kwds)
- highlight = self.dataStore.get_highlight(res, keywords, "content_with_weight")
+ highlightDic = self.dataStore.get_highlight(res, keywords, "content_with_weight") if highlight else {}
aggs = self.dataStore.get_aggregation(res, "docnm_kwd")
- return self.SearchResult(total=total, ids=ids, query_vector=q_vec, aggregation=aggs, highlight=highlight, field=self.dataStore.get_fields(res, src + ["_score"]), keywords=keywords)
+ return self.SearchResult(total=total, ids=ids, query_vector=q_vec, aggregation=aggs, highlight=highlightDic, field=self.dataStore.get_fields(res, src + ["_score"]), keywords=keywords)
@staticmethod
def trans2floats(txt):
@@ -784,11 +780,8 @@ class Dealer:
"mom_id": chunk.get("mom_id", ""),
"row_id": chunk.get("row_id()"),
}
- if highlight and sres.highlight:
- if id in sres.highlight:
- d["highlight"] = remove_redundant_spaces(sres.highlight[id])
- else:
- d["highlight"] = d["content_with_weight"]
+ if id in sres.highlight:
+ d["highlight"] = sres.highlight[id]
ranks["chunks"].append(d)
if aggs:
diff --git a/test/unit_test/rag/nlp/test_gaussdb_retrieval.py b/test/unit_test/rag/nlp/test_gaussdb_retrieval.py
index 3a887d5cc9..057684293b 100644
--- a/test/unit_test/rag/nlp/test_gaussdb_retrieval.py
+++ b/test/unit_test/rag/nlp/test_gaussdb_retrieval.py
@@ -488,6 +488,7 @@ async def test_tc_ret_311_retrieval_keeps_term_only_gaussdb_scores_when_vector_w
total=2,
ids=["term-low", "term-zero"],
query_vector=[0.1, 0.2],
+ highlight={},
field={
"term-low": retrieval_chunk(0.1, "doc-low", "Low"),
"term-zero": retrieval_chunk(0.0, "doc-zero", "Zero"),