RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
[Refactor] Use regex to do hightlight (#19488)
Wang Qia1261c810608
diff --git a/common/doc_store/es_conn_base.py b/common/doc_store/es_conn_base.pyindex e745a697b3..6bdb3bdea8 100644--- a/common/doc_store/es_conn_base.py+++ b/common/doc_store/es_conn_base.py@@ -28,7 +28,7 @@ from elasticsearch.client import IndicesClientfrom common.file_utils import get_project_base_directoryfrom common.misc_utils import convert_bytesfrom common.doc_store.doc_store_base import DocStoreConnection, OrderByExpr, MatchExpr-from rag.nlp import is_english, rag_tokenizer+from rag.nlp import rag_tokenizerfrom common import settingsATTEMPT_TIME = 2@@ -323,25 +323,10 @@ class ESConnectionBase(DocStoreConnection):def get_highlight(self, res, keywords: list[str], field_name: str):ans = {}+ pattern = re.compile("|".join(re.escape(w) for w in sorted(filter(None, keywords), key=len, reverse=True)), re.IGNORECASE) if any(keywords) else Nonefor d in res["hits"]["hits"]:- highlights = d.get("highlight")- if not highlights:- continue- txt = "...".join([a for a in list(highlights.items())[0][1]])- if not is_english(txt.split()):- ans[d["_id"]] = txt- continue-txt = d["_source"][field_name]- txt = re.sub(r"[\r\n]", " ", txt, flags=re.IGNORECASE | re.MULTILINE)- txt_list = []- for t in re.split(r"[.?!;\n]", txt):- for w in keywords:- t = re.sub(r"(^|[ .?/'\"\(\)!,:;-])(%s)([ .?/'\"\(\)!,:;-])" % re.escape(w), r"\1<em>\2</em>\3", t, flags=re.IGNORECASE | re.MULTILINE)- if not re.search(r"<em>[^<>]+</em>", t, flags=re.IGNORECASE | re.MULTILINE):- continue- txt_list.append(t)- ans[d["_id"]] = "...".join(txt_list) if txt_list else "...".join([a for a in list(highlights.items())[0][1]])+ ans[d["_id"]] = pattern.sub(lambda match: f"<em>{match.group(0)}</em>", txt) if pattern else txtreturn ansdiff --git a/common/doc_store/infinity_conn_base.py b/common/doc_store/infinity_conn_base.pyindex 59e5051aec..07c9ce3cb0 100644--- a/common/doc_store/infinity_conn_base.py+++ b/common/doc_store/infinity_conn_base.py@@ -32,7 +32,6 @@ from infinity.index import IndexInfo, IndexTypefrom common import settingsfrom common.doc_store.doc_store_base import DocStoreConnection, MatchExpr, OrderByExprfrom common.file_utils import get_project_base_directory-from rag.nlp import is_english# Concurrent CREATE/DROP TABLE on the same Infinity instance can race on# Infinity's RocksDB-backed catalog counters (e.g. ``db|1|next_table_id``).@@ -788,55 +787,23 @@ class InfinityConnectionBase(DocStoreConnection):raise NotImplementedError("Not implemented")def get_highlight(self, res: tuple[pd.DataFrame, int] | pd.DataFrame, keywords: list[str], field_name: str):- # Extract DataFrame from resultif isinstance(res, tuple):df, _ = reselse:df = res- if df.empty or field_name not in df.columns:- return {}-- ans = {}- num_rows = len(res)- column_id = res["id"]- if field_name not in res:- if field_name == "content_with_weight" and "content" in res:+ if field_name not in df:+ if field_name == "content_with_weight" and "content" in df:field_name = "content"else:return {}- for i in range(num_rows):- id = column_id[i]- txt = res[field_name][i]- if re.search(r"<em>[^<>]+</em>", txt, flags=re.IGNORECASE | re.MULTILINE):- ans[id] = txt- continue- txt = re.sub(r"[\r\n]", " ", txt, flags=re.IGNORECASE | re.MULTILINE)- txt_list = []- for t in re.split(r"[.?!;\n]", txt):- if is_english([t]):- for w in keywords:- t = re.sub(- r"(^|[ .?/'\"\(\)!,:;-])(%s)([ .?/'\"\(\)!,:;-])" % re.escape(w),- r"\1<em>\2</em>\3",- t,- flags=re.IGNORECASE | re.MULTILINE,- )- else:- for w in sorted(keywords, key=len, reverse=True):- t = re.sub(- re.escape(w),- f"<em>{w}</em>",- t,- flags=re.IGNORECASE | re.MULTILINE,- )- if not re.search(r"<em>[^<>]+</em>", t, flags=re.IGNORECASE | re.MULTILINE):- continue- txt_list.append(t)- if txt_list:- ans[id] = "...".join(txt_list)- else:- ans[id] = txt+ if df.empty:+ return {}++ ans = {}+ pattern = re.compile("|".join(re.escape(w) for w in sorted(filter(None, keywords), key=len, reverse=True)), re.IGNORECASE) if any(keywords) else None+ for id, txt in zip(df["id"], df[field_name]):+ ans[id] = pattern.sub(lambda match: f"<em>{match.group(0)}</em>", txt) if pattern else txtreturn ansdef get_aggregation(self, res: tuple[pd.DataFrame, int] | pd.DataFrame, field_name: str):diff --git a/rag/nlp/search.py b/rag/nlp/search.pyindex cd07f6b918..22baac4160 100644--- a/rag/nlp/search.py+++ b/rag/nlp/search.py@@ -231,11 +231,7 @@ class Dealer:total = self.dataStore.get_total(res)logging.debug("Dealer.search TOTAL: {}".format(total))else:- highlightFields = ["content_ltks", "title_tks"]- if not highlight:- highlightFields = []- elif isinstance(highlight, list):- highlightFields = highlight+ highlightFields = []matchText, keywords = self.qryr.question(qst, min_match=(0.3 if min_match else 0))if emb_mdl is None:matchExprs = [matchText] if matchText else []@@ -309,9 +305,9 @@ class Dealer:logging.debug(f"TOTAL: {total}")ids = self.dataStore.get_doc_ids(res)keywords = list(kwds)- highlight = self.dataStore.get_highlight(res, keywords, "content_with_weight")+ highlightDic = self.dataStore.get_highlight(res, keywords, "content_with_weight") if highlight else {}aggs = self.dataStore.get_aggregation(res, "docnm_kwd")- return self.SearchResult(total=total, ids=ids, query_vector=q_vec, aggregation=aggs, highlight=highlight, field=self.dataStore.get_fields(res, src + ["_score"]), keywords=keywords)+ return self.SearchResult(total=total, ids=ids, query_vector=q_vec, aggregation=aggs, highlight=highlightDic, field=self.dataStore.get_fields(res, src + ["_score"]), keywords=keywords)@staticmethoddef trans2floats(txt):@@ -784,11 +780,8 @@ class Dealer:"mom_id": chunk.get("mom_id", ""),"row_id": chunk.get("row_id()"),}- if highlight and sres.highlight:- if id in sres.highlight:- d["highlight"] = remove_redundant_spaces(sres.highlight[id])- else:- d["highlight"] = d["content_with_weight"]+ if id in sres.highlight:+ d["highlight"] = sres.highlight[id]ranks["chunks"].append(d)if aggs:diff --git a/test/unit_test/rag/nlp/test_gaussdb_retrieval.py b/test/unit_test/rag/nlp/test_gaussdb_retrieval.pyindex 3a887d5cc9..057684293b 100644--- a/test/unit_test/rag/nlp/test_gaussdb_retrieval.py+++ b/test/unit_test/rag/nlp/test_gaussdb_retrieval.py@@ -488,6 +488,7 @@ async def test_tc_ret_311_retrieval_keeps_term_only_gaussdb_scores_when_vector_wtotal=2,ids=["term-low", "term-zero"],query_vector=[0.1, 0.2],+ highlight={},field={"term-low": retrieval_chunk(0.1, "doc-low", "Low"),"term-zero": retrieval_chunk(0.0, "doc-zero", "Zero"),