RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(parser): decode non-UTF-8 document text (#19253)
Sbaaoui Idrissebb2174236f0
diff --git a/deepdoc/parser/excel_parser.py b/deepdoc/parser/excel_parser.pyindex eef73fb97e..e329017a91 100644--- a/deepdoc/parser/excel_parser.py+++ b/deepdoc/parser/excel_parser.py@@ -14,12 +14,12 @@import loggingimport reimport sys-from io import BytesIO+from io import BytesIO, StringIOimport pandas as pdfrom openpyxl import Workbook, load_workbook-from rag.nlp import find_codec+from rag.nlp import decode_text, find_codecfrom rag.utils.lazy_image import LazyImage# copied from `/openpyxl/cell/cell.py`@@ -27,6 +27,17 @@ ILLEGAL_CHARACTERS_RE = re.compile(r"[\000-\010]|[\013-\014]|[\016-\037]")class RAGFlowExcelParser:+ @staticmethod+ def _read_csv(file_like_object):+ if isinstance(file_like_object, str):+ with open(file_like_object, "rb") as file:+ binary = file.read()+ else:+ file_like_object.seek(0)+ binary = file_like_object.read()+ text, _ = decode_text(binary, document_type="CSV document")+ return pd.read_csv(StringIO(text), on_bad_lines="skip")+@staticmethoddef _load_excel_to_workbook(file_like_object):if isinstance(file_like_object, bytes):@@ -41,8 +52,7 @@ class RAGFlowExcelParser:logging.info("Not an Excel file, converting CSV to Excel Workbook")try:- file_like_object.seek(0)- df = pd.read_csv(file_like_object, on_bad_lines="skip")+ df = RAGFlowExcelParser._read_csv(file_like_object)return RAGFlowExcelParser._dataframe_to_workbook(df)except Exception as e_csv:@@ -265,8 +275,7 @@ class RAGFlowExcelParser:df = pd.read_excel(file_like_object)except Exception as e:logging.warning(f"Parse spreadsheet error: {e}, trying to interpret as CSV file")- file_like_object.seek(0)- df = pd.read_csv(file_like_object, on_bad_lines="skip")+ df = RAGFlowExcelParser._read_csv(file_like_object)df = df.replace(r"^\s*$", "", regex=True)return df.to_markdown(index=False)diff --git a/deepdoc/parser/html_parser.py b/deepdoc/parser/html_parser.pyindex 1f0c5b3bb4..3e7d7882af 100644--- a/deepdoc/parser/html_parser.py+++ b/deepdoc/parser/html_parser.py@@ -15,7 +15,7 @@# limitations under the License.#-from rag.nlp import find_codec, rag_tokenizer+from rag.nlp import decode_text, rag_tokenizerimport loggingimport reimport uuid@@ -37,8 +37,7 @@ TITLE_TAGS = {"h1": "#", "h2": "##", "h3": "###", "h4": "####", "h5": "#####", "class RAGFlowHtmlParser:def __call__(self, fnm, binary=None, chunk_token_num=512):if binary is not None:- encoding = find_codec(binary)- txt = binary.decode(encoding, errors="ignore")+ txt, _ = decode_text(binary, document_type="HTML document")else:with open(fnm, "r", encoding=get_encoding(fnm)) as f:txt = f.read()diff --git a/deepdoc/parser/json_parser.py b/deepdoc/parser/json_parser.pyindex 36045cbd4c..d82760efbd 100644--- a/deepdoc/parser/json_parser.py+++ b/deepdoc/parser/json_parser.py@@ -21,7 +21,7 @@import jsonfrom typing import Any-from rag.nlp import find_codec+from rag.nlp import decode_textclass RAGFlowJsonParser:@@ -31,8 +31,7 @@ class RAGFlowJsonParser:self.min_chunk_size = min_chunk_size if min_chunk_size is not None else max(max_chunk_size - 200, 50)def __call__(self, binary):- encoding = find_codec(binary)- txt = binary.decode(encoding, errors="ignore")+ txt, _ = decode_text(binary, document_type="JSON document")if self.is_jsonl_format(txt):sections = self._parse_jsonl(txt)diff --git a/deepdoc/parser/utils.py b/deepdoc/parser/utils.pyindex b36af08fa5..f75d53a6cb 100644--- a/deepdoc/parser/utils.py+++ b/deepdoc/parser/utils.py@@ -18,14 +18,13 @@ from io import BytesIOfrom pypdf import PdfReader as pdf2_read-from rag.nlp import find_codec+from rag.nlp import decode_textdef get_text(fnm: str, binary=None) -> str:txt = ""if binary is not None:- encoding = find_codec(binary)- txt = binary.decode(encoding, errors="ignore")+ txt, _ = decode_text(binary, document_type="text document")else:with open(fnm, "r") as f:while True:diff --git a/rag/app/naive.py b/rag/app/naive.pyindex eaf941d3e5..494d677e85 100644--- a/rag/app/naive.py+++ b/rag/app/naive.py@@ -53,7 +53,7 @@ from common.text_utils import normalize_arabic_presentation_formsfrom rag.nlp import (concat_img,DEFAULT_DELIMITER,- find_codec,+ decode_text,naive_merge,naive_merge_with_images,naive_merge_docx,@@ -1002,8 +1002,7 @@ class Markdown(MarkdownParser):def __call__(self, filename, binary=None, separate_tables=True, delimiter=None, return_section_images=False):"""Parse markdown into text sections and optional standalone table chunks."""if binary is not None:- encoding = find_codec(binary)- txt = binary.decode(encoding, errors="ignore")+ txt, _ = decode_text(binary, document_type="Markdown document")else:with open(filename, "r") as f:txt = f.read()diff --git a/rag/nlp/__init__.py b/rag/nlp/__init__.pyindex ab101218d9..bf1eaed5ed 100644--- a/rag/nlp/__init__.py+++ b/rag/nlp/__init__.py@@ -197,6 +197,37 @@ def find_codec(blob):return "utf-8"+def decode_text(blob, document_type="text"):+ """Decode document bytes without silently accepting weak codec guesses."""+ bom_codecs = (+ (b"\x00\x00\xfe\xff", "utf-32-be"),+ (b"\xff\xfe\x00\x00", "utf-32-le"),+ (b"\xfe\xff", "utf-16-be"),+ (b"\xff\xfe", "utf-16-le"),+ (b"\xef\xbb\xbf", "utf-8-sig"),+ )+ for bom, encoding in bom_codecs:+ if blob.startswith(bom):+ return blob.decode(encoding), encoding++ try:+ return blob.decode("utf-8"), "utf-8"+ except UnicodeDecodeError:+ pass++ detected = chardet.detect(blob[:1024])+ encoding = detected.get("encoding")+ confidence = detected.get("confidence") or 0.0+ if encoding and encoding.lower().replace("-", "") in {"gb2312", "gbk"}:+ encoding = "gb18030"+ if not encoding or confidence < 0.8:+ raise UnicodeError(f"Unable to reliably detect {document_type} encoding (confidence={confidence:.2f})")+ try:+ return blob.decode(encoding), encoding+ except (LookupError, UnicodeDecodeError) as exc:+ raise UnicodeError(f"Unable to decode {document_type} as {encoding}") from exc++QUESTION_PATTERN = [r"第([零一二三四五六七八九十百0-9]+)问",r"第([零一二三四五六七八九十百0-9]+)条",diff --git a/test/unit_test/deepdoc/parser/test_excel_parser.py b/test/unit_test/deepdoc/parser/test_excel_parser.pyindex b137779ca4..c91c69cad7 100644--- a/test/unit_test/deepdoc/parser/test_excel_parser.py+++ b/test/unit_test/deepdoc/parser/test_excel_parser.py@@ -24,7 +24,7 @@ import pytest# Import RAGFlowExcelParser directly by file path to avoid triggering# deepdoc/parser/__init__.py and rag.nlp, which pull in heavy dependencies.-for _m in ["pandas", "rag.nlp", "rag.utils", "rag.utils.lazy_image"]:+for _m in ["rag.nlp", "rag.utils", "rag.utils.lazy_image"]:if _m not in sys.modules:sys.modules[_m] = mock.MagicMock()@@ -50,6 +50,20 @@ _spec.loader.exec_module(_mod)RAGFlowExcelParser = _mod.RAGFlowExcelParser+def _test_find_codec(binary):+ for encoding in ("utf-8-sig", "gb18030"):+ try:+ binary.decode(encoding)+ return encoding+ except UnicodeDecodeError:+ continue+ raise UnicodeDecodeError("test", binary, 0, 1, "undecodable")+++_mod.find_codec = _test_find_codec+_mod.decode_text = lambda binary, document_type="text": (binary.decode(_test_find_codec(binary)), _test_find_codec(binary))++def _make_xlsx(n_data_rows):from openpyxl import Workbook@@ -142,6 +156,29 @@ def test_call_skips_truly_empty_cells():assert "note" not in joined, lines[email protected]("encoding", ["utf-8", "utf-8-sig", "gbk", "gb18030"])+def test_csv_encoding_is_detected_without_losing_unicode(encoding):+ binary = "项目名称,备注\n核心系统重构,包含GBK语料导入测试\n".encode(encoding)+ lines = RAGFlowExcelParser()(binary)+ joined = " ".join(text for text, _ in lines)+ assert "项目名称" in joined+ assert "包含GBK语料导入测试" in joined+++def test_csv_accepts_gb18030_characters_not_supported_by_gbk():+ binary = "项目名称,备注\n扩展字符,𠀀\n".encode("gb18030")+ lines = RAGFlowExcelParser()(binary)+ assert "备注:𠀀" in lines[0][0]+++def test_corrupt_csv_encoding_still_fails():+ with pytest.raises(Exception, match="Failed to parse CSV"):+ RAGFlowExcelParser()(b"\xff\xfe\xfa\xfb")++def _make_two_sheet_xlsx():from openpyxl import Workbookdiff --git a/test/unit_test/rag/test_find_codec.py b/test/unit_test/rag/test_find_codec.pyindex bcc5b344a6..89a408d58e 100644--- a/test/unit_test/rag/test_find_codec.py+++ b/test/unit_test/rag/test_find_codec.py@@ -163,3 +163,36 @@ def test_find_codec_utf8_wins_over_confident_wrong_detection(find_codec, monkeyp)blob = "Ελληνικά".encode("utf-8")assert find_codec(blob) == "utf-8"+++def decode_text(find_codec):+ from rag.nlp import decode_text as _decode_text++ return _decode_text+++def test_decode_text_preserves_gbk_and_gb18030(decode_text):+ gbk = "项目名称,核心系统重构".encode("gbk")+ text, encoding = decode_text(gbk, document_type="CSV document")+ assert text == "项目名称,核心系统重构"+ assert encoding == "gb18030"++ gb18030_only = "𠀀".encode("gb18030")+ text, encoding = decode_text(gb18030_only, document_type="CSV document")+ assert text == "𠀀"+ assert encoding == "gb18030"+++def test_decode_text_rejects_undetectable_bytes_without_loss(decode_text):+ with pytest.raises(UnicodeError, match="CSV document"):+ decode_text(b"prefix\xc3suffix", document_type="CSV document")+++def test_decode_text_accepts_literal_replacement_character(decode_text):+ text, encoding = decode_text("literal � content".encode("utf-8"))+ assert text == "literal � content"+ assert encoding == "utf-8"diff --git a/test/unit_test/rag/test_naive_dispatch.py b/test/unit_test/rag/test_naive_dispatch.pyindex 8ea2b43302..0f5f30c71f 100644--- a/test/unit_test/rag/test_naive_dispatch.py+++ b/test/unit_test/rag/test_naive_dispatch.py@@ -218,6 +218,7 @@ def naive_module():DEFAULT_DELIMITER="\n!?;。;!?",num_tokens_from_string=lambda s: len((s or "").split()),find_codec=lambda b: "utf-8",+ decode_text=lambda b, document_type="text": (b.decode("utf-8"), "utf-8"),rag_tokenizer=types.SimpleNamespace(tokenize=lambda s: ((s or "").split(), [])),concat_img=lambda *a, **k: None,naive_merge=lambda *a, **k: [],