Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(parser): decode non-UTF-8 document text (#19253)

Sbaaoui Idrissebb2174236f0
diff --git a/deepdoc/parser/excel_parser.py b/deepdoc/parser/excel_parser.py
index eef73fb97e..e329017a91 100644
--- a/deepdoc/parser/excel_parser.py
+++ b/deepdoc/parser/excel_parser.py
@@ -14,12 +14,12 @@
import logging
import re
import sys
-from io import BytesIO
+from io import BytesIO, StringIO
import pandas as pd
from openpyxl import Workbook, load_workbook
-from rag.nlp import find_codec
+from rag.nlp import decode_text, find_codec
from rag.utils.lazy_image import LazyImage
# copied from `/openpyxl/cell/cell.py`
@@ -27,6 +27,17 @@ ILLEGAL_CHARACTERS_RE = re.compile(r"[\000-\010]|[\013-\014]|[\016-\037]")
class RAGFlowExcelParser:
+ @staticmethod
+ def _read_csv(file_like_object):
+ if isinstance(file_like_object, str):
+ with open(file_like_object, "rb") as file:
+ binary = file.read()
+ else:
+ file_like_object.seek(0)
+ binary = file_like_object.read()
+ text, _ = decode_text(binary, document_type="CSV document")
+ return pd.read_csv(StringIO(text), on_bad_lines="skip")
+
@staticmethod
def _load_excel_to_workbook(file_like_object):
if isinstance(file_like_object, bytes):
@@ -41,8 +52,7 @@ class RAGFlowExcelParser:
logging.info("Not an Excel file, converting CSV to Excel Workbook")
try:
- file_like_object.seek(0)
- df = pd.read_csv(file_like_object, on_bad_lines="skip")
+ df = RAGFlowExcelParser._read_csv(file_like_object)
return RAGFlowExcelParser._dataframe_to_workbook(df)
except Exception as e_csv:
@@ -265,8 +275,7 @@ class RAGFlowExcelParser:
df = pd.read_excel(file_like_object)
except Exception as e:
logging.warning(f"Parse spreadsheet error: {e}, trying to interpret as CSV file")
- file_like_object.seek(0)
- df = pd.read_csv(file_like_object, on_bad_lines="skip")
+ df = RAGFlowExcelParser._read_csv(file_like_object)
df = df.replace(r"^\s*$", "", regex=True)
return df.to_markdown(index=False)
diff --git a/deepdoc/parser/html_parser.py b/deepdoc/parser/html_parser.py
index 1f0c5b3bb4..3e7d7882af 100644
--- a/deepdoc/parser/html_parser.py
+++ b/deepdoc/parser/html_parser.py
@@ -15,7 +15,7 @@
# limitations under the License.
#
-from rag.nlp import find_codec, rag_tokenizer
+from rag.nlp import decode_text, rag_tokenizer
import logging
import re
import uuid
@@ -37,8 +37,7 @@ TITLE_TAGS = {"h1": "#", "h2": "##", "h3": "###", "h4": "####", "h5": "#####", "
class RAGFlowHtmlParser:
def __call__(self, fnm, binary=None, chunk_token_num=512):
if binary is not None:
- encoding = find_codec(binary)
- txt = binary.decode(encoding, errors="ignore")
+ txt, _ = decode_text(binary, document_type="HTML document")
else:
with open(fnm, "r", encoding=get_encoding(fnm)) as f:
txt = f.read()
diff --git a/deepdoc/parser/json_parser.py b/deepdoc/parser/json_parser.py
index 36045cbd4c..d82760efbd 100644
--- a/deepdoc/parser/json_parser.py
+++ b/deepdoc/parser/json_parser.py
@@ -21,7 +21,7 @@
import json
from typing import Any
-from rag.nlp import find_codec
+from rag.nlp import decode_text
class RAGFlowJsonParser:
@@ -31,8 +31,7 @@ class RAGFlowJsonParser:
self.min_chunk_size = min_chunk_size if min_chunk_size is not None else max(max_chunk_size - 200, 50)
def __call__(self, binary):
- encoding = find_codec(binary)
- txt = binary.decode(encoding, errors="ignore")
+ txt, _ = decode_text(binary, document_type="JSON document")
if self.is_jsonl_format(txt):
sections = self._parse_jsonl(txt)
diff --git a/deepdoc/parser/utils.py b/deepdoc/parser/utils.py
index b36af08fa5..f75d53a6cb 100644
--- a/deepdoc/parser/utils.py
+++ b/deepdoc/parser/utils.py
@@ -18,14 +18,13 @@ from io import BytesIO
from pypdf import PdfReader as pdf2_read
-from rag.nlp import find_codec
+from rag.nlp import decode_text
def get_text(fnm: str, binary=None) -> str:
txt = ""
if binary is not None:
- encoding = find_codec(binary)
- txt = binary.decode(encoding, errors="ignore")
+ txt, _ = decode_text(binary, document_type="text document")
else:
with open(fnm, "r") as f:
while True:
diff --git a/rag/app/naive.py b/rag/app/naive.py
index eaf941d3e5..494d677e85 100644
--- a/rag/app/naive.py
+++ b/rag/app/naive.py
@@ -53,7 +53,7 @@ from common.text_utils import normalize_arabic_presentation_forms
from rag.nlp import (
concat_img,
DEFAULT_DELIMITER,
- find_codec,
+ decode_text,
naive_merge,
naive_merge_with_images,
naive_merge_docx,
@@ -1002,8 +1002,7 @@ class Markdown(MarkdownParser):
def __call__(self, filename, binary=None, separate_tables=True, delimiter=None, return_section_images=False):
"""Parse markdown into text sections and optional standalone table chunks."""
if binary is not None:
- encoding = find_codec(binary)
- txt = binary.decode(encoding, errors="ignore")
+ txt, _ = decode_text(binary, document_type="Markdown document")
else:
with open(filename, "r") as f:
txt = f.read()
diff --git a/rag/nlp/__init__.py b/rag/nlp/__init__.py
index ab101218d9..bf1eaed5ed 100644
--- a/rag/nlp/__init__.py
+++ b/rag/nlp/__init__.py
@@ -197,6 +197,37 @@ def find_codec(blob):
return "utf-8"
+def decode_text(blob, document_type="text"):
+ """Decode document bytes without silently accepting weak codec guesses."""
+ bom_codecs = (
+ (b"\x00\x00\xfe\xff", "utf-32-be"),
+ (b"\xff\xfe\x00\x00", "utf-32-le"),
+ (b"\xfe\xff", "utf-16-be"),
+ (b"\xff\xfe", "utf-16-le"),
+ (b"\xef\xbb\xbf", "utf-8-sig"),
+ )
+ for bom, encoding in bom_codecs:
+ if blob.startswith(bom):
+ return blob.decode(encoding), encoding
+
+ try:
+ return blob.decode("utf-8"), "utf-8"
+ except UnicodeDecodeError:
+ pass
+
+ detected = chardet.detect(blob[:1024])
+ encoding = detected.get("encoding")
+ confidence = detected.get("confidence") or 0.0
+ if encoding and encoding.lower().replace("-", "") in {"gb2312", "gbk"}:
+ encoding = "gb18030"
+ if not encoding or confidence < 0.8:
+ raise UnicodeError(f"Unable to reliably detect {document_type} encoding (confidence={confidence:.2f})")
+ try:
+ return blob.decode(encoding), encoding
+ except (LookupError, UnicodeDecodeError) as exc:
+ raise UnicodeError(f"Unable to decode {document_type} as {encoding}") from exc
+
+
QUESTION_PATTERN = [
r"第([零一二三四五六七八九十百0-9]+)问",
r"第([零一二三四五六七八九十百0-9]+)条",
diff --git a/test/unit_test/deepdoc/parser/test_excel_parser.py b/test/unit_test/deepdoc/parser/test_excel_parser.py
index b137779ca4..c91c69cad7 100644
--- a/test/unit_test/deepdoc/parser/test_excel_parser.py
+++ b/test/unit_test/deepdoc/parser/test_excel_parser.py
@@ -24,7 +24,7 @@ import pytest
# Import RAGFlowExcelParser directly by file path to avoid triggering
# deepdoc/parser/__init__.py and rag.nlp, which pull in heavy dependencies.
-for _m in ["pandas", "rag.nlp", "rag.utils", "rag.utils.lazy_image"]:
+for _m in ["rag.nlp", "rag.utils", "rag.utils.lazy_image"]:
if _m not in sys.modules:
sys.modules[_m] = mock.MagicMock()
@@ -50,6 +50,20 @@ _spec.loader.exec_module(_mod)
RAGFlowExcelParser = _mod.RAGFlowExcelParser
+def _test_find_codec(binary):
+ for encoding in ("utf-8-sig", "gb18030"):
+ try:
+ binary.decode(encoding)
+ return encoding
+ except UnicodeDecodeError:
+ continue
+ raise UnicodeDecodeError("test", binary, 0, 1, "undecodable")
+
+
+_mod.find_codec = _test_find_codec
+_mod.decode_text = lambda binary, document_type="text": (binary.decode(_test_find_codec(binary)), _test_find_codec(binary))
+
+
def _make_xlsx(n_data_rows):
from openpyxl import Workbook
@@ -142,6 +156,29 @@ def test_call_skips_truly_empty_cells():
assert "note" not in joined, lines
[email protected]("encoding", ["utf-8", "utf-8-sig", "gbk", "gb18030"])
+def test_csv_encoding_is_detected_without_losing_unicode(encoding):
+ binary = "项目名称,备注\n核心系统重构,包含GBK语料导入测试\n".encode(encoding)
+ lines = RAGFlowExcelParser()(binary)
+ joined = " ".join(text for text, _ in lines)
+ assert "项目名称" in joined
+ assert "包含GBK语料导入测试" in joined
+
+
+def test_csv_accepts_gb18030_characters_not_supported_by_gbk():
+ binary = "项目名称,备注\n扩展字符,𠀀\n".encode("gb18030")
+ lines = RAGFlowExcelParser()(binary)
+ assert "备注:𠀀" in lines[0][0]
+
+
+def test_corrupt_csv_encoding_still_fails():
+ with pytest.raises(Exception, match="Failed to parse CSV"):
+ RAGFlowExcelParser()(b"\xff\xfe\xfa\xfb")
+
+
def _make_two_sheet_xlsx():
from openpyxl import Workbook
diff --git a/test/unit_test/rag/test_find_codec.py b/test/unit_test/rag/test_find_codec.py
index bcc5b344a6..89a408d58e 100644
--- a/test/unit_test/rag/test_find_codec.py
+++ b/test/unit_test/rag/test_find_codec.py
@@ -163,3 +163,36 @@ def test_find_codec_utf8_wins_over_confident_wrong_detection(find_codec, monkeyp
)
blob = "Ελληνικά".encode("utf-8")
assert find_codec(blob) == "utf-8"
+
+
+def decode_text(find_codec):
+ from rag.nlp import decode_text as _decode_text
+
+ return _decode_text
+
+
+def test_decode_text_preserves_gbk_and_gb18030(decode_text):
+ gbk = "项目名称,核心系统重构".encode("gbk")
+ text, encoding = decode_text(gbk, document_type="CSV document")
+ assert text == "项目名称,核心系统重构"
+ assert encoding == "gb18030"
+
+ gb18030_only = "𠀀".encode("gb18030")
+ text, encoding = decode_text(gb18030_only, document_type="CSV document")
+ assert text == "𠀀"
+ assert encoding == "gb18030"
+
+
+def test_decode_text_rejects_undetectable_bytes_without_loss(decode_text):
+ with pytest.raises(UnicodeError, match="CSV document"):
+ decode_text(b"prefix\xc3suffix", document_type="CSV document")
+
+
+def test_decode_text_accepts_literal_replacement_character(decode_text):
+ text, encoding = decode_text("literal � content".encode("utf-8"))
+ assert text == "literal � content"
+ assert encoding == "utf-8"
diff --git a/test/unit_test/rag/test_naive_dispatch.py b/test/unit_test/rag/test_naive_dispatch.py
index 8ea2b43302..0f5f30c71f 100644
--- a/test/unit_test/rag/test_naive_dispatch.py
+++ b/test/unit_test/rag/test_naive_dispatch.py
@@ -218,6 +218,7 @@ def naive_module():
DEFAULT_DELIMITER="\n!?;。;!?",
num_tokens_from_string=lambda s: len((s or "").split()),
find_codec=lambda b: "utf-8",
+ decode_text=lambda b, document_type="text": (b.decode("utf-8"), "utf-8"),
rag_tokenizer=types.SimpleNamespace(tokenize=lambda s: ((s or "").split(), [])),
concat_img=lambda *a, **k: None,
naive_merge=lambda *a, **k: [],