Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

Fix: find_codec discards confident encoding detection and returns a garbage codec (#15793)

Yash Raj Pandey16642140fef9
diff --git a/rag/nlp/__init__.py b/rag/nlp/__init__.py
index f257bbecec..dfa41f6e57 100644
--- a/rag/nlp/__init__.py
+++ b/rag/nlp/__init__.py
@@ -151,14 +151,39 @@ all_codecs = [
def find_codec(blob):
- detected = chardet.detect(blob[:1024])
- if detected["confidence"] > 0.5:
- if detected["encoding"] == "ascii":
+ sample = blob[:1024]
+
+ # A blob that decodes as UTF-8 is UTF-8; nothing else needs to be guessed.
+ # Check this first because chardet can report a confident single-byte guess
+ # for short UTF-8 text, and callers decode with errors="ignore", so a wrong
+ # codec is silently lossy instead of raising. The second decode covers a
+ # multi-byte character that the 1024-byte sample cuts in half.
+ try:
+ sample.decode("utf-8")
+ return "utf-8"
+ except UnicodeDecodeError:
+ try:
+ blob.decode("utf-8")
return "utf-8"
+ except UnicodeDecodeError:
+ pass
+
+ detected = chardet.detect(sample)
+ encoding = detected["encoding"]
+ if encoding:
+ # Honor the detection whenever it decodes the sample. The loop below
+ # returns the first codec that does not raise, and legacy single-byte
+ # codecs (cp037, utf_16) decode arbitrary bytes without error, so a
+ # low-confidence detection still beats the loop's first non-raising hit.
+ try:
+ sample.decode(encoding)
+ return encoding
+ except (UnicodeDecodeError, LookupError) as e:
+ logging.debug("find_codec: detection %r (%.2f) did not decode the sample: %s", encoding, detected["confidence"] or 0.0, e)
for c in all_codecs:
try:
- blob[:1024].decode(c)
+ sample.decode(c)
return c
except Exception:
pass
diff --git a/test/unit_test/rag/test_find_codec.py b/test/unit_test/rag/test_find_codec.py
new file mode 100644
index 0000000000..bcc5b344a6
--- /dev/null
+++ b/test/unit_test/rag/test_find_codec.py
@@ -0,0 +1,165 @@
+#
+# Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import codecs
+import sys
+import types
+
+import pytest
+
+
+def find_codec(monkeypatch):
+ """Import ``rag.nlp.find_codec`` with heavy module-level deps stubbed.
+
+ The stubs are installed through ``monkeypatch`` so they are restored after
+ each test instead of leaking fakes into ``sys.modules`` for unrelated tests.
+ ``chardet`` is intentionally left real, since ``find_codec`` relies on real
+ encoding detection.
+ """
+
+ def _stub(name, **attrs):
+ module = types.ModuleType(name)
+ for key, value in attrs.items():
+ setattr(module, key, value)
+ monkeypatch.setitem(sys.modules, name, module)
+ return module
+
+ _stub("common.token_utils", num_tokens_from_string=lambda *a, **k: 0)
+ _stub("roman_numbers")
+ _stub("word2number", w2n=types.SimpleNamespace())
+ _stub("cn2an", cn2an=lambda *a, **k: 0)
+ pil = _stub("PIL")
+ pil.Image = _stub("PIL.Image")
+
+ # Another test module stubs chardet at import time; drop any such stub so the
+ # fresh rag.nlp import below binds the real chardet (find_codec needs it).
+ monkeypatch.delitem(sys.modules, "chardet", raising=False)
+
+ # Force a fresh import so the stubs above are picked up; monkeypatch restores
+ # the module table (including rag.nlp) afterwards.
+ monkeypatch.delitem(sys.modules, "rag.nlp", raising=False)
+ from rag.nlp import find_codec as _find_codec
+
+ return _find_codec
+
+
+ "text, encoding",
+ [
+ ("こんにちは世界。これは日本語のサンプルテキストです。" * 14, "shift_jis"),
+ ("안녕하세요 세계. 이것은 한국어 샘플 텍스트입니다. " * 12, "euc_kr"),
+ ],
+)
+def test_find_codec_honors_confident_detection(find_codec, text, encoding):
+ # Regression: find_codec computed a confident chardet detection but only
+ # used it for the ascii special case; for any other encoding it fell through
+ # to the brute-force all_codecs loop, which returns the first codec that
+ # does not raise. Legacy single-byte codecs like cp037 (and gbk/gb2312 for
+ # these bytes) decode arbitrary input without error, so a non-UTF-8 upload
+ # came back mis-decoded as garbage. The returned codec must reproduce the
+ # original text.
+ blob = text.encode(encoding)
+ codec = find_codec(blob)
+ assert blob.decode(codec) == text
+
+
+def test_find_codec_ascii_maps_to_utf8(find_codec):
+ assert find_codec(b"plain ascii content, nothing special here") == "utf-8"
+
+
+def test_find_codec_utf8_roundtrips(find_codec):
+ text = "Lorem ipsum UTF-8 sample 日本語 and Ελληνικά mixed together."
+ blob = text.encode("utf-8")
+ assert blob.decode(find_codec(blob)) == text
+
+
+def test_find_codec_utf8_emoji_roundtrips(find_codec):
+ # Regression: the repo-pinned chardet 5.2.0 reports Windows-1254 at 0.56
+ # confidence, so trusting it before checking UTF-8 mis-decodes this blob.
+ text = "Sales are up 20% this quarter \U0001f4c8\U0001f680"
+ blob = text.encode("utf-8")
+ assert blob.decode(find_codec(blob)) == text
+
+
+def test_find_codec_rejects_utf8_when_only_the_sample_looks_valid(find_codec):
+ # An incremental decoder buffers a trailing partial sequence instead of
+ # rejecting it, so a sample-only check calls this blob UTF-8. The full blob
+ # is not valid UTF-8, and callers decode with errors="ignore", which drops
+ # bytes. The returned codec must decode the whole blob.
+ blob = b"a" * 1023 + b"\xe9" + b" "
+ codec = find_codec(blob)
+ assert codec != "utf-8"
+ assert isinstance(blob.decode(codec), str)
+
+
+ "text, encoding",
+ [
+ ("Ежедневный отчет о продажах за прошлую неделю показывает рост.", "cp1251"),
+ ("Η ετήσια έκθεση πωλήσεων δείχνει σημαντική αύξηση στα έσοδα.", "cp1253"),
+ ],
+)
+def test_find_codec_uses_low_confidence_detection(find_codec, monkeypatch, text, encoding):
+ # The old code gated on confidence > 0.5, so a correct detection below the
+ # gate was discarded and the brute-force loop returned utf_16 instead,
+ # which mis-decodes. Both samples are verified not to decode as UTF-8, so
+ # the UTF-8 check does not short-circuit.
+ nlp = sys.modules["rag.nlp"]
+ monkeypatch.setattr(
+ nlp.chardet,
+ "detect",
+ lambda blob: {"encoding": encoding, "confidence": 0.3},
+ )
+ blob = text.encode(encoding)
+ with pytest.raises(UnicodeDecodeError):
+ blob.decode("utf-8")
+ assert find_codec(blob) == encoding
+ assert blob.decode(find_codec(blob)) == text
+
+
+def test_find_codec_invalid_detected_codec_falls_back(find_codec, monkeypatch):
+ # This covers the LookupError fallback; it need not fail against old code.
+ nlp = sys.modules["rag.nlp"]
+ monkeypatch.setattr(
+ nlp.chardet,
+ "detect",
+ lambda blob: {"encoding": "not-a-real-codec", "confidence": 0.99},
+ )
+ blob = "Ελληνικά".encode("cp1253")
+ codec = find_codec(blob)
+ codecs.lookup(codec)
+ assert isinstance(blob.decode(codec), str)
+
+
+def test_find_codec_utf8_wins_over_confident_wrong_detection(find_codec, monkeypatch):
+ # These UTF-8 bytes are valid cp1254, so old code returns the wrong guess.
+ nlp = sys.modules["rag.nlp"]
+ monkeypatch.setattr(
+ nlp.chardet,
+ "detect",
+ lambda blob: {"encoding": "cp1254", "confidence": 0.99},
+ )
+ blob = "Ελληνικά".encode("utf-8")
+ assert find_codec(blob) == "utf-8"