RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
test(rag/nlp): cover the BOM and LookupError branches of decode_text (#19292)
Harsh Kashyap88641273f069
diff --git a/test/unit_test/rag/test_find_codec.py b/test/unit_test/rag/test_find_codec.pyindex 89a408d58e..b4ba0c3416 100644--- a/test/unit_test/rag/test_find_codec.py+++ b/test/unit_test/rag/test_find_codec.py@@ -196,3 +196,60 @@ def test_decode_text_accepts_literal_replacement_character(decode_text):text, encoding = decode_text("literal � content".encode("utf-8"))assert text == "literal � content"assert encoding == "utf-8"+++ "bom, encoding, text",+ [+ # Each blob is the BOM bytes followed by the payload encoded in+ # the matching codec (without its own BOM). `decode_text` matches+ # the leading BOM against the table at the top of the function+ # and returns before the chardet fall-through runs. The+ # codec-name suffix (-be / -le / -sig) is the exact string the+ # function passes to `bytes.decode`, so the returned encoding+ # string is pinned too.+ (b"\x00\x00\xfe\xff", "utf-32-be", "hello, world"),+ (b"\xff\xfe\x00\x00", "utf-32-le", "hello, world"),+ (b"\xfe\xff", "utf-16-be", "hello, world"),+ (b"\xff\xfe", "utf-16-le", "hello, world"),+ (b"\xef\xbb\xbf", "utf-8-sig", "hello, world"),+ ],+)+def test_decode_text_matches_each_bom_branch(decode_text, bom, encoding, text):+ # The function uses the BE/LE-suffixed codecs (e.g. "utf-32-be"),+ # which preserve the BOM character on decode; only the BOM-stripping+ # "utf-8-sig" branch returns the payload without the leading+ # U+FEFF. The expected output reflects this per-encoding behavior.+ # The payload is encoded in the BE/LE/utf-8 base codec (without its+ # own BOM); the BOM is prepended by the test so the blob matches+ # exactly what an upload of a BOM-prefixed file looks like on disk.+ payload_encoding = "utf-8" if encoding == "utf-8-sig" else encoding+ payload_bytes = text.encode(payload_encoding)+ blob = bom + payload_bytes+ result, returned_encoding = decode_text(blob)+ if encoding == "utf-8-sig":+ assert result == text+ else:+ assert result == "\ufeff" + text+ assert returned_encoding == encoding+++def test_decode_text_reraises_lookup_error_as_unicode_error(decode_text, monkeypatch):+ # Regression: chardet can return a codec name Python does not recognise+ # (e.g. "not-a-real-codec"). The function catches the resulting+ # `LookupError` (and the parallel `UnicodeDecodeError`) and re-raises+ # a single `UnicodeError` with the document_type in the message, so+ # callers can present a uniform error to the user. Without the+ # catch, the raw `LookupError` would surface as a less actionable+ # `LookupError: unknown encoding` from a parser.+ nlp = sys.modules["rag.nlp"]+ monkeypatch.setattr(+ nlp.chardet,+ "detect",+ lambda blob: {"encoding": "not-a-real-codec", "confidence": 0.99},+ )+ blob = "Ελληνικά".encode("cp1253")+ with pytest.raises(UnicodeError, match=r"Unable to decode text as not-a-real-codec"):+ decode_text(blob)