Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

7.0 KB
Raw Blame
"""
Tests for reading text uploads in the encoding they were written in (#6532).
"""

import pytest

from local_deep_research.document_loaders.loader_registry import (
    get_loader_for_path,
)
from local_deep_research.document_loaders.text_encoding import (
    detect_text_encoding,
)

GREETING = "Sehr geehrte Frau Müller,\nviele Grüße aus Köln.\n"
ROWS = "Name,Stadt\nMüller,Köln\nGroß,Düsseldorf\n"
# The bytes pinned by test_binary_bytes_under_a_txt_name_are_refused.
BINARY = b"\x00\x01\x02\xff\xfe binary \x80"


class TestDetectTextEncoding:
    def test_utf8_is_read_as_utf8_with_or_without_a_byte_order_mark(self):
        assert detect_text_encoding(GREETING.encode("utf-8")) == "utf-8-sig"
        assert detect_text_encoding(GREETING.encode("utf-8-sig")) == "utf-8-sig"

    def test_utf8_is_read_as_before_even_with_control_characters(self):
        assert detect_text_encoding(b"\x1b[31mred\x1b[0m \x00") == "utf-8-sig"

    def test_utf16_and_utf32_are_recognized_by_their_byte_order_mark(self):
        assert detect_text_encoding(GREETING.encode("utf-16")) == "utf-16"
        assert detect_text_encoding(GREETING.encode("utf-16-be")) is None
        assert detect_text_encoding(GREETING.encode("utf-32")) == "utf-32"

    def test_big_endian_byte_order_marks_are_recognized(self):
        # Notepad's "Unicode big endian" writes the UTF-16 one.
        text = "Grüße"
        assert (
            detect_text_encoding(b"\xfe\xff" + text.encode("utf-16-be"))
            == "utf-16"
        )
        assert (
            detect_text_encoding(b"\x00\x00\xfe\xff" + text.encode("utf-32-be"))
            == "utf-32"
        )

    def test_a_mark_that_does_not_decode_falls_back_to_windows_1252(self):
        # Three bytes after a UTF-16 mark cannot be UTF-16, so the mark is
        # read as the two Windows-1252 characters it also is: "ÿþabc".
        assert detect_text_encoding(b"\xff\xfeabc") == "cp1252"

    def test_windows_1252_text_is_read_as_windows_1252(self):
        assert detect_text_encoding(GREETING.encode("cp1252")) == "cp1252"
        assert detect_text_encoding("Preis: 5 €\r\n\x1a".encode("cp1252")) == (
            "cp1252"
        )
        assert detect_text_encoding("\x1b[1mGrüße\x1b[0m".encode("cp1252")) == (
            "cp1252"
        )

    def test_bytes_that_are_not_text_have_no_encoding(self):
        assert detect_text_encoding(BINARY) is None
        # 0x81 is one of the five bytes Windows-1252 leaves undefined.
        assert detect_text_encoding(b"hello \x81 world") is None
        assert (
            detect_text_encoding(b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR") is None
        )


def _load(path):
    loader = get_loader_for_path(path)
    assert loader is not None
    return loader.load()


class TestTextUploads:
    def test_a_windows_1252_text_file_is_read(self, tmp_path):
        path = tmp_path / "letter.txt"
        path.write_bytes(GREETING.encode("cp1252"))

        assert [d.page_content for d in _load(path)] == [GREETING]

    def test_a_utf16_text_file_is_read(self, tmp_path):
        path = tmp_path / "notes.txt"
        path.write_bytes(GREETING.encode("utf-16"))

        assert [d.page_content for d in _load(path)] == [GREETING]

    def test_a_utf8_byte_order_mark_is_not_part_of_the_text(self, tmp_path):
        path = tmp_path / "notes.txt"
        path.write_bytes(GREETING.encode("utf-8-sig"))

        assert [d.page_content for d in _load(path)] == [GREETING]

    def test_binary_bytes_under_a_txt_name_are_refused(self, tmp_path):
        path = tmp_path / "bin.txt"
        path.write_bytes(BINARY)

        with pytest.raises(ValueError, match="no recognized text encoding"):
            _load(path)

    @pytest.mark.parametrize(
        "encoding,detected",
        [("utf-8", "utf-8-sig"), ("cp1252", "cp1252"), ("utf-16", "utf-16")],
    )
    def test_the_encoding_read_is_recorded_on_the_document(
        self, tmp_path, encoding, detected
    ):
        path = tmp_path / "notes.txt"
        path.write_bytes(GREETING.encode(encoding))

        assert [d.metadata["detected_encoding"] for d in _load(path)] == [
            detected
        ]


class TestCsvUploads:
    @pytest.mark.parametrize(
        "encoding", ["utf-8", "utf-8-sig", "cp1252", "utf-16"]
    )
    def test_rows_are_read_in_the_encoding_of_the_file(
        self, tmp_path, encoding
    ):
        path = tmp_path / "people.csv"
        path.write_bytes(ROWS.encode(encoding))

        assert [d.page_content for d in _load(path)] == [
            "Name: Müller\nStadt: Köln",
            "Name: Groß\nStadt: Düsseldorf",
        ]

    @pytest.mark.parametrize(
        "encoding,detected",
        [
            ("utf-8", "utf-8-sig"),
            ("utf-8-sig", "utf-8-sig"),
            ("cp1252", "cp1252"),
            ("utf-16", "utf-16"),
        ],
    )
    def test_every_row_records_the_encoding_read(
        self, tmp_path, encoding, detected
    ):
        path = tmp_path / "people.csv"
        path.write_bytes(ROWS.encode(encoding))

        assert [d.metadata["detected_encoding"] for d in _load(path)] == [
            detected,
            detected,
        ]

    def test_a_windows_1252_tsv_file_is_read(self, tmp_path):
        path = tmp_path / "people.tsv"
        path.write_bytes(ROWS.replace(",", "\t").encode("cp1252"))

        assert [d.page_content for d in _load(path)][0] == (
            "Name: Müller\nStadt: Köln"
        )


class TestStructuredUploads:
    """JSON and YAML uploads are decoded the way text uploads are."""

    @pytest.mark.parametrize("suffix", [".json", ".yaml"])
    @pytest.mark.parametrize("encoding", ["utf-8-sig", "utf-16", "cp1252"])
    def test_a_json_or_yaml_file_is_read_in_its_encoding(
        self, tmp_path, suffix, encoding
    ):
        # Notepad saves UTF-8 with a byte-order mark or as Windows-1252, and
        # PowerShell's Out-File writes UTF-16
        path = tmp_path / f"data{suffix}"
        path.write_bytes('{"city": "Köln", "note": "Groß"}'.encode(encoding))

        [document] = _load(path)

        assert "Köln" in document.page_content
        assert "Groß" in document.page_content
        assert not document.metadata.get("parse_error")

    @pytest.mark.parametrize("suffix", [".json", ".yaml"])
    @pytest.mark.parametrize(
        "encoding,detected",
        [("utf-8", "utf-8-sig"), ("cp1252", "cp1252"), ("utf-16", "utf-16")],
    )
    def test_the_encoding_read_is_recorded_on_the_document(
        self, tmp_path, suffix, encoding, detected
    ):
        path = tmp_path / f"data{suffix}"
        path.write_bytes('{"city": "Köln"}'.encode(encoding))

        assert [d.metadata["detected_encoding"] for d in _load(path)] == [
            detected
        ]

    @pytest.mark.parametrize("suffix", [".json", ".yaml"])
    def test_a_file_that_does_not_parse_records_the_encoding_too(
        self, tmp_path, suffix
    ):
        path = tmp_path / f"broken{suffix}"
        path.write_bytes('{"city": "Köln", "note": [}'.encode("cp1252"))

        [document] = _load(path)

        assert document.metadata["parse_error"] is True
        assert document.metadata["detected_encoding"] == "cp1252"