~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""
Tests for reading text uploads in the encoding they were written in (#6532).
"""
import pytest
from local_deep_research.document_loaders.loader_registry import (
get_loader_for_path,
)
from local_deep_research.document_loaders.text_encoding import (
detect_text_encoding,
)
GREETING = "Sehr geehrte Frau Müller,\nviele Grüße aus Köln.\n"
ROWS = "Name,Stadt\nMüller,Köln\nGroß,Düsseldorf\n"
# The bytes pinned by test_binary_bytes_under_a_txt_name_are_refused.
BINARY = b"\x00\x01\x02\xff\xfe binary \x80"
class TestDetectTextEncoding:
def test_utf8_is_read_as_utf8_with_or_without_a_byte_order_mark(self):
assert detect_text_encoding(GREETING.encode("utf-8")) == "utf-8-sig"
assert detect_text_encoding(GREETING.encode("utf-8-sig")) == "utf-8-sig"
def test_utf8_is_read_as_before_even_with_control_characters(self):
assert detect_text_encoding(b"\x1b[31mred\x1b[0m \x00") == "utf-8-sig"
def test_utf16_and_utf32_are_recognized_by_their_byte_order_mark(self):
assert detect_text_encoding(GREETING.encode("utf-16")) == "utf-16"
assert detect_text_encoding(GREETING.encode("utf-16-be")) is None
assert detect_text_encoding(GREETING.encode("utf-32")) == "utf-32"
def test_big_endian_byte_order_marks_are_recognized(self):
# Notepad's "Unicode big endian" writes the UTF-16 one.
text = "Grüße"
assert (
detect_text_encoding(b"\xfe\xff" + text.encode("utf-16-be"))
== "utf-16"
)
assert (
detect_text_encoding(b"\x00\x00\xfe\xff" + text.encode("utf-32-be"))
== "utf-32"
)
def test_a_mark_that_does_not_decode_falls_back_to_windows_1252(self):
# Three bytes after a UTF-16 mark cannot be UTF-16, so the mark is
# read as the two Windows-1252 characters it also is: "ÿþabc".
assert detect_text_encoding(b"\xff\xfeabc") == "cp1252"
def test_windows_1252_text_is_read_as_windows_1252(self):
assert detect_text_encoding(GREETING.encode("cp1252")) == "cp1252"
assert detect_text_encoding("Preis: 5 €\r\n\x1a".encode("cp1252")) == (
"cp1252"
)
assert detect_text_encoding("\x1b[1mGrüße\x1b[0m".encode("cp1252")) == (
"cp1252"
)
def test_bytes_that_are_not_text_have_no_encoding(self):
assert detect_text_encoding(BINARY) is None
# 0x81 is one of the five bytes Windows-1252 leaves undefined.
assert detect_text_encoding(b"hello \x81 world") is None
assert (
detect_text_encoding(b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR") is None
)
def _load(path):
loader = get_loader_for_path(path)
assert loader is not None
return loader.load()
class TestTextUploads:
def test_a_windows_1252_text_file_is_read(self, tmp_path):
path = tmp_path / "letter.txt"
path.write_bytes(GREETING.encode("cp1252"))
assert [d.page_content for d in _load(path)] == [GREETING]
def test_a_utf16_text_file_is_read(self, tmp_path):
path = tmp_path / "notes.txt"
path.write_bytes(GREETING.encode("utf-16"))
assert [d.page_content for d in _load(path)] == [GREETING]
def test_a_utf8_byte_order_mark_is_not_part_of_the_text(self, tmp_path):
path = tmp_path / "notes.txt"
path.write_bytes(GREETING.encode("utf-8-sig"))
assert [d.page_content for d in _load(path)] == [GREETING]
def test_binary_bytes_under_a_txt_name_are_refused(self, tmp_path):
path = tmp_path / "bin.txt"
path.write_bytes(BINARY)
with pytest.raises(ValueError, match="no recognized text encoding"):
_load(path)
@pytest.mark.parametrize(
"encoding,detected",
[("utf-8", "utf-8-sig"), ("cp1252", "cp1252"), ("utf-16", "utf-16")],
)
def test_the_encoding_read_is_recorded_on_the_document(
self, tmp_path, encoding, detected
):
path = tmp_path / "notes.txt"
path.write_bytes(GREETING.encode(encoding))
assert [d.metadata["detected_encoding"] for d in _load(path)] == [
detected
]
class TestCsvUploads:
@pytest.mark.parametrize(
"encoding", ["utf-8", "utf-8-sig", "cp1252", "utf-16"]
)
def test_rows_are_read_in_the_encoding_of_the_file(
self, tmp_path, encoding
):
path = tmp_path / "people.csv"
path.write_bytes(ROWS.encode(encoding))
assert [d.page_content for d in _load(path)] == [
"Name: Müller\nStadt: Köln",
"Name: Groß\nStadt: Düsseldorf",
]
@pytest.mark.parametrize(
"encoding,detected",
[
("utf-8", "utf-8-sig"),
("utf-8-sig", "utf-8-sig"),
("cp1252", "cp1252"),
("utf-16", "utf-16"),
],
)
def test_every_row_records_the_encoding_read(
self, tmp_path, encoding, detected
):
path = tmp_path / "people.csv"
path.write_bytes(ROWS.encode(encoding))
assert [d.metadata["detected_encoding"] for d in _load(path)] == [
detected,
detected,
]
def test_a_windows_1252_tsv_file_is_read(self, tmp_path):
path = tmp_path / "people.tsv"
path.write_bytes(ROWS.replace(",", "\t").encode("cp1252"))
assert [d.page_content for d in _load(path)][0] == (
"Name: Müller\nStadt: Köln"
)
class TestStructuredUploads:
"""JSON and YAML uploads are decoded the way text uploads are."""
@pytest.mark.parametrize("suffix", [".json", ".yaml"])
@pytest.mark.parametrize("encoding", ["utf-8-sig", "utf-16", "cp1252"])
def test_a_json_or_yaml_file_is_read_in_its_encoding(
self, tmp_path, suffix, encoding
):
# Notepad saves UTF-8 with a byte-order mark or as Windows-1252, and
# PowerShell's Out-File writes UTF-16
path = tmp_path / f"data{suffix}"
path.write_bytes('{"city": "Köln", "note": "Groß"}'.encode(encoding))
[document] = _load(path)
assert "Köln" in document.page_content
assert "Groß" in document.page_content
assert not document.metadata.get("parse_error")
@pytest.mark.parametrize("suffix", [".json", ".yaml"])
@pytest.mark.parametrize(
"encoding,detected",
[("utf-8", "utf-8-sig"), ("cp1252", "cp1252"), ("utf-16", "utf-16")],
)
def test_the_encoding_read_is_recorded_on_the_document(
self, tmp_path, suffix, encoding, detected
):
path = tmp_path / f"data{suffix}"
path.write_bytes('{"city": "Köln"}'.encode(encoding))
assert [d.metadata["detected_encoding"] for d in _load(path)] == [
detected
]
@pytest.mark.parametrize("suffix", [".json", ".yaml"])
def test_a_file_that_does_not_parse_records_the_encoding_too(
self, tmp_path, suffix
):
path = tmp_path / f"broken{suffix}"
path.write_bytes('{"city": "Köln", "note": [}'.encode("cp1252"))
[document] = _load(path)
assert document.metadata["parse_error"] is True
assert document.metadata["detected_encoding"] == "cp1252"