Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

4.2 KB
Raw Blame
"""
Read text uploads in the encoding they were written in.

LangChain's ``autodetect_encoding`` passes every candidate that
``chardet.detect_all`` returns to ``FileEncoding(**candidate)``. chardet 7
adds a ``mime_type`` key to each candidate, so detection raises ``TypeError``
and every ``.txt`` file that is not UTF-8 is refused, readable or not. With an
older chardet, the same detection decodes binary bytes into text. Which of the
two happened depended on the installed chardet (#6532).

These loaders decide the encoding themselves, the same way everywhere. The
scope is deliberately narrow: UTF-8, UTF-16 and UTF-32 with a byte-order mark,
and Windows-1252, the Western-European legacy encoding Notepad and Excel write.
A file in any other legacy encoding has one of two outcomes:

- it is refused, when its bytes include one Windows-1252 leaves undefined or
  decode into control characters (cp866, and most Shift_JIS text);
- otherwise it is read as Windows-1252 and stored as mojibake (KOI8-R,
  GB18030, EUC-KR and EUC-JP text all decode this way).

The encoding each file was read in is recorded on its documents as
``detected_encoding``, for callers that keep the documents. The upload path
joins their text in ``extract_text_from_bytes`` and does not keep it yet
(#6836).
"""

import re
from collections.abc import Iterator
from pathlib import Path

from langchain_community.document_loaders import CSVLoader, TextLoader
from langchain_core.documents import Document

# Characters that legacy text does not contain: NUL and the C0 controls other
# than tab, line feed, vertical tab, form feed, carriage return, the DOS
# end-of-file mark (SUB) and escape.
_NOT_TEXT = re.compile(r"[\x00-\x08\x0e-\x19\x1c-\x1f]")


def _fallback_encodings(raw: bytes) -> list[str]:
    # UTF-32's little-endian mark begins with UTF-16's.
    if raw.startswith((b"\xff\xfe\x00\x00", b"\x00\x00\xfe\xff")):
        return ["utf-32", "cp1252"]
    if raw.startswith((b"\xff\xfe", b"\xfe\xff")):
        return ["utf-16", "cp1252"]
    # What Notepad and Excel write on Western-European Windows ("ANSI").
    return ["cp1252"]


def detect_text_encoding(raw: bytes) -> str | None:
    """
    Return the encoding that decodes ``raw`` into text, or None.

    Bytes that are UTF-8, with or without a byte-order mark, are read as
    UTF-8, as before. Other bytes are read as UTF-16 or UTF-32 when they start
    with its byte-order mark, and otherwise as Windows-1252, but only when the
    result is text: bytes that decode into control characters are refused.
    """
    try:
        raw.decode("utf-8-sig")
        return "utf-8-sig"
    except UnicodeDecodeError:
        pass
    for encoding in _fallback_encodings(raw):
        try:
            text = raw.decode(encoding)
        except UnicodeDecodeError:
            continue
        if not _NOT_TEXT.search(text):
            return encoding
    return None


def _file_encoding(file_path: str | Path) -> str:
    # nosemgrep: semgrep.rules.path-traversal-risk, reason: Encoding loader reads its explicit local file argument; byte uploads pass a server-created temporary file
    encoding = detect_text_encoding(Path(file_path).read_bytes())
    if encoding is None:
        raise ValueError(
            f"{Path(file_path).name}: no recognized text encoding (tried "
            "UTF-8, UTF-16 or UTF-32 with a byte-order mark, and "
            "Windows-1252)"
        )
    return encoding


def _with_encoding(
    documents: Iterator[Document], encoding: str
) -> Iterator[Document]:
    for document in documents:
        document.metadata["detected_encoding"] = encoding
        yield document


class EncodingAwareTextLoader(TextLoader):
    """A ``TextLoader`` that reads a file in the encoding it was written in."""

    def lazy_load(self) -> Iterator[Document]:
        self.encoding = _file_encoding(self.file_path)
        yield from _with_encoding(super().lazy_load(), self.encoding)


class EncodingAwareCSVLoader(CSVLoader):
    """A ``CSVLoader`` that reads a file in the encoding it was written in.

    A byte-order mark is dropped rather than read into the first column name.
    """

    def lazy_load(self) -> Iterator[Document]:
        self.encoding = _file_encoding(self.file_path)
        yield from _with_encoding(super().lazy_load(), self.encoding)