Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

1.9 KB
Raw Blame
"""Load TOML with Python's bundled parser, within the shared TOML limits."""

import json
from pathlib import Path
from typing import Iterator

from langchain_core.document_loaders import BaseLoader
from langchain_core.documents import Document

from ..security import toml_limits


def _read_limited_text(path: Path) -> str:
    """Return *path*'s text without reading past the TOML size limit."""
    with path.open("rb") as stream:
        content = stream.read(toml_limits.MAX_TOML_BYTES + 1)
    # Checked before decoding, so a read cut off inside a multi-byte
    # character is reported as too large rather than as invalid UTF-8.
    toml_limits.check_toml_size(len(content))
    return content.decode()


class TOMLLoader(BaseLoader):
    """Load a TOML file or directory without requiring the tomli backport.

    Each file is refused with TOMLLimitError, before tomllib parses it, when
    it is larger than MAX_TOML_BYTES or has a key or table header of more
    than MAX_TOML_KEY_PARTS dotted parts (see security.toml_limits).
    """

    def __init__(self, source: str | Path):
        # nosemgrep: semgrep.rules.path-traversal-risk, reason: load_from_bytes passes the temp file it wrote; no route passes a request path
        self.source = Path(source)

    def lazy_load(self) -> Iterator[Document]:
        if self.source.is_file() and self.source.suffix.lower() == ".toml":
            files = [self.source]
        elif self.source.is_dir():
            files = sorted(self.source.rglob("*.toml"))
        else:
            raise ValueError("Invalid source path or file type")

        for file_path in files:
            data = toml_limits.loads_toml_within_limits(
                _read_limited_text(file_path)
            )
            yield Document(
                page_content=json.dumps(data, ensure_ascii=False, default=str),
                metadata={"source": str(file_path)},
            )