Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

7.9 KB
Raw Blame
"""Real extraction controls for the bundled TOML parser."""

import builtins
import json
import tomllib
from pathlib import Path

import pytest

from local_deep_research.document_loaders import (
    extract_text_from_bytes,
    load_from_bytes,
)
from local_deep_research.document_loaders.loader_registry import (
    get_loader_for_path,
)
from local_deep_research.document_loaders.toml_loader import TOMLLoader
from local_deep_research.security import toml_limits

# One part more than MAX_TOML_KEY_PARTS (32).
_OVERLONG_KEY = ".".join(["k"] * 33)


def _forbid_parsing(monkeypatch):
    """Fail if tomllib runs, so a refusal must happen before parsing."""

    def parse_attempted(*args, **kwargs):
        raise AssertionError("tomllib ran on input the loader must refuse")

    monkeypatch.setattr(tomllib, "load", parse_attempted)
    monkeypatch.setattr(tomllib, "loads", parse_attempted)


def test_upload_reads_nested_values_and_unicode_without_tomli(monkeypatch):
    original_import = builtins.__import__

    def without_backport(name, *args, **kwargs):
        if name == "tomli":
            raise ModuleNotFoundError("tomli is intentionally unavailable")
        return original_import(name, *args, **kwargs)

    monkeypatch.setattr(builtins, "__import__", without_backport)
    documents = load_from_bytes(
        '[project]\nname = "研究"\nactive = true\nports = [80, 443]\n'.encode(),
        ".toml",
        "project.toml",
    )
    assert len(documents) == 1
    assert json.loads(documents[0].page_content) == {
        "project": {"name": "研究", "active": True, "ports": [80, 443]}
    }
    assert documents[0].metadata["original_filename"] == "project.toml"


def test_local_file_reads_toml_dates(tmp_path):
    path = tmp_path / "dates.toml"
    path.write_text("day = 2026-10-02\nclock = 12:34:56\n", encoding="utf-8")
    documents = get_loader_for_path(path).load()
    assert json.loads(documents[0].page_content) == {
        "day": "2026-10-02",
        "clock": "12:34:56",
    }
    assert documents[0].metadata["source"] == str(path)


def test_directory_loads_each_nested_toml_file(tmp_path):
    (tmp_path / "a.toml").write_text('name = "first"\n')
    nested = tmp_path / "nested"
    nested.mkdir()
    (nested / "b.toml").write_text('name = "second"\n')
    (nested / "ignored.txt").write_text("not TOML")
    documents = TOMLLoader(tmp_path).load()
    assert [json.loads(doc.page_content)["name"] for doc in documents] == [
        "first",
        "second",
    ]


def test_malformed_toml_does_not_produce_a_document(tmp_path):
    path = tmp_path / "invalid.toml"
    path.write_text("name = [\n")
    with pytest.raises(tomllib.TOMLDecodeError):
        TOMLLoader(path).load()


@pytest.mark.parametrize("filename", ["missing.toml", "wrong.txt"])
def test_invalid_source_is_rejected(tmp_path, filename):
    path = tmp_path / filename
    if path.suffix == ".txt":
        path.write_text("not TOML")
    with pytest.raises(ValueError, match="Invalid source path or file type"):
        TOMLLoader(path).load()


def test_default_limits_accept_this_repository_pyproject():
    path = Path(__file__).resolve().parents[2] / "pyproject.toml"
    documents = TOMLLoader(path).load()
    data = json.loads(documents[0].page_content)
    assert data["project"]["name"] == "local-deep-research"


def test_size_limit_admits_its_own_size_and_refuses_one_byte_more(
    tmp_path, monkeypatch
):
    monkeypatch.setattr(toml_limits, "MAX_TOML_BYTES", 32)
    at_limit = tmp_path / "at_limit.toml"
    at_limit.write_bytes(b'name = "' + b"x" * 22 + b'"\n')
    assert at_limit.stat().st_size == 32
    documents = TOMLLoader(at_limit).load()
    assert json.loads(documents[0].page_content) == {"name": "x" * 22}

    over_limit = tmp_path / "over_limit.toml"
    over_limit.write_bytes(b'name = "' + b"x" * 23 + b'"\n')
    _forbid_parsing(monkeypatch)
    with pytest.raises(toml_limits.TOMLLimitError, match="larger than 32"):
        TOMLLoader(over_limit).load()


@pytest.mark.parametrize(
    "document",
    [
        pytest.param(f"[{_OVERLONG_KEY}]\n", id="table-header"),
        pytest.param(f"[[{_OVERLONG_KEY}]]\n", id="array-of-tables-header"),
        pytest.param(f"{_OVERLONG_KEY} = 1\n", id="dotted-key"),
        pytest.param(f"x = {{ {_OVERLONG_KEY} = 1 }}\n", id="inline-table"),
        pytest.param(" . ".join(["k"] * 33) + " = 1\n", id="spaced-dots"),
        pytest.param(
            ".".join(['"k k"', "'k.k'", "k"] * 11) + " = 1\n",
            id="quoted-parts",
        ),
        pytest.param(
            f'x = ["""\n""", {{ {_OVERLONG_KEY} = 1 }}]\n',
            id="after-a-multiline-string-closes",
        ),
        pytest.param(
            f"x = [\n  # 'quote\n  {{ {_OVERLONG_KEY} = 1 }},\n]\n",
            id="inside-a-multiline-array",
        ),
        pytest.param(f"x = 1\r\n{_OVERLONG_KEY} = 1\r\n", id="crlf"),
        # Each case below is skipped by a scanner that takes the quotes or
        # backslashes before the key as an unterminated string.
        pytest.param(
            f'# """\n{_OVERLONG_KEY} = 1\n', id="after-quotes-in-a-comment"
        ),
        pytest.param(
            f'x = "a\\"b"\n{_OVERLONG_KEY} = 1\n', id="after-an-escaped-quote"
        ),
        pytest.param(
            f'x = "a\\\\"\n{_OVERLONG_KEY} = 1\n',
            id="after-an-escaped-backslash",
        ),
        pytest.param(
            f'x = """a\\"""b"""\n{_OVERLONG_KEY} = 1\n',
            id="after-a-multiline-escaped-quote",
        ),
        pytest.param(
            f'x = """a\\\\"""\n{_OVERLONG_KEY} = 1\n',
            id="after-a-multiline-escaped-backslash",
        ),
        pytest.param(
            f"x = 'a\\'\n{_OVERLONG_KEY} = 1\n", id="after-a-literal-backslash"
        ),
    ],
)
def test_overlong_key_is_refused_before_parsing(
    tmp_path, monkeypatch, document
):
    path = tmp_path / "deep.toml"
    path.write_bytes(document.encode())
    _forbid_parsing(monkeypatch)
    with pytest.raises(
        toml_limits.TOMLLimitError, match="more than 32 dotted parts"
    ):
        TOMLLoader(path).load()


@pytest.mark.parametrize("opening", ['"', '"""', "'", "'''"])
def test_scan_stops_where_tomllib_rejects_an_unterminated_string(
    tmp_path, opening
):
    path = tmp_path / "unterminated.toml"
    path.write_bytes(f"x = {opening}open\n{_OVERLONG_KEY} = 1\n".encode())
    with pytest.raises(tomllib.TOMLDecodeError):
        TOMLLoader(path).load()


def _nested(parts, leaf):
    for part in reversed(parts):
        leaf = {part: leaf}
    return leaf


@pytest.mark.parametrize(
    ("document", "expected"),
    [
        pytest.param(
            ".".join(["k"] * 32) + " = 1\n",
            _nested(["k"] * 32, 1),
            id="key-at-the-limit",
        ),
        pytest.param(
            "[" + ".".join(['"k.k"'] * 32) + "]\n",
            _nested(["k.k"] * 32, {}),
            id="dots-inside-quoted-parts",
        ),
        pytest.param(
            f'text = "{_OVERLONG_KEY}"\n', {"text": _OVERLONG_KEY}, id="string"
        ),
        pytest.param(
            f"text = '''\n[{_OVERLONG_KEY}]\n'''\n",
            {"text": f"[{_OVERLONG_KEY}]\n"},
            id="multiline-string",
        ),
        pytest.param(
            f'# [{_OVERLONG_KEY}] """ \'\nx = 1\n', {"x": 1}, id="comment"
        ),
        pytest.param(
            "values = [" + ", ".join(["1.5"] * 40) + "]\n",
            {"values": [1.5] * 40},
            id="float-array",
        ),
    ],
)
def test_dotted_text_outside_overlong_keys_still_loads(
    tmp_path, document, expected
):
    path = tmp_path / "ordinary.toml"
    path.write_bytes(document.encode())
    documents = TOMLLoader(path).load()
    assert json.loads(documents[0].page_content) == expected


def test_upload_refusal_reports_no_text_instead_of_parsing(monkeypatch):
    content = f"{_OVERLONG_KEY} = 1\n".encode()
    _forbid_parsing(monkeypatch)
    with pytest.raises(toml_limits.TOMLLimitError):
        load_from_bytes(content, ".toml", "deep.toml")
    assert extract_text_from_bytes(content, ".toml", "deep.toml") is None