~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""Real extraction controls for the bundled TOML parser."""
import builtins
import json
import tomllib
from pathlib import Path
import pytest
from local_deep_research.document_loaders import (
extract_text_from_bytes,
load_from_bytes,
)
from local_deep_research.document_loaders.loader_registry import (
get_loader_for_path,
)
from local_deep_research.document_loaders.toml_loader import TOMLLoader
from local_deep_research.security import toml_limits
# One part more than MAX_TOML_KEY_PARTS (32).
_OVERLONG_KEY = ".".join(["k"] * 33)
def _forbid_parsing(monkeypatch):
"""Fail if tomllib runs, so a refusal must happen before parsing."""
def parse_attempted(*args, **kwargs):
raise AssertionError("tomllib ran on input the loader must refuse")
monkeypatch.setattr(tomllib, "load", parse_attempted)
monkeypatch.setattr(tomllib, "loads", parse_attempted)
def test_upload_reads_nested_values_and_unicode_without_tomli(monkeypatch):
original_import = builtins.__import__
def without_backport(name, *args, **kwargs):
if name == "tomli":
raise ModuleNotFoundError("tomli is intentionally unavailable")
return original_import(name, *args, **kwargs)
monkeypatch.setattr(builtins, "__import__", without_backport)
documents = load_from_bytes(
'[project]\nname = "研究"\nactive = true\nports = [80, 443]\n'.encode(),
".toml",
"project.toml",
)
assert len(documents) == 1
assert json.loads(documents[0].page_content) == {
"project": {"name": "研究", "active": True, "ports": [80, 443]}
}
assert documents[0].metadata["original_filename"] == "project.toml"
def test_local_file_reads_toml_dates(tmp_path):
path = tmp_path / "dates.toml"
path.write_text("day = 2026-10-02\nclock = 12:34:56\n", encoding="utf-8")
documents = get_loader_for_path(path).load()
assert json.loads(documents[0].page_content) == {
"day": "2026-10-02",
"clock": "12:34:56",
}
assert documents[0].metadata["source"] == str(path)
def test_directory_loads_each_nested_toml_file(tmp_path):
(tmp_path / "a.toml").write_text('name = "first"\n')
nested = tmp_path / "nested"
nested.mkdir()
(nested / "b.toml").write_text('name = "second"\n')
(nested / "ignored.txt").write_text("not TOML")
documents = TOMLLoader(tmp_path).load()
assert [json.loads(doc.page_content)["name"] for doc in documents] == [
"first",
"second",
]
def test_malformed_toml_does_not_produce_a_document(tmp_path):
path = tmp_path / "invalid.toml"
path.write_text("name = [\n")
with pytest.raises(tomllib.TOMLDecodeError):
TOMLLoader(path).load()
@pytest.mark.parametrize("filename", ["missing.toml", "wrong.txt"])
def test_invalid_source_is_rejected(tmp_path, filename):
path = tmp_path / filename
if path.suffix == ".txt":
path.write_text("not TOML")
with pytest.raises(ValueError, match="Invalid source path or file type"):
TOMLLoader(path).load()
def test_default_limits_accept_this_repository_pyproject():
path = Path(__file__).resolve().parents[2] / "pyproject.toml"
documents = TOMLLoader(path).load()
data = json.loads(documents[0].page_content)
assert data["project"]["name"] == "local-deep-research"
def test_size_limit_admits_its_own_size_and_refuses_one_byte_more(
tmp_path, monkeypatch
):
monkeypatch.setattr(toml_limits, "MAX_TOML_BYTES", 32)
at_limit = tmp_path / "at_limit.toml"
at_limit.write_bytes(b'name = "' + b"x" * 22 + b'"\n')
assert at_limit.stat().st_size == 32
documents = TOMLLoader(at_limit).load()
assert json.loads(documents[0].page_content) == {"name": "x" * 22}
over_limit = tmp_path / "over_limit.toml"
over_limit.write_bytes(b'name = "' + b"x" * 23 + b'"\n')
_forbid_parsing(monkeypatch)
with pytest.raises(toml_limits.TOMLLimitError, match="larger than 32"):
TOMLLoader(over_limit).load()
@pytest.mark.parametrize(
"document",
[
pytest.param(f"[{_OVERLONG_KEY}]\n", id="table-header"),
pytest.param(f"[[{_OVERLONG_KEY}]]\n", id="array-of-tables-header"),
pytest.param(f"{_OVERLONG_KEY} = 1\n", id="dotted-key"),
pytest.param(f"x = {{ {_OVERLONG_KEY} = 1 }}\n", id="inline-table"),
pytest.param(" . ".join(["k"] * 33) + " = 1\n", id="spaced-dots"),
pytest.param(
".".join(['"k k"', "'k.k'", "k"] * 11) + " = 1\n",
id="quoted-parts",
),
pytest.param(
f'x = ["""\n""", {{ {_OVERLONG_KEY} = 1 }}]\n',
id="after-a-multiline-string-closes",
),
pytest.param(
f"x = [\n # 'quote\n {{ {_OVERLONG_KEY} = 1 }},\n]\n",
id="inside-a-multiline-array",
),
pytest.param(f"x = 1\r\n{_OVERLONG_KEY} = 1\r\n", id="crlf"),
# Each case below is skipped by a scanner that takes the quotes or
# backslashes before the key as an unterminated string.
pytest.param(
f'# """\n{_OVERLONG_KEY} = 1\n', id="after-quotes-in-a-comment"
),
pytest.param(
f'x = "a\\"b"\n{_OVERLONG_KEY} = 1\n', id="after-an-escaped-quote"
),
pytest.param(
f'x = "a\\\\"\n{_OVERLONG_KEY} = 1\n',
id="after-an-escaped-backslash",
),
pytest.param(
f'x = """a\\"""b"""\n{_OVERLONG_KEY} = 1\n',
id="after-a-multiline-escaped-quote",
),
pytest.param(
f'x = """a\\\\"""\n{_OVERLONG_KEY} = 1\n',
id="after-a-multiline-escaped-backslash",
),
pytest.param(
f"x = 'a\\'\n{_OVERLONG_KEY} = 1\n", id="after-a-literal-backslash"
),
],
)
def test_overlong_key_is_refused_before_parsing(
tmp_path, monkeypatch, document
):
path = tmp_path / "deep.toml"
path.write_bytes(document.encode())
_forbid_parsing(monkeypatch)
with pytest.raises(
toml_limits.TOMLLimitError, match="more than 32 dotted parts"
):
TOMLLoader(path).load()
@pytest.mark.parametrize("opening", ['"', '"""', "'", "'''"])
def test_scan_stops_where_tomllib_rejects_an_unterminated_string(
tmp_path, opening
):
path = tmp_path / "unterminated.toml"
path.write_bytes(f"x = {opening}open\n{_OVERLONG_KEY} = 1\n".encode())
with pytest.raises(tomllib.TOMLDecodeError):
TOMLLoader(path).load()
def _nested(parts, leaf):
for part in reversed(parts):
leaf = {part: leaf}
return leaf
@pytest.mark.parametrize(
("document", "expected"),
[
pytest.param(
".".join(["k"] * 32) + " = 1\n",
_nested(["k"] * 32, 1),
id="key-at-the-limit",
),
pytest.param(
"[" + ".".join(['"k.k"'] * 32) + "]\n",
_nested(["k.k"] * 32, {}),
id="dots-inside-quoted-parts",
),
pytest.param(
f'text = "{_OVERLONG_KEY}"\n', {"text": _OVERLONG_KEY}, id="string"
),
pytest.param(
f"text = '''\n[{_OVERLONG_KEY}]\n'''\n",
{"text": f"[{_OVERLONG_KEY}]\n"},
id="multiline-string",
),
pytest.param(
f'# [{_OVERLONG_KEY}] """ \'\nx = 1\n', {"x": 1}, id="comment"
),
pytest.param(
"values = [" + ", ".join(["1.5"] * 40) + "]\n",
{"values": [1.5] * 40},
id="float-array",
),
],
)
def test_dotted_text_outside_overlong_keys_still_loads(
tmp_path, document, expected
):
path = tmp_path / "ordinary.toml"
path.write_bytes(document.encode())
documents = TOMLLoader(path).load()
assert json.loads(documents[0].page_content) == expected
def test_upload_refusal_reports_no_text_instead_of_parsing(monkeypatch):
content = f"{_OVERLONG_KEY} = 1\n".encode()
_forbid_parsing(monkeypatch)
with pytest.raises(toml_limits.TOMLLimitError):
load_from_bytes(content, ".toml", "deep.toml")
assert extract_text_from_bytes(content, ".toml", "deep.toml") is None