~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""Shared upload-parser behavior for registered structured text formats."""
import json
from email.message import EmailMessage
from pathlib import Path
from time import strptime, struct_time
import pytest
from langchain_core.documents import Document
from local_deep_research.document_loaders import bytes_loader
from local_deep_research.security import isolated_loader, isolated_worker
TEXT = "Otters build dams near rivers."
ENEX = (
"<en-export><note><title>Sample note</title>"
f"<content><![CDATA[<en-note><p>{TEXT}</p></en-note>]]></content>"
"<created>20240305T134500Z</created><updated/>"
"</note></en-export>"
).encode()
def _email() -> bytes:
message = EmailMessage()
message["From"] = "[email protected]"
message["To"] = "[email protected]"
message["Subject"] = "Sample document"
message.set_content("Plain message alternative.")
message.add_alternative(
f"<html><body><p>{TEXT}</p></body></html>", subtype="html"
)
return message.as_bytes()
SAMPLES = {
".md": f"# Sample\n\n{TEXT}\n".encode(),
".markdown": f"# Sample\n\n{TEXT}\n".encode(),
".eml": _email(),
".xml": f"<root><paragraph>{TEXT}</paragraph></root>".encode(),
".enex": ENEX,
}
@pytest.mark.parametrize("extension", SAMPLES)
def test_structured_text_uploads_use_the_shared_child_path(
monkeypatch, extension
):
calls = []
paths = []
def isolated(path, ext):
paths.append(Path(path))
assert Path(path).read_bytes() == SAMPLES[extension]
calls.append(ext)
return [Document(page_content=TEXT)]
def in_process(*args, **kwargs):
raise AssertionError("structured text did not use the child path")
monkeypatch.setattr(bytes_loader, "load_documents_isolated", isolated)
monkeypatch.setattr(
bytes_loader,
"get_loader_class_for_extension",
lambda ext: (in_process, {}),
)
documents = bytes_loader.load_from_bytes(
SAMPLES[extension], extension, f"sample{extension}"
)
assert calls == [extension]
assert documents[0].metadata["original_filename"] == f"sample{extension}"
assert paths and all(not path.exists() for path in paths)
def test_upload_input_is_cleaned_when_staging_write_fails(monkeypatch):
named_temporary_file = bytes_loader.tempfile.NamedTemporaryFile
paths = []
class FailedWrite:
def __init__(self, *args, **kwargs):
self.stream = named_temporary_file(*args, **kwargs)
self.name = self.stream.name
paths.append(Path(self.name))
def __enter__(self):
return self
def __exit__(self, *args):
self.stream.close()
def write(self, data):
self.stream.write(data[:4])
raise OSError("temporary input write unavailable")
monkeypatch.setattr(
bytes_loader.tempfile, "NamedTemporaryFile", FailedWrite
)
with pytest.raises(OSError, match="write unavailable"):
bytes_loader.load_from_bytes(SAMPLES[".md"], ".md", "sample.md")
assert paths and all(not path.exists() for path in paths)
@pytest.mark.parametrize("extension", SAMPLES)
def test_normal_structured_text_round_trips_through_the_real_child(extension):
documents = bytes_loader.load_from_bytes(
SAMPLES[extension], extension, f"sample{extension}"
)
assert TEXT in "\n".join(document.page_content for document in documents)
assert all(
document.metadata["original_filename"] == f"sample{extension}"
for document in documents
)
if extension == ".enex":
assert documents[0].metadata["title"] == "Sample note"
assert documents[0].metadata["created"] == strptime(
"20240305T134500Z", "%Y%m%dT%H%M%SZ"
)
assert isinstance(documents[0].metadata["created"], struct_time)
assert documents[0].metadata["updated"] is None
def test_evernote_metadata_keeps_dates_tags_and_optional_values(monkeypatch):
created = strptime("20240305T134500Z", "%Y%m%dT%H%M%SZ")
metadata = {"created": created, "updated": None, "tag": ["work", "urgent"]}
class Loader:
def __init__(self, path, **kwargs):
pass
def load(self):
return [Document(page_content=TEXT, metadata=metadata)]
from local_deep_research.document_loaders import loader_registry
monkeypatch.setattr(
loader_registry,
"get_loader_class_for_extension",
lambda extension: (Loader, {}),
)
result = isolated_worker._result("unused", ".enex")
assert result["status"] == "ok"
encoded = json.dumps(result).encode()
restored = isolated_loader._documents_from_output(
encoded, extension=".enex"
)
assert restored[0].metadata == metadata
with pytest.raises(
isolated_loader.IsolatedLoaderError, match="invalid documents"
):
isolated_loader._documents_from_output(encoded, extension=".docx")
@pytest.mark.parametrize(
"value",
[[1] * 8, [True] * 9, {"unexpected": "object"}, [["nested"]]],
)
def test_evernote_protocol_refuses_unsupported_date_values(value):
encoded = json.dumps(
{
"status": "ok",
"documents": [
{"page_content": TEXT, "metadata": {"created": value}}
],
}
).encode()
with pytest.raises(
isolated_loader.IsolatedLoaderError, match="invalid documents"
):
isolated_loader._documents_from_output(encoded, extension=".enex")