Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

5.4 KB
Raw Blame
"""Shared upload-parser behavior for registered structured text formats."""

import json
from email.message import EmailMessage
from pathlib import Path
from time import strptime, struct_time

import pytest
from langchain_core.documents import Document

from local_deep_research.document_loaders import bytes_loader
from local_deep_research.security import isolated_loader, isolated_worker

TEXT = "Otters build dams near rivers."
ENEX = (
    "<en-export><note><title>Sample note</title>"
    f"<content><![CDATA[<en-note><p>{TEXT}</p></en-note>]]></content>"
    "<created>20240305T134500Z</created><updated/>"
    "</note></en-export>"
).encode()


def _email() -> bytes:
    message = EmailMessage()
    message["From"] = "[email protected]"
    message["To"] = "[email protected]"
    message["Subject"] = "Sample document"
    message.set_content("Plain message alternative.")
    message.add_alternative(
        f"<html><body><p>{TEXT}</p></body></html>", subtype="html"
    )
    return message.as_bytes()


SAMPLES = {
    ".md": f"# Sample\n\n{TEXT}\n".encode(),
    ".markdown": f"# Sample\n\n{TEXT}\n".encode(),
    ".eml": _email(),
    ".xml": f"<root><paragraph>{TEXT}</paragraph></root>".encode(),
    ".enex": ENEX,
}


@pytest.mark.parametrize("extension", SAMPLES)
def test_structured_text_uploads_use_the_shared_child_path(
    monkeypatch, extension
):
    calls = []
    paths = []

    def isolated(path, ext):
        paths.append(Path(path))
        assert Path(path).read_bytes() == SAMPLES[extension]
        calls.append(ext)
        return [Document(page_content=TEXT)]

    def in_process(*args, **kwargs):
        raise AssertionError("structured text did not use the child path")

    monkeypatch.setattr(bytes_loader, "load_documents_isolated", isolated)
    monkeypatch.setattr(
        bytes_loader,
        "get_loader_class_for_extension",
        lambda ext: (in_process, {}),
    )
    documents = bytes_loader.load_from_bytes(
        SAMPLES[extension], extension, f"sample{extension}"
    )
    assert calls == [extension]
    assert documents[0].metadata["original_filename"] == f"sample{extension}"
    assert paths and all(not path.exists() for path in paths)


def test_upload_input_is_cleaned_when_staging_write_fails(monkeypatch):
    named_temporary_file = bytes_loader.tempfile.NamedTemporaryFile
    paths = []

    class FailedWrite:
        def __init__(self, *args, **kwargs):
            self.stream = named_temporary_file(*args, **kwargs)
            self.name = self.stream.name
            paths.append(Path(self.name))

        def __enter__(self):
            return self

        def __exit__(self, *args):
            self.stream.close()

        def write(self, data):
            self.stream.write(data[:4])
            raise OSError("temporary input write unavailable")

    monkeypatch.setattr(
        bytes_loader.tempfile, "NamedTemporaryFile", FailedWrite
    )
    with pytest.raises(OSError, match="write unavailable"):
        bytes_loader.load_from_bytes(SAMPLES[".md"], ".md", "sample.md")
    assert paths and all(not path.exists() for path in paths)


@pytest.mark.parametrize("extension", SAMPLES)
def test_normal_structured_text_round_trips_through_the_real_child(extension):
    documents = bytes_loader.load_from_bytes(
        SAMPLES[extension], extension, f"sample{extension}"
    )
    assert TEXT in "\n".join(document.page_content for document in documents)
    assert all(
        document.metadata["original_filename"] == f"sample{extension}"
        for document in documents
    )
    if extension == ".enex":
        assert documents[0].metadata["title"] == "Sample note"
        assert documents[0].metadata["created"] == strptime(
            "20240305T134500Z", "%Y%m%dT%H%M%SZ"
        )
        assert isinstance(documents[0].metadata["created"], struct_time)
        assert documents[0].metadata["updated"] is None


def test_evernote_metadata_keeps_dates_tags_and_optional_values(monkeypatch):
    created = strptime("20240305T134500Z", "%Y%m%dT%H%M%SZ")
    metadata = {"created": created, "updated": None, "tag": ["work", "urgent"]}

    class Loader:
        def __init__(self, path, **kwargs):
            pass

        def load(self):
            return [Document(page_content=TEXT, metadata=metadata)]

    from local_deep_research.document_loaders import loader_registry

    monkeypatch.setattr(
        loader_registry,
        "get_loader_class_for_extension",
        lambda extension: (Loader, {}),
    )
    result = isolated_worker._result("unused", ".enex")
    assert result["status"] == "ok"
    encoded = json.dumps(result).encode()
    restored = isolated_loader._documents_from_output(
        encoded, extension=".enex"
    )
    assert restored[0].metadata == metadata
    with pytest.raises(
        isolated_loader.IsolatedLoaderError, match="invalid documents"
    ):
        isolated_loader._documents_from_output(encoded, extension=".docx")


@pytest.mark.parametrize(
    "value",
    [[1] * 8, [True] * 9, {"unexpected": "object"}, [["nested"]]],
)
def test_evernote_protocol_refuses_unsupported_date_values(value):
    encoded = json.dumps(
        {
            "status": "ok",
            "documents": [
                {"page_content": TEXT, "metadata": {"created": value}}
            ],
        }
    ).encode()
    with pytest.raises(
        isolated_loader.IsolatedLoaderError, match="invalid documents"
    ):
        isolated_loader._documents_from_output(encoded, extension=".enex")