Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

1.6 KB
Raw Blame
"""Check the production image's NLTK data and normal upload parser offline."""

import io

import nltk
from docx import Document

from local_deep_research.document_loaders import load_from_bytes


def main() -> None:
    # Both resources must be usable by the service user, with no download.
    sentence = nltk.sent_tokenize("Otters build dams.")[0]
    assert nltk.pos_tag(nltk.word_tokenize(sentence))
    document = Document()
    document.add_paragraph("Production parser round trip.")
    source = io.BytesIO()
    document.save(source)
    records = load_from_bytes(source.getvalue(), ".docx", "check.docx")
    assert [record.page_content for record in records] == [
        "Production parser round trip."
    ]
    assert records[0].metadata["original_filename"] == "check.docx"
    for extension, source in {
        ".md": b"# Check\n\nProduction parser round trip.",
        ".xml": b"<root><p>Production parser round trip.</p></root>",
        ".eml": (
            b"From: [email protected]\nTo: [email protected]\n"
            b"Subject: Check\nContent-Type: text/html\n\n"
            b"<p>Production parser round trip.</p>"
        ),
        ".enex": (
            b"<en-export><note><content><![CDATA[<en-note><p>"
            b"Production parser round trip."
            b"</p></en-note>]]></content></note></en-export>"
        ),
    }.items():
        records = load_from_bytes(source, extension, f"check{extension}")
        assert "Production parser round trip." in "\n".join(
            record.page_content for record in records
        )
    print("Upload parser check passed")


if __name__ == "__main__":
    main()