~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""Regression tests for office-document upload support (issue #4414).
ODT/DOC/DOCX/PPT/PPTX/XLS/XLSX were advertised as supported but failed at
runtime because their ``unstructured`` parser dependencies (python-docx,
python-pptx, openpyxl) were never installed. These tests assert that:
1. With the parser dependencies installed, the office extensions are
registered and text actually round-trips through the upload extraction
path (``extract_text_from_bytes``).
2. The registry is *honest*: a format is advertised only when its real
runtime dependency is importable, so a missing dependency hides the
format instead of accepting an upload that later fails silently.
"""
import importlib
import shutil
import tempfile
from pathlib import Path
import pytest
REGISTRY_MODULE = "local_deep_research.document_loaders.loader_registry"
def _reload(module_name):
return importlib.reload(importlib.import_module(module_name))
class TestOfficeExtensionsRegistered:
"""Office formats are advertised when their parser deps are present."""
def test_modern_office_formats_registered(self):
# Modern office formats work with pip-only parser deps (no system
# binaries), so they are registered whenever those deps are present.
pytest.importorskip("docx")
pytest.importorskip("pptx")
pytest.importorskip("openpyxl")
pytest.importorskip("msoffcrypto")
from local_deep_research.document_loaders.loader_registry import (
LOADER_REGISTRY,
)
for ext in (".docx", ".pptx", ".xlsx"):
assert ext in LOADER_REGISTRY, f"{ext} should be registered"
def test_legacy_binary_formats_gated_on_libreoffice(self):
# .doc/.ppt are OLE binary formats that unstructured converts via
# LibreOffice; they must be advertised only when soffice is present.
import local_deep_research.document_loaders.loader_registry as mod
for ext in (".doc", ".ppt"):
assert (ext in mod.LOADER_REGISTRY) == (
mod.HAS_LIBREOFFICE
and (mod.HAS_DOCX_DEP if ext == ".doc" else mod.HAS_PPTX_DEP)
)
def test_odt_registered_when_pandoc_present(self):
pytest.importorskip("pypandoc")
from local_deep_research.document_loaders.loader_registry import (
LOADER_REGISTRY,
)
assert ".odt" in LOADER_REGISTRY
class TestOfficeExtractionRoundTrip:
"""Real files extract text through the public upload path."""
def test_docx_extraction(self):
pytest.importorskip("docx")
from docx import Document
from local_deep_research.document_loaders import (
extract_text_from_bytes,
)
marker = "Hello DOCX world, extraction works."
with tempfile.NamedTemporaryFile(suffix=".docx") as tmp:
doc = Document()
doc.add_paragraph(marker)
doc.save(tmp.name)
content = Path(tmp.name).read_bytes()
text = extract_text_from_bytes(content, ".docx", "sample.docx")
assert text is not None
assert marker in text
def test_xls_extraction(self):
# xlwt (a dev-only writer) builds the legacy fixture; the loader reads
# it with pandas + xlrd, bypassing the fragile unstructured path.
xlwt = pytest.importorskip("xlwt")
pytest.importorskip("xlrd")
from local_deep_research.document_loaders import (
extract_text_from_bytes,
)
wb = xlwt.Workbook()
ws = wb.add_sheet("Data")
for r, (a, b) in enumerate(
[("Animal", "Count"), ("Otter", "3"), ("Badger", "7")]
):
ws.write(r, 0, a)
ws.write(r, 1, b)
with tempfile.NamedTemporaryFile(suffix=".xls", delete=False) as tmp:
xls_path = tmp.name
try:
wb.save(xls_path)
content = Path(xls_path).read_bytes()
finally:
Path(xls_path).unlink(missing_ok=True)
text = extract_text_from_bytes(content, ".xls", "sample.xls")
assert text is not None
assert "Animal" in text and "Otter" in text and "Badger" in text
def test_xls_keeps_a_cell_that_literally_says_na(self):
"""pandas reads "N/A" and friends as missing values.
Such a cell was dropped by the ``pd.notna`` filter, so a status column
full of "not applicable" left no trace in the extracted text.
"""
xlwt = pytest.importorskip("xlwt")
pytest.importorskip("xlrd")
from local_deep_research.document_loaders import (
extract_text_from_bytes,
)
literal_na = ["N/A", "NA", "n/a", "NULL", "None", "NaN", "nan"]
wb = xlwt.Workbook()
ws = wb.add_sheet("Data")
ws.write(0, 0, "Code")
ws.write(0, 1, "Status")
for row, value in enumerate(literal_na, start=1):
ws.write(row, 0, f"A{row}")
ws.write(row, 1, value)
with tempfile.NamedTemporaryFile(suffix=".xls", delete=False) as tmp:
xls_path = tmp.name
try:
wb.save(xls_path)
content = Path(xls_path).read_bytes()
finally:
Path(xls_path).unlink(missing_ok=True)
text = extract_text_from_bytes(content, ".xls", "sample.xls")
assert text is not None
for row, value in enumerate(literal_na, start=1):
assert f"A{row} {value}" in text
def test_xls_skips_a_genuinely_empty_cell(self):
"""The control: keeping the literal strings must not make a blank visible."""
xlwt = pytest.importorskip("xlwt")
pytest.importorskip("xlrd")
from local_deep_research.document_loaders import (
extract_text_from_bytes,
)
wb = xlwt.Workbook()
ws = wb.add_sheet("Data")
ws.write(0, 0, "Code")
ws.write(0, 1, "Status")
ws.write(0, 2, "Units")
ws.write(1, 0, "A1")
# column 1 left unwritten: a real blank
ws.write(1, 2, 12)
with tempfile.NamedTemporaryFile(suffix=".xls", delete=False) as tmp:
xls_path = tmp.name
try:
wb.save(xls_path)
content = Path(xls_path).read_bytes()
finally:
Path(xls_path).unlink(missing_ok=True)
text = extract_text_from_bytes(content, ".xls", "sample.xls")
assert text is not None
# A blank contributes nothing, and the number keeps its form rather
# than being upcast to 12.0 by the NaN beside it.
assert "A1 12" in text
def test_odt_extraction(self):
pypandoc = pytest.importorskip("pypandoc")
pytest.importorskip("docx")
if not shutil.which("pandoc"):
try:
pypandoc.get_pandoc_version()
except OSError:
pytest.skip("pandoc binary not available")
from local_deep_research.document_loaders import (
extract_text_from_bytes,
)
marker = "Hello ODT world, extraction works."
with tempfile.NamedTemporaryFile(suffix=".odt", delete=False) as tmp:
odt_path = tmp.name
try:
# Generate a genuine ODT (with styles.xml etc.) via pandoc.
pypandoc.convert_text(
marker, "odt", format="md", outputfile=odt_path
)
content = Path(odt_path).read_bytes()
finally:
Path(odt_path).unlink(missing_ok=True)
text = extract_text_from_bytes(content, ".odt", "sample.odt")
assert text is not None
assert marker in text
class TestXLSLoaderEncryption:
"""Encrypted XLS files produce a clear error, not a generic traceback."""
def test_encrypted_xls_raises_value_error(self, monkeypatch):
pytest.importorskip("xlrd")
from local_deep_research.document_loaders.xls_loader import XLSLoader
def fake_read_excel(*args, **kwargs):
raise Exception("Workbook is encrypted")
monkeypatch.setattr("pandas.read_excel", fake_read_excel)
loader = XLSLoader("/tmp/fake.xls")
with pytest.raises(ValueError, match="encrypted"):
loader.load()
def test_corrupted_xls_raises_original_error(self, monkeypatch):
pytest.importorskip("xlrd")
from local_deep_research.document_loaders.xls_loader import XLSLoader
def fake_read_excel(*args, **kwargs):
raise Exception("Unsupported format")
monkeypatch.setattr("pandas.read_excel", fake_read_excel)
loader = XLSLoader("/tmp/fake.xls")
with pytest.raises(Exception, match="Unsupported format"):
loader.load()
class TestHonestDetection:
"""A missing runtime dependency hides the format from the registry.
The registry computes its capability flags at import time, so we reload
the module with ``importlib.util.find_spec`` patched to report specific
modules as absent. Patching ``_module_available`` directly would not
survive the reload (it redefines that function from source), so we patch
the lower-level ``find_spec`` it calls.
"""
@pytest.fixture
def restore_module(self):
# Restore the pre-test module dict instead of reloading: pytest
# finalizes this fixture before monkeypatch (LIFO, it is listed
# after monkeypatch in the test signatures), so a teardown reload
# would re-execute the registry with find_spec still patched and
# leak the blocked-dependency state to later tests.
module = importlib.import_module(REGISTRY_MODULE)
snapshot = dict(module.__dict__)
yield
module.__dict__.clear()
module.__dict__.update(snapshot)
@staticmethod
def _reload_without(monkeypatch, blocked):
import importlib.util as importlib_util
real_find_spec = importlib_util.find_spec
def fake_find_spec(name, *args, **kwargs):
if name in blocked:
return None
return real_find_spec(name, *args, **kwargs)
monkeypatch.setattr(importlib_util, "find_spec", fake_find_spec)
return _reload(REGISTRY_MODULE)
def test_missing_python_docx_hides_word(self, monkeypatch, restore_module):
reloaded = self._reload_without(monkeypatch, {"docx"})
assert reloaded.HAS_DOCX_DEP is False
for ext in (".doc", ".docx"):
assert ext not in reloaded.LOADER_REGISTRY
# ODT converts through pandoc, not python-docx.
assert (".odt" in reloaded.LOADER_REGISTRY) == (
reloaded.HAS_ODT_LOADER and reloaded.HAS_PANDOC_DEP
)
def test_missing_pypandoc_hides_every_pandoc_format(
self, monkeypatch, restore_module
):
reloaded = self._reload_without(monkeypatch, {"pypandoc"})
assert reloaded.HAS_PANDOC_DEP is False
for ext in (".odt", ".epub", ".rtf", ".rst", ".org"):
assert ext not in reloaded.LOADER_REGISTRY
def test_missing_pptx_hides_powerpoint(self, monkeypatch, restore_module):
reloaded = self._reload_without(monkeypatch, {"pptx"})
assert reloaded.HAS_PPTX_DEP is False
for ext in (".ppt", ".pptx"):
assert ext not in reloaded.LOADER_REGISTRY
def test_missing_ocr_dep_hides_image_formats(
self, monkeypatch, restore_module
):
reloaded = self._reload_without(
monkeypatch, {"pytesseract", "unstructured.pytesseract"}
)
assert reloaded.HAS_OCR_DEP is False
for ext in (".png", ".jpg", ".jpeg", ".tiff", ".tif", ".bmp", ".heic"):
assert ext not in reloaded.LOADER_REGISTRY