Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

fix(library): make ODT and office-doc uploads actually work (#4414) (#4415)

LearningCircuit433927068e4e
diff --git a/changelog.d/+odt-office-upload-deps.bugfix.md b/changelog.d/+odt-office-upload-deps.bugfix.md
new file mode 100644
index 0000000000..1a0644190e
--- /dev/null
+++ b/changelog.d/+odt-office-upload-deps.bugfix.md
@@ -0,0 +1 @@
+Library/collection uploads now actually accept ODT, DOCX, PPTX and XLSX/XLS files. These formats were advertised as supported but failed at runtime with a swallowed ``ModuleNotFoundError`` because their ``unstructured`` parser dependencies (``python-docx``, ``python-pptx``, ``openpyxl``, ``msoffcrypto-tool``, ``xlrd``) were never installed — only PDF worked. Those parsers are now declared dependencies, and the loader registry probes each format's real runtime dependency so it only advertises formats it can actually extract: missing deps now yield a clear "Unsupported format" instead of a silent failure. Legacy binary ``.doc``/``.ppt`` (which need a LibreOffice ``soffice`` binary) and image OCR (which needs ``pytesseract`` + ``tesseract``) are offered only when those tools are present.
diff --git a/docs/library-and-rag.md b/docs/library-and-rag.md
index 1084a7bae4..6af9f8a443 100644
--- a/docs/library-and-rag.md
+++ b/docs/library-and-rag.md
@@ -34,8 +34,29 @@ Access the library at: `http://localhost:5000/library`
|--------|-----------|-------|
| PDF | `.pdf` | Text extracted automatically |
| Plain Text | `.txt` | Direct text storage |
-| Markdown | `.md` | Rendered as text |
+| Markdown | `.md`, `.markdown` | Rendered as text |
| HTML | `.html`, `.htm` | Tags stripped, text extracted |
+| Word | `.docx` (and `.doc`*) | Text extracted via `unstructured` |
+| OpenDocument Text | `.odt` | Text extracted via `unstructured` |
+| PowerPoint | `.pptx` (and `.ppt`*) | Slide text extracted |
+| Excel | `.xlsx`, `.xls` | Cell text extracted |
+| Rich Text | `.rtf` | Text extracted |
+| EPUB | `.epub` | Text extracted |
+| Email | `.eml` | Body text extracted |
+| Data | `.csv`, `.tsv`, `.json`, `.yaml`, `.yml`, `.xml`, `.toml` | Parsed to text |
+| Notebooks | `.ipynb` | Cell sources and outputs |
+| Web archives | `.mhtml`, `.mht` | Saved web pages |
+
+The upload dialog's file picker is populated from the live list of formats the
+server can actually parse (`GET /library/api/config/supported-formats`), so it
+only offers formats whose parser dependencies are installed.
+
+\* The legacy binary formats `.doc` and `.ppt` are offered **only** when
+LibreOffice (`soffice`) is installed, because `unstructured` converts them to
+the modern format with it. Image formats (`.png`, `.jpg`, …) are offered
+**only** when the optional OCR extras (`pytesseract` plus the `tesseract`
+system binary) are installed. The default Docker image ships neither, so those
+formats are not offered there.
### Uploading Documents
diff --git a/pdm.lock b/pdm.lock
index abf18e6ce4..972faa77e2 100644
--- a/pdm.lock
+++ b/pdm.lock
@@ -5,7 +5,7 @@
groups = ["default", "dev", "mcp"]
strategy = ["inherit_metadata"]
lock_version = "4.5.0"
-content_hash = "sha256:63ab9cd74a01a92e437eb7257bfad6cc6e7cd7db822471b596083c4be01624af"
+content_hash = "sha256:c4a93e538a08e54cbbe966ec598c3bd9edaad7d88976f9f9886012319ab64b42"
[[metadata.targets]]
requires_python = ">=3.12,<3.15"
@@ -1366,6 +1366,17 @@ files = [
{file = "emoji-2.15.0.tar.gz", hash = "sha256:eae4ab7d86456a70a00a985125a03263a5eac54cd55e51d7e184b1ed3b6757e4"},
]
+[[package]]
+name = "et-xmlfile"
+version = "2.0.0"
+requires_python = ">=3.8"
+summary = "An implementation of lxml.xmlfile for the standard library"
+groups = ["default"]
+files = [
+ {file = "et_xmlfile-2.0.0-py3-none-any.whl", hash = "sha256:7a91720bc756843502c3b7504c77b8fe44217c85c537d85037f0f536151b2caa"},
+ {file = "et_xmlfile-2.0.0.tar.gz", hash = "sha256:dab3f4764309081ce75662649be815c4c9081e88f0837825f90fd28317d4da54"},
+]
+
[[package]]
name = "execnet"
version = "2.1.2"
@@ -3442,6 +3453,21 @@ files = [
{file = "mpmath-1.3.0.tar.gz", hash = "sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f"},
]
+[[package]]
+name = "msoffcrypto-tool"
+version = "6.0.0"
+requires_python = "<4.0,>=3.10"
+summary = "Python tool and library for decrypting and encrypting MS Office files using a password or other keys"
+groups = ["default"]
+dependencies = [
+ "cryptography>=39.0",
+ "olefile>=0.46",
+]
+files = [
+ {file = "msoffcrypto_tool-6.0.0-py3-none-any.whl", hash = "sha256:46c394ed5d9641e802fc79bf3fb0666a53748b23fa8c4aa634ae9d30d46fe397"},
+ {file = "msoffcrypto_tool-6.0.0.tar.gz", hash = "sha256:9a5ebc4c0096b42e5d7ebc2350afdc92dc511061e935ca188468094fdd032bbe"},
+]
+
[[package]]
name = "multidict"
version = "6.7.1"
@@ -3884,6 +3910,20 @@ files = [
{file = "openai-2.40.0.tar.gz", hash = "sha256:9a756f91f274a24ad6026cbcb2042fd356c8d4a10e8f347b08d34465e585f7a2"},
]
+[[package]]
+name = "openpyxl"
+version = "3.1.5"
+requires_python = ">=3.8"
+summary = "A Python library to read/write Excel 2010 xlsx/xlsm files"
+groups = ["default"]
+dependencies = [
+ "et-xmlfile",
+]
+files = [
+ {file = "openpyxl-3.1.5-py2.py3-none-any.whl", hash = "sha256:5282c12b107bffeef825f4617dc029afaf41d0ea60823bbb665ef3079dc79de2"},
+ {file = "openpyxl-3.1.5.tar.gz", hash = "sha256:cf0e3cf56142039133628b5acffe8ef0c12bc902d2aadd3e0fe5878dc08d1050"},
+]
+
[[package]]
name = "optuna"
version = "4.9.0"
@@ -4920,6 +4960,21 @@ files = [
{file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"},
]
+[[package]]
+name = "python-docx"
+version = "1.2.0"
+requires_python = ">=3.9"
+summary = "Create, read, and update Microsoft Word .docx files."
+groups = ["default"]
+dependencies = [
+ "lxml>=3.1.0",
+ "typing-extensions>=4.9.0",
+]
+files = [
+ {file = "python_docx-1.2.0-py3-none-any.whl", hash = "sha256:3fd478f3250fbbbfd3b94fe1e985955737c145627498896a8a6bf81f4baf66c7"},
+ {file = "python_docx-1.2.0.tar.gz", hash = "sha256:7bc9d7b7d8a69c9c02ca09216118c86552704edc23bac179283f2e38f86220ce"},
+]
+
[[package]]
name = "python-dotenv"
version = "1.2.2"
@@ -4994,6 +5049,23 @@ files = [
{file = "python_oxmsg-0.0.2.tar.gz", hash = "sha256:a6aff4deb1b5975d44d49dab1d9384089ffeec819e19c6940bc7ffbc84775fad"},
]
+[[package]]
+name = "python-pptx"
+version = "1.0.2"
+requires_python = ">=3.8"
+summary = "Create, read, and update PowerPoint 2007+ (.pptx) files."
+groups = ["default"]
+dependencies = [
+ "Pillow>=3.3.2",
+ "XlsxWriter>=0.5.7",
+ "lxml>=3.1.0",
+ "typing-extensions>=4.9.0",
+]
+files = [
+ {file = "python_pptx-1.0.2-py3-none-any.whl", hash = "sha256:160838e0b8565a8b1f67947675886e9fea18aa5e795db7ae531606d68e785cba"},
+ {file = "python_pptx-1.0.2.tar.gz", hash = "sha256:479a8af0eaf0f0d76b6f00b0887732874ad2e3188230315290cd1f9dd9cc7095"},
+]
+
[[package]]
name = "python-slugify"
version = "8.0.4"
@@ -6498,7 +6570,7 @@ version = "2026.2"
requires_python = ">=2"
summary = "Provider of IANA time zone data"
groups = ["default", "dev"]
-marker = "platform_system == \"Windows\" or sys_platform == \"win32\" or sys_platform == \"emscripten\" or python_version >= \"3.9\""
+marker = "platform_system == \"Windows\" or sys_platform == \"emscripten\" or sys_platform == \"win32\" or python_version >= \"3.9\""
files = [
{file = "tzdata-2026.2-py2.py3-none-any.whl", hash = "sha256:bbe9af844f658da81a5f95019480da3a89415801f6cc966806612cc7169bffe7"},
{file = "tzdata-2026.2.tar.gz", hash = "sha256:9173fde7d80d9018e02a662e168e5a2d04f87c41ea174b139fbef642eda62d10"},
@@ -6962,6 +7034,38 @@ files = [
{file = "wtforms-3.2.2.tar.gz", hash = "sha256:7b00c73f8670f35d4edb0293dcd81b980528bee72fd662b182aaba27ae570b93"},
]
+[[package]]
+name = "xlrd"
+version = "2.0.2"
+requires_python = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7"
+summary = "Library for developers to extract data from Microsoft Excel (tm) .xls spreadsheet files"
+groups = ["default"]
+files = [
+ {file = "xlrd-2.0.2-py2.py3-none-any.whl", hash = "sha256:ea762c3d29f4cca48d82df517b6d89fbce4db3107f9d78713e48cd321d5c9aa9"},
+ {file = "xlrd-2.0.2.tar.gz", hash = "sha256:08b5e25de58f21ce71dc7db3b3b8106c1fa776f3024c54e45b45b374e89234c9"},
+]
+
+[[package]]
+name = "xlsxwriter"
+version = "3.2.9"
+requires_python = ">=3.8"
+summary = "A Python module for creating Excel XLSX files."
+groups = ["default"]
+files = [
+ {file = "xlsxwriter-3.2.9-py3-none-any.whl", hash = "sha256:9a5db42bc5dff014806c58a20b9eae7322a134abb6fce3c92c181bfb275ec5b3"},
+ {file = "xlsxwriter-3.2.9.tar.gz", hash = "sha256:254b1c37a368c444eac6e2f867405cc9e461b0ed97a3233b2ac1e574efb4140c"},
+]
+
+[[package]]
+name = "xlwt"
+version = "1.3.0"
+summary = "Library to create spreadsheet files compatible with MS Excel 97/2000/XP/2003 XLS files, on any platform, with Python 2.6, 2.7, 3.3+"
+groups = ["dev"]
+files = [
+ {file = "xlwt-1.3.0-py2.py3-none-any.whl", hash = "sha256:a082260524678ba48a297d922cc385f58278b8aa68741596a87de01a9c628b2e"},
+ {file = "xlwt-1.3.0.tar.gz", hash = "sha256:c59912717a9b28f1a3c2a98fd60741014b06b043936dcecbc113eaaada156c88"},
+]
+
[[package]]
name = "xmltodict"
version = "1.0.4"
diff --git a/pyproject.toml b/pyproject.toml
index e6d7aafba6..8ff6185aeb 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -65,6 +65,16 @@ dependencies = [
"nh3~=0.3", # Backend HTML sanitization
"pdfplumber~=0.11",
"unstructured~=0.18",
+ # Runtime parsers for the unstructured office-document loaders. These are
+ # NOT pulled in by bare `unstructured` (only by its `[docx]`/`[odt]`/`[pptx]`/
+ # `[xlsx]` extras), yet the loader registry advertises .doc/.docx/.odt/.ppt/
+ # .pptx/.xls/.xlsx as supported. Without them those uploads fail at runtime
+ # with `ModuleNotFoundError: No module named 'docx'` (see issue #4414).
+ "python-docx~=1.2", # .doc, .docx, .odt (odt parser imports python-docx)
+ "python-pptx~=1.0", # .ppt, .pptx
+ "openpyxl~=3.1", # .xlsx (unstructured Excel partition)
+ "msoffcrypto-tool~=6.0", # imported at module load by unstructured.partition.xlsx
+ "xlrd~=2.0", # legacy .xls reading (pandas engine)
"google-search-results~=2.4",
"jaraco-context~=6.1", # CVE GHSA-58pv-8j8x-9vj2 fix for path traversal (also pinned as transitive dep)
"flask-wtf~=1.2",
@@ -180,6 +190,7 @@ dev = [
"hypothesis~=6.151",
"vulture~=2.14",
"towncrier~=24.8",
+ "xlwt~=1.3", # test-only: writes legacy .xls fixtures for XLSLoader tests
]
[tool.pytest.ini_options]
diff --git a/src/local_deep_research/document_loaders/bytes_loader.py b/src/local_deep_research/document_loaders/bytes_loader.py
index f9b7da93e5..32606f36fd 100644
--- a/src/local_deep_research/document_loaders/bytes_loader.py
+++ b/src/local_deep_research/document_loaders/bytes_loader.py
@@ -93,6 +93,17 @@ def load_from_bytes(
)
return documents # type: ignore[no-any-return]
+ except (ImportError, ModuleNotFoundError):
+ # The extension is registered but the loader's runtime parser
+ # dependency is missing. The registry normally prevents this
+ # (see loader_registry capability flags), so surface it loudly
+ # rather than as a generic extraction failure.
+ logger.exception(
+ f"Missing optional dependency for {ext} files while loading "
+ f"{filename}; the loader for this format is not fully installed"
+ )
+ raise
+
except Exception:
logger.exception(f"Error loading {filename} ({ext})")
raise
diff --git a/src/local_deep_research/document_loaders/loader_registry.py b/src/local_deep_research/document_loaders/loader_registry.py
index 490d0786c8..e041b83fd6 100644
--- a/src/local_deep_research/document_loaders/loader_registry.py
+++ b/src/local_deep_research/document_loaders/loader_registry.py
@@ -5,6 +5,8 @@ This module provides a centralized registry for document loaders that can be use
by both collection uploads (bytes) and local search (file paths).
"""
+import importlib.util
+import shutil
from pathlib import Path
from typing import Optional
@@ -83,9 +85,59 @@ except ImportError:
# Import our custom loaders
from .json_loader import SimpleJSONLoader
+from .xls_loader import XLSLoader
from .yaml_loader import YAMLLoader
+def _module_available(module_name: str) -> bool:
+ """Return True if *module_name* can be imported in this environment.
+
+ Uses ``importlib.util.find_spec`` so the (potentially heavy) module is
+ located but not actually imported.
+ """
+ try:
+ return importlib.util.find_spec(module_name) is not None
+ except (ImportError, ValueError, ModuleNotFoundError):
+ # A parent package may itself be missing, which surfaces as
+ # ModuleNotFoundError/ImportError from find_spec.
+ return False
+
+
+# Whether a loader *class* can be imported is NOT sufficient to know whether a
+# format actually works: the ``unstructured`` loaders import their real parser
+# (and its third-party dependency) lazily inside ``.load()``. So a format like
+# ODT can be advertised as supported, accept an upload, and only then fail with
+# ``ModuleNotFoundError: No module named 'docx'`` (issue #4414).
+#
+# These flags probe the actual runtime dependency so the registry only
+# advertises formats that will really extract text. Missing deps mean the
+# format is omitted and the upload path returns a clear "Unsupported format"
+# instead of a swallowed extraction failure.
+HAS_DOCX_DEP = _module_available("docx") # python-docx: .docx/.odt
+HAS_PPTX_DEP = _module_available("pptx") # python-pptx: .pptx
+# Legacy OLE binary office formats (.doc/.ppt) are NOT read by python-docx/
+# python-pptx; unstructured converts them to the modern format by shelling out
+# to LibreOffice (``soffice``). Without that binary they fail at runtime with
+# "soffice command was not found", so only advertise them when it is present.
+HAS_LIBREOFFICE = bool(
+ shutil.which("soffice") or shutil.which("libreoffice")
+) # .doc/.ppt conversion
+# unstructured.partition.xlsx imports msoffcrypto at module load, so openpyxl
+# alone is not enough to extract a modern .xlsx spreadsheet.
+HAS_XLSX_DEP = _module_available("openpyxl") and _module_available(
+ "msoffcrypto"
+) # openpyxl + msoffcrypto: .xlsx
+# Legacy .xls is read by our own XLSLoader (pandas + xlrd), which avoids the
+# msoffcrypto pre-check that crashes on some .xls files in the unstructured path.
+HAS_XLS_DEP = _module_available("xlrd") # xlrd: .xls
+HAS_PANDOC_DEP = _module_available(
+ "pypandoc"
+) # pandoc bridge: epub/rtf/rst/org
+HAS_OCR_DEP = _module_available("pytesseract") or _module_available(
+ "unstructured.pytesseract"
+) # tesseract OCR for image formats (also needs the tesseract binary)
+
+
# Extension to loader mapping
# Each entry contains:
# - loader_class: The LangChain loader class
@@ -111,28 +163,11 @@ LOADER_REGISTRY: dict = {
"loader_class": UnstructuredMarkdownLoader,
"loader_kwargs": {},
},
- # Word documents
- ".docx": {
- "loader_class": UnstructuredWordDocumentLoader,
- "loader_kwargs": {},
- },
- ".doc": {
- "loader_class": UnstructuredWordDocumentLoader,
- "loader_kwargs": {},
- },
- # Spreadsheets
+ # CSV (Word/Excel are registered below, gated on their runtime deps)
".csv": {
"loader_class": CSVLoader,
"loader_kwargs": {},
},
- ".xlsx": {
- "loader_class": UnstructuredExcelLoader,
- "loader_kwargs": {},
- },
- ".xls": {
- "loader_class": UnstructuredExcelLoader,
- "loader_kwargs": {},
- },
# HTML
".html": {
"loader_class": UnstructuredHTMLLoader,
@@ -144,32 +179,62 @@ LOADER_REGISTRY: dict = {
},
}
-# ODT (requires pypandoc)
-if HAS_ODT_LOADER:
+# Word documents - .docx needs python-docx; legacy .doc additionally needs
+# LibreOffice to convert the OLE binary to .docx first.
+if HAS_DOCX_DEP:
+ LOADER_REGISTRY[".docx"] = {
+ "loader_class": UnstructuredWordDocumentLoader,
+ "loader_kwargs": {},
+ }
+ if HAS_LIBREOFFICE:
+ LOADER_REGISTRY[".doc"] = {
+ "loader_class": UnstructuredWordDocumentLoader,
+ "loader_kwargs": {},
+ }
+
+# Modern spreadsheets - require openpyxl + msoffcrypto (unstructured path)
+if HAS_XLSX_DEP:
+ LOADER_REGISTRY[".xlsx"] = {
+ "loader_class": UnstructuredExcelLoader,
+ "loader_kwargs": {},
+ }
+
+# Legacy spreadsheets - read directly via our XLSLoader (pandas + xlrd)
+if HAS_XLS_DEP:
+ LOADER_REGISTRY[".xls"] = {
+ "loader_class": XLSLoader,
+ "loader_kwargs": {},
+ }
+
+# ODT - the unstructured ODT partitioner imports python-docx internally
+if HAS_ODT_LOADER and HAS_DOCX_DEP:
LOADER_REGISTRY[".odt"] = {
"loader_class": UnstructuredODTLoader,
"loader_kwargs": {},
}
-# PowerPoint presentations
-LOADER_REGISTRY[".ppt"] = {
- "loader_class": UnstructuredPowerPointLoader,
- "loader_kwargs": {},
-}
-LOADER_REGISTRY[".pptx"] = {
- "loader_class": UnstructuredPowerPointLoader,
- "loader_kwargs": {},
-}
+# PowerPoint presentations - .pptx needs python-pptx; legacy .ppt additionally
+# needs LibreOffice to convert the OLE binary to .pptx first.
+if HAS_PPTX_DEP:
+ LOADER_REGISTRY[".pptx"] = {
+ "loader_class": UnstructuredPowerPointLoader,
+ "loader_kwargs": {},
+ }
+ if HAS_LIBREOFFICE:
+ LOADER_REGISTRY[".ppt"] = {
+ "loader_class": UnstructuredPowerPointLoader,
+ "loader_kwargs": {},
+ }
# EPUB (ebooks, technical manuals) - requires pandoc
-if HAS_EPUB_LOADER:
+if HAS_EPUB_LOADER and HAS_PANDOC_DEP:
LOADER_REGISTRY[".epub"] = {
"loader_class": UnstructuredEPubLoader,
"loader_kwargs": {},
}
# RTF (Rich Text Format) - requires pandoc
-if HAS_RTF_LOADER:
+if HAS_RTF_LOADER and HAS_PANDOC_DEP:
LOADER_REGISTRY[".rtf"] = {
"loader_class": UnstructuredRTFLoader,
"loader_kwargs": {},
@@ -182,14 +247,14 @@ LOADER_REGISTRY[".xml"] = {
}
# RST (reStructuredText) - requires pandoc
-if HAS_RST_LOADER:
+if HAS_RST_LOADER and HAS_PANDOC_DEP:
LOADER_REGISTRY[".rst"] = {
"loader_class": UnstructuredRSTLoader,
"loader_kwargs": {},
}
# Org-mode files - requires pandoc
-if HAS_ORG_LOADER:
+if HAS_ORG_LOADER and HAS_PANDOC_DEP:
LOADER_REGISTRY[".org"] = {
"loader_class": UnstructuredOrgModeLoader,
"loader_kwargs": {},
@@ -223,8 +288,8 @@ LOADER_REGISTRY[".yml"] = {
"loader_kwargs": {},
}
-# Images with OCR support (requires tesseract)
-if HAS_IMAGE_LOADER:
+# Images with OCR support - require pytesseract (and the tesseract binary)
+if HAS_IMAGE_LOADER and HAS_OCR_DEP:
for ext in [".png", ".jpg", ".jpeg", ".tiff", ".tif", ".bmp", ".heic"]:
LOADER_REGISTRY[ext] = {
"loader_class": UnstructuredImageLoader,
diff --git a/src/local_deep_research/document_loaders/xls_loader.py b/src/local_deep_research/document_loaders/xls_loader.py
new file mode 100644
index 0000000000..230f0f77bf
--- /dev/null
+++ b/src/local_deep_research/document_loaders/xls_loader.py
@@ -0,0 +1,66 @@
+"""Custom loader for legacy ``.xls`` (BIFF) spreadsheets.
+
+``unstructured``'s Excel partitioner runs a ``msoffcrypto`` "is encrypted?"
+pre-check on every workbook. That check crashes on some legacy ``.xls`` files
+(``struct.error: unpack requires a buffer of 4 bytes`` in its xls97 record
+reader) even though ``xlrd`` reads the same file fine. To make ``.xls`` uploads
+extract reliably, this loader reads the workbook directly with pandas + xlrd
+and bypasses that pre-check. Modern ``.xlsx`` files still use the unstructured
+loader, whose OOXML path is unaffected.
+"""
+
+from pathlib import Path
+from typing import Iterator
+
+import pandas as pd
+from langchain_core.document_loaders import BaseLoader
+from langchain_core.documents import Document
+from loguru import logger
+
+
+class XLSLoader(BaseLoader):
+ """Load a legacy ``.xls`` workbook and convert each sheet to text.
+
+ Every sheet is read with the ``xlrd`` engine and rendered as one
+ space-joined line per row (skipping empty cells), which mirrors the plain
+ text that the unstructured Excel loader produces for ``.xlsx``.
+ """
+
+ def __init__(self, file_path: str | Path, **kwargs: object):
+ self.file_path = Path(file_path)
+
+ def lazy_load(self) -> Iterator[Document]:
+ try:
+ # sheet_name=None reads all sheets; header=None keeps the first row
+ # as data so column headers are included in the extracted text.
+ sheets = pd.read_excel(
+ self.file_path,
+ sheet_name=None,
+ header=None,
+ engine="xlrd",
+ )
+ except Exception:
+ logger.exception(f"Error loading XLS file: {self.file_path}")
+ raise
+
+ for sheet_name, frame in sheets.items():
+ lines = []
+ for row in frame.itertuples(index=False, name=None):
+ cells = [str(value) for value in row if pd.notna(value)]
+ if cells:
+ lines.append(" ".join(cells))
+ text = "\n".join(lines).strip()
+ if not text:
+ continue
+ yield Document(
+ page_content=text,
+ metadata={
+ "source": str(self.file_path),
+ "file_type": "xls",
+ "sheet_name": sheet_name,
+ },
+ )
+
+ def load(self) -> list[Document]:
+ """Load all sheets from the XLS file."""
+ return list(self.lazy_load())
diff --git a/tests/document_loaders/test_loader_registry.py b/tests/document_loaders/test_loader_registry.py
index 1f7dceabd5..db02b98d6e 100644
--- a/tests/document_loaders/test_loader_registry.py
+++ b/tests/document_loaders/test_loader_registry.py
@@ -30,7 +30,8 @@ class TestGetSupportedExtensions:
assert ".txt" in extensions
assert ".md" in extensions
assert ".docx" in extensions
- assert ".doc" in extensions
+ # .doc (legacy OLE binary) is only registered when LibreOffice is
+ # installed, so it is intentionally not asserted here.
# Spreadsheets
assert ".csv" in extensions
diff --git a/tests/document_loaders/test_loader_registry_coverage.py b/tests/document_loaders/test_loader_registry_coverage.py
index 334ea33179..95eb906e50 100644
--- a/tests/document_loaders/test_loader_registry_coverage.py
+++ b/tests/document_loaders/test_loader_registry_coverage.py
@@ -63,13 +63,11 @@ class TestRegistrySpecificLoaderClasses:
(".md", "UnstructuredMarkdownLoader"),
(".markdown", "UnstructuredMarkdownLoader"),
(".docx", "UnstructuredWordDocumentLoader"),
- (".doc", "UnstructuredWordDocumentLoader"),
(".csv", "CSVLoader"),
(".xlsx", "UnstructuredExcelLoader"),
- (".xls", "UnstructuredExcelLoader"),
+ (".xls", "XLSLoader"),
(".html", "UnstructuredHTMLLoader"),
(".htm", "UnstructuredHTMLLoader"),
- (".ppt", "UnstructuredPowerPointLoader"),
(".pptx", "UnstructuredPowerPointLoader"),
(".xml", "UnstructuredXMLLoader"),
(".eml", "UnstructuredEmailLoader"),
@@ -168,15 +166,17 @@ class TestOptionalLoaderFlags:
from local_deep_research.document_loaders import loader_registry as mod
image_exts = [".png", ".jpg", ".jpeg", ".tiff", ".tif", ".bmp", ".heic"]
- if mod.HAS_IMAGE_LOADER:
+ # Images require both the loader class and the OCR runtime dep
+ # (pytesseract + the tesseract binary).
+ if mod.HAS_IMAGE_LOADER and mod.HAS_OCR_DEP:
for ext in image_exts:
assert ext in LOADER_REGISTRY, (
- f"{ext} missing despite HAS_IMAGE_LOADER"
+ f"{ext} missing despite image loader + OCR dep"
)
else:
for ext in image_exts:
assert ext not in LOADER_REGISTRY, (
- f"{ext} present despite no HAS_IMAGE_LOADER"
+ f"{ext} present despite missing image loader or OCR dep"
)
@@ -271,23 +271,30 @@ class TestGetLoaderClassForExtensionCoverage:
assert r1 is not None and r2 is not None
assert r1[0] is r2[0]
- def test_xls_same_class_as_xlsx(self):
- r1 = get_loader_class_for_extension(".xls")
- r2 = get_loader_class_for_extension(".xlsx")
- assert r1 is not None and r2 is not None
- assert r1[0] is r2[0]
-
- def test_doc_same_class_as_docx(self):
+ def test_xls_and_xlsx_use_distinct_loaders(self):
+ # .xlsx uses the unstructured Excel loader; legacy .xls uses our own
+ # XLSLoader (pandas + xlrd) to avoid the msoffcrypto pre-check crash.
+ r_xls = get_loader_class_for_extension(".xls")
+ r_xlsx = get_loader_class_for_extension(".xlsx")
+ assert r_xls is not None and r_xlsx is not None
+ assert r_xls[0].__name__ == "XLSLoader"
+ assert r_xlsx[0].__name__ == "UnstructuredExcelLoader"
+
+ def test_doc_same_class_as_docx_when_registered(self):
+ # .doc is only registered when LibreOffice is available; when it is,
+ # it must share the Word loader class with .docx.
r1 = get_loader_class_for_extension(".doc")
r2 = get_loader_class_for_extension(".docx")
- assert r1 is not None and r2 is not None
- assert r1[0] is r2[0]
+ assert r2 is not None
+ if r1 is not None:
+ assert r1[0] is r2[0]
- def test_ppt_same_class_as_pptx(self):
+ def test_ppt_same_class_as_pptx_when_registered(self):
r1 = get_loader_class_for_extension(".ppt")
r2 = get_loader_class_for_extension(".pptx")
- assert r1 is not None and r2 is not None
- assert r1[0] is r2[0]
+ assert r2 is not None
+ if r1 is not None:
+ assert r1[0] is r2[0]
@pytest.mark.parametrize("ext", list(LOADER_REGISTRY.keys()))
def test_all_registry_extensions_resolve(self, ext):
diff --git a/tests/document_loaders/test_loader_registry_deep_coverage.py b/tests/document_loaders/test_loader_registry_deep_coverage.py
index 5b1120aebb..0afdac9579 100644
--- a/tests/document_loaders/test_loader_registry_deep_coverage.py
+++ b/tests/document_loaders/test_loader_registry_deep_coverage.py
@@ -157,13 +157,11 @@ class TestAlwaysPresentExtensions:
".md",
".markdown",
".docx",
- ".doc",
".csv",
".xlsx",
".xls",
".html",
".htm",
- ".ppt",
".pptx",
".xml",
".eml",
diff --git a/tests/document_loaders/test_loader_registry_edge_cases.py b/tests/document_loaders/test_loader_registry_edge_cases.py
index 9ef7eacd82..3f288d0d15 100644
--- a/tests/document_loaders/test_loader_registry_edge_cases.py
+++ b/tests/document_loaders/test_loader_registry_edge_cases.py
@@ -179,7 +179,8 @@ class TestOptionalLoaderFlagGating:
assert (".org" in mod.LOADER_REGISTRY) == mod.HAS_ORG_LOADER
def test_image_extensions_gated_by_flag(self):
- """All image extensions are present iff HAS_IMAGE_LOADER is True."""
+ """Image extensions are present iff the loader class AND the OCR
+ runtime dependency (pytesseract + tesseract binary) are available."""
import local_deep_research.document_loaders.loader_registry as mod
image_exts = [
@@ -191,9 +192,12 @@ class TestOptionalLoaderFlagGating:
".bmp",
".heic",
]
+ expected = mod.HAS_IMAGE_LOADER and mod.HAS_OCR_DEP
for ext in image_exts:
- assert (ext in mod.LOADER_REGISTRY) == mod.HAS_IMAGE_LOADER, (
- f"{ext} registration does not match HAS_IMAGE_LOADER={mod.HAS_IMAGE_LOADER}"
+ assert (ext in mod.LOADER_REGISTRY) == expected, (
+ f"{ext} registration does not match "
+ f"HAS_IMAGE_LOADER={mod.HAS_IMAGE_LOADER} "
+ f"HAS_OCR_DEP={mod.HAS_OCR_DEP}"
)
diff --git a/tests/document_loaders/test_loader_registry_full_coverage.py b/tests/document_loaders/test_loader_registry_full_coverage.py
index c1df0a05b9..db0cd97e91 100644
--- a/tests/document_loaders/test_loader_registry_full_coverage.py
+++ b/tests/document_loaders/test_loader_registry_full_coverage.py
@@ -113,55 +113,73 @@ class TestConditionalLoaderRegistration:
assert isinstance(HAS_IMAGE_LOADER, bool)
- def test_odt_in_registry_iff_flag_true(self):
+ def test_odt_in_registry_iff_flag_and_dep(self):
+ # ODT needs both the loader class and python-docx (the partitioner
+ # imports python-docx and shells out to pandoc).
from local_deep_research.document_loaders.loader_registry import (
+ HAS_DOCX_DEP,
HAS_ODT_LOADER,
LOADER_REGISTRY,
)
- assert (".odt" in LOADER_REGISTRY) == HAS_ODT_LOADER
+ assert (".odt" in LOADER_REGISTRY) == (HAS_ODT_LOADER and HAS_DOCX_DEP)
- def test_epub_in_registry_iff_flag_true(self):
+ def test_epub_in_registry_iff_flag_and_dep(self):
from local_deep_research.document_loaders.loader_registry import (
HAS_EPUB_LOADER,
+ HAS_PANDOC_DEP,
LOADER_REGISTRY,
)
- assert (".epub" in LOADER_REGISTRY) == HAS_EPUB_LOADER
+ assert (".epub" in LOADER_REGISTRY) == (
+ HAS_EPUB_LOADER and HAS_PANDOC_DEP
+ )
- def test_rtf_in_registry_iff_flag_true(self):
+ def test_rtf_in_registry_iff_flag_and_dep(self):
from local_deep_research.document_loaders.loader_registry import (
+ HAS_PANDOC_DEP,
HAS_RTF_LOADER,
LOADER_REGISTRY,
)
- assert (".rtf" in LOADER_REGISTRY) == HAS_RTF_LOADER
+ assert (".rtf" in LOADER_REGISTRY) == (
+ HAS_RTF_LOADER and HAS_PANDOC_DEP
+ )
- def test_rst_in_registry_iff_flag_true(self):
+ def test_rst_in_registry_iff_flag_and_dep(self):
from local_deep_research.document_loaders.loader_registry import (
+ HAS_PANDOC_DEP,
HAS_RST_LOADER,
LOADER_REGISTRY,
)
- assert (".rst" in LOADER_REGISTRY) == HAS_RST_LOADER
+ assert (".rst" in LOADER_REGISTRY) == (
+ HAS_RST_LOADER and HAS_PANDOC_DEP
+ )
- def test_org_in_registry_iff_flag_true(self):
+ def test_org_in_registry_iff_flag_and_dep(self):
from local_deep_research.document_loaders.loader_registry import (
HAS_ORG_LOADER,
+ HAS_PANDOC_DEP,
LOADER_REGISTRY,
)
- assert (".org" in LOADER_REGISTRY) == HAS_ORG_LOADER
+ assert (".org" in LOADER_REGISTRY) == (
+ HAS_ORG_LOADER and HAS_PANDOC_DEP
+ )
- def test_image_extensions_in_registry_iff_flag_true(self):
+ def test_image_extensions_in_registry_iff_flag_and_dep(self):
from local_deep_research.document_loaders.loader_registry import (
HAS_IMAGE_LOADER,
+ HAS_OCR_DEP,
LOADER_REGISTRY,
)
image_exts = [".png", ".jpg", ".jpeg", ".tiff", ".tif", ".bmp", ".heic"]
for ext in image_exts:
- assert (ext in LOADER_REGISTRY) == HAS_IMAGE_LOADER
+ assert (ext in LOADER_REGISTRY) == (
+ HAS_IMAGE_LOADER and HAS_OCR_DEP
+ )
class TestIsExtensionSupportedCaseInsensitive:
diff --git a/tests/document_loaders/test_office_format_extraction.py b/tests/document_loaders/test_office_format_extraction.py
new file mode 100644
index 0000000000..4cb14be0ca
--- /dev/null
+++ b/tests/document_loaders/test_office_format_extraction.py
@@ -0,0 +1,204 @@
+"""Regression tests for office-document upload support (issue #4414).
+
+ODT/DOC/DOCX/PPT/PPTX/XLS/XLSX were advertised as supported but failed at
+runtime because their ``unstructured`` parser dependencies (python-docx,
+python-pptx, openpyxl) were never installed. These tests assert that:
+
+1. With the parser dependencies installed, the office extensions are
+ registered and text actually round-trips through the upload extraction
+ path (``extract_text_from_bytes``).
+2. The registry is *honest*: a format is advertised only when its real
+ runtime dependency is importable, so a missing dependency hides the
+ format instead of accepting an upload that later fails silently.
+"""
+
+import importlib
+import shutil
+import tempfile
+from pathlib import Path
+
+import pytest
+
+REGISTRY_MODULE = "local_deep_research.document_loaders.loader_registry"
+
+
+def _reload(module_name):
+ return importlib.reload(importlib.import_module(module_name))
+
+
+class TestOfficeExtensionsRegistered:
+ """Office formats are advertised when their parser deps are present."""
+
+ def test_modern_office_formats_registered(self):
+ # Modern office formats work with pip-only parser deps (no system
+ # binaries), so they are registered whenever those deps are present.
+ pytest.importorskip("docx")
+ pytest.importorskip("pptx")
+ pytest.importorskip("openpyxl")
+ pytest.importorskip("msoffcrypto")
+
+ from local_deep_research.document_loaders.loader_registry import (
+ LOADER_REGISTRY,
+ )
+
+ for ext in (".docx", ".pptx", ".xlsx"):
+ assert ext in LOADER_REGISTRY, f"{ext} should be registered"
+
+ def test_legacy_binary_formats_gated_on_libreoffice(self):
+ # .doc/.ppt are OLE binary formats that unstructured converts via
+ # LibreOffice; they must be advertised only when soffice is present.
+ import local_deep_research.document_loaders.loader_registry as mod
+
+ for ext in (".doc", ".ppt"):
+ assert (ext in mod.LOADER_REGISTRY) == (
+ mod.HAS_LIBREOFFICE
+ and (mod.HAS_DOCX_DEP if ext == ".doc" else mod.HAS_PPTX_DEP)
+ )
+
+ def test_odt_registered_when_docx_and_pandoc_present(self):
+ pytest.importorskip("docx")
+ pytest.importorskip("pypandoc")
+
+ from local_deep_research.document_loaders.loader_registry import (
+ LOADER_REGISTRY,
+ )
+
+ assert ".odt" in LOADER_REGISTRY
+
+
+class TestOfficeExtractionRoundTrip:
+ """Real files extract text through the public upload path."""
+
+ def test_docx_extraction(self):
+ pytest.importorskip("docx")
+ from docx import Document
+
+ from local_deep_research.document_loaders import (
+ extract_text_from_bytes,
+ )
+
+ marker = "Hello DOCX world, extraction works."
+ with tempfile.NamedTemporaryFile(suffix=".docx") as tmp:
+ doc = Document()
+ doc.add_paragraph(marker)
+ doc.save(tmp.name)
+ content = Path(tmp.name).read_bytes()
+
+ text = extract_text_from_bytes(content, ".docx", "sample.docx")
+ assert text is not None
+ assert marker in text
+
+ def test_xls_extraction(self):
+ # xlwt (a dev-only writer) builds the legacy fixture; the loader reads
+ # it with pandas + xlrd, bypassing the fragile unstructured path.
+ xlwt = pytest.importorskip("xlwt")
+ pytest.importorskip("xlrd")
+
+ from local_deep_research.document_loaders import (
+ extract_text_from_bytes,
+ )
+
+ wb = xlwt.Workbook()
+ ws = wb.add_sheet("Data")
+ for r, (a, b) in enumerate(
+ [("Animal", "Count"), ("Otter", "3"), ("Badger", "7")]
+ ):
+ ws.write(r, 0, a)
+ ws.write(r, 1, b)
+ with tempfile.NamedTemporaryFile(suffix=".xls", delete=False) as tmp:
+ xls_path = tmp.name
+ try:
+ wb.save(xls_path)
+ content = Path(xls_path).read_bytes()
+ finally:
+ Path(xls_path).unlink(missing_ok=True)
+
+ text = extract_text_from_bytes(content, ".xls", "sample.xls")
+ assert text is not None
+ assert "Animal" in text and "Otter" in text and "Badger" in text
+
+ def test_odt_extraction(self):
+ pypandoc = pytest.importorskip("pypandoc")
+ pytest.importorskip("docx")
+ if not shutil.which("pandoc"):
+ try:
+ pypandoc.get_pandoc_version()
+ except OSError:
+ pytest.skip("pandoc binary not available")
+
+ from local_deep_research.document_loaders import (
+ extract_text_from_bytes,
+ )
+
+ marker = "Hello ODT world, extraction works."
+ with tempfile.NamedTemporaryFile(suffix=".odt", delete=False) as tmp:
+ odt_path = tmp.name
+ try:
+ # Generate a genuine ODT (with styles.xml etc.) via pandoc.
+ pypandoc.convert_text(
+ marker, "odt", format="md", outputfile=odt_path
+ )
+ content = Path(odt_path).read_bytes()
+ finally:
+ Path(odt_path).unlink(missing_ok=True)
+
+ text = extract_text_from_bytes(content, ".odt", "sample.odt")
+ assert text is not None
+ assert marker in text
+
+
+class TestHonestDetection:
+ """A missing runtime dependency hides the format from the registry.
+
+ The registry computes its capability flags at import time, so we reload
+ the module with ``importlib.util.find_spec`` patched to report specific
+ modules as absent. Patching ``_module_available`` directly would not
+ survive the reload (it redefines that function from source), so we patch
+ the lower-level ``find_spec`` it calls.
+ """
+
+ @pytest.fixture
+ def restore_module(self):
+ yield
+ _reload(REGISTRY_MODULE)
+
+ @staticmethod
+ def _reload_without(monkeypatch, blocked):
+ import importlib.util as importlib_util
+
+ real_find_spec = importlib_util.find_spec
+
+ def fake_find_spec(name, *args, **kwargs):
+ if name in blocked:
+ return None
+ return real_find_spec(name, *args, **kwargs)
+
+ monkeypatch.setattr(importlib_util, "find_spec", fake_find_spec)
+ return _reload(REGISTRY_MODULE)
+
+ def test_missing_python_docx_hides_word_and_odt(
+ self, monkeypatch, restore_module
+ ):
+ reloaded = self._reload_without(monkeypatch, {"docx"})
+
+ assert reloaded.HAS_DOCX_DEP is False
+ for ext in (".doc", ".docx", ".odt"):
+ assert ext not in reloaded.LOADER_REGISTRY
+
+ def test_missing_pptx_hides_powerpoint(self, monkeypatch, restore_module):
+ reloaded = self._reload_without(monkeypatch, {"pptx"})
+
+ assert reloaded.HAS_PPTX_DEP is False
+ for ext in (".ppt", ".pptx"):
+ assert ext not in reloaded.LOADER_REGISTRY
+
+ def test_missing_ocr_dep_hides_image_formats(
+ self, monkeypatch, restore_module
+ ):
+ reloaded = self._reload_without(
+ monkeypatch, {"pytesseract", "unstructured.pytesseract"}
+ )
+
+ assert reloaded.HAS_OCR_DEP is False
+ for ext in (".png", ".jpg", ".jpeg", ".tiff", ".tif", ".bmp", ".heic"):
+ assert ext not in reloaded.LOADER_REGISTRY
diff --git a/tests/research_library/routes/test_rag_routes.py b/tests/research_library/routes/test_rag_routes.py
index b793c49780..e0bc1c2624 100644
--- a/tests/research_library/routes/test_rag_routes.py
+++ b/tests/research_library/routes/test_rag_routes.py
@@ -1124,8 +1124,8 @@ class TestSupportedFormatsEndpoint:
extensions = get_supported_extensions()
- # Presentation formats should be present
- assert ".ppt" in extensions
+ # Presentation formats should be present. Legacy .ppt is only
+ # registered when LibreOffice is installed, so only .pptx is asserted.
assert ".pptx" in extensions
def test_get_supported_extensions_all_start_with_dot(self):
@@ -1198,16 +1198,50 @@ class TestSupportedFormatsEndpoint:
assert is_extension_supported(".dll") is False
def test_supported_formats_count(self):
- """Test that we have at least 30 supported formats."""
+ """The always-available formats are advertised regardless of env.
+
+ Under honest capability detection (#4414) the registry only
+ advertises a format when its real runtime dependency is present,
+ so the *total* count varies by environment (office docs need
+ python-docx/openpyxl/python-pptx, OCR needs the tesseract binary,
+ etc.). The stable contract is the set of formats whose loaders
+ have NO optional dependency — those are always registered. Assert
+ that guaranteed set is present rather than a magic total that the
+ old "advertise everything" registry happened to reach.
+ """
from local_deep_research.document_loaders import (
get_supported_extensions,
)
- extensions = get_supported_extensions()
+ extensions = set(get_supported_extensions())
+
+ # Formats backed only by hard dependencies / pure-Python loaders —
+ # registered unconditionally (see LOADER_REGISTRY base entries and
+ # the ungated additions in loader_registry.py).
+ always_available = {
+ ".pdf",
+ ".txt",
+ ".md",
+ ".markdown",
+ ".csv",
+ ".html",
+ ".htm",
+ ".xml",
+ ".eml",
+ ".mht",
+ ".mhtml",
+ ".enex",
+ ".tsv",
+ ".json",
+ ".yaml",
+ ".yml",
+ ".ipynb",
+ ".toml",
+ }
- # We expect 30+ formats as documented
- assert len(extensions) >= 30, (
- f"Expected 30+ formats, got {len(extensions)}"
+ missing = always_available - extensions
+ assert not missing, (
+ f"Guaranteed formats missing from registry: {sorted(missing)}"
)
def test_accept_string_generation(self):
diff --git a/tests/research_library/routes/test_rag_routes_coverage.py b/tests/research_library/routes/test_rag_routes_coverage.py
index a97724546f..42f4d53c6b 100644
--- a/tests/research_library/routes/test_rag_routes_coverage.py
+++ b/tests/research_library/routes/test_rag_routes_coverage.py
@@ -1510,6 +1510,77 @@ class TestUploadToCollection:
assert rdata["success"] is True
assert rdata["summary"]["successful"] == 1
+ def test_successful_odt_upload_real_extraction(self, app):
+ """End-to-end: a real ODT uploaded through the HTTP route is parsed by
+ the real extraction stack (issue #4414 regression). Nothing in the
+ document-loaders path is mocked; only the DB and password store are."""
+ pypandoc = pytest.importorskip("pypandoc")
+ pytest.importorskip("docx")
+ try:
+ pypandoc.get_pandoc_version()
+ except OSError:
+ pytest.skip("pandoc binary not available")
+
+ import tempfile
+ from pathlib import Path
+
+ marker = "Otters build dams in the river."
+ with tempfile.NamedTemporaryFile(suffix=".odt", delete=False) as tmp:
+ odt_path = tmp.name
+ try:
+ pypandoc.convert_text(
+ marker, "odt", format="md", outputfile=odt_path
+ )
+ odt_bytes = Path(odt_path).read_bytes()
+ finally:
+ Path(odt_path).unlink(missing_ok=True)
+
+ mock_coll = Mock()
+ mock_coll.id = "coll-1"
+
+ db_session = _make_db_session()
+ call_count = [0]
+
+ def query_side_effect(model):
+ call_count[0] += 1
+ q = _build_mock_query()
+ if call_count[0] == 1:
+ q.first.return_value = mock_coll
+ elif call_count[0] == 2:
+ q.first.return_value = None # No existing doc by hash
+ elif call_count[0] == 3:
+ mock_source = Mock()
+ mock_source.id = "src-1"
+ q.first.return_value = mock_source
+ return q
+
+ db_session.query = Mock(side_effect=query_side_effect)
+
+ mock_password_store = Mock()
+ mock_password_store.get_session_password.return_value = None
+
+ # Note: extract_text_from_bytes / is_extension_supported are NOT
+ # patched here — the real ODT must flow through the real loaders.
+ with _auth_client(
+ app,
+ mock_db_session=db_session,
+ extra_patches=[
+ patch(
+ f"{_DB_PASS}.session_password_store", mock_password_store
+ ),
+ ],
+ ) as (client, ctx):
+ data = {"files": (BytesIO(odt_bytes), "report.odt")}
+ resp = client.post(
+ "/library/api/collections/coll-1/upload",
+ data=data,
+ content_type="multipart/form-data",
+ )
+ assert resp.status_code == 200
+ rdata = resp.get_json()
+ assert rdata["success"] is True
+ assert rdata["summary"]["successful"] == 1
+
def test_upload_existing_doc_not_in_collection(self, app):
mock_coll = Mock()
mock_coll.id = "coll-1"