~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""Hostile-input contracts for ``document_loaders`` and the PDF paths.
These loaders parse *untrusted uploaded bytes*. The properties pinned here
are the ones an attacker gets to probe:
* every advertised extension must turn a zero-byte / truncated / junk file
into a *handled* result, never an exception escaping into the threadpool
worker that runs extraction;
* the ``%PDF`` magic-byte gate must be applied consistently -- this file
shows it is **not**: three code paths disagree about what counts as a PDF;
* extraction must not amplify a small upload into an unbounded allocation on
the single worker (measured on deliberately tiny fixtures -- nothing here
allocates more than a couple of MB);
* encrypted / password-protected documents must be refused cleanly;
* extraction must not depend on the host timezone;
* no loader may shell out to, or evaluate, document content.
Every rejection assertion below is paired with a positive control that shows
a *valid* file of the same type still loads, so a rejection can never pass
because the loader is simply broken for that format.
Fixtures are built in-process and are all under ~40 KB except two bounded
amplification probes; nothing is written to the repo and no large file is
ever generated.
"""
import glob
import io
import json
import os
import re
import shutil
import tempfile
import time
from pathlib import Path
import pytest
from loguru import logger
from local_deep_research.document_loaders import (
extract_text_from_bytes,
get_supported_extensions,
is_extension_supported,
load_from_bytes,
)
from local_deep_research.document_loaders import loader_registry
from local_deep_research.security.file_upload_validator import (
FileUploadValidator,
)
from local_deep_research.web.services.pdf_extraction_service import (
PDFExtractionService,
)
LOADER_PKG_DIR = Path(loader_registry.__file__).parent
TEMP_DIR = Path(tempfile.gettempdir())
UPLOAD_TEMP_GLOB = str(TEMP_DIR / "ldr_upload_*")
# --------------------------------------------------------------------------
# Fixtures: minimal *valid* documents (positive controls) + hostile variants
# --------------------------------------------------------------------------
def _minimal_pdf(text: bytes = b"Hello LDR") -> bytes:
"""Build a 1-page PDF with a real text stream, hand-rolled.
Hand-rolled rather than library-generated so the byte layout (and the
exact ``%PDF`` header offset) is under this file's control -- the
magic-byte tests below depend on it.
"""
stream = b"BT /F1 24 Tf 72 700 Td (" + text + b") Tj ET"
objs = [
b"<< /Type /Catalog /Pages 2 0 R >>",
b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>",
b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] "
b"/Contents 4 0 R /Resources << /Font << /F1 5 0 R >> >> >>",
b"<< /Length "
+ str(len(stream)).encode()
+ b" >>\nstream\n"
+ stream
+ b"\nendstream",
b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>",
]
out = bytearray(b"%PDF-1.4\n")
offsets = []
for index, obj in enumerate(objs, 1):
offsets.append(len(out))
out += str(index).encode() + b" 0 obj\n" + obj + b"\nendobj\n"
xref_at = len(out)
out += b"xref\n0 " + str(len(objs) + 1).encode() + b"\n"
out += b"0000000000 65535 f \n"
for offset in offsets:
out += ("%010d 00000 n \n" % offset).encode()
out += (
b"trailer\n<< /Size "
+ str(len(objs) + 1).encode()
+ b" /Root 1 0 R >>\n"
)
out += b"startxref\n" + str(xref_at).encode() + b"\n%%EOF\n"
return bytes(out)
VALID_PDF = _minimal_pdf()
# A polyglot: valid PDF body, but the ``%PDF`` header is NOT at offset 0.
# pypdf reads it anyway (it logs "invalid pdf header" and recovers via
# startxref); a byte-prefix magic check rejects it. That disagreement is the
# subject of the magic-byte section below.
POLYGLOT_PDF = b"<html><script>alert(1)</script>\n" + VALID_PDF
VALID_NOTEBOOK = json.dumps(
{
"cells": [
{
"cell_type": "markdown",
"source": ["hello ipynb"],
"metadata": {},
}
],
"metadata": {},
"nbformat": 4,
"nbformat_minor": 5,
}
).encode()
VALID_MHTML = (
b"MIME-Version: 1.0\r\nContent-Type: text/html\r\n\r\n"
b"<html><body><p>hello mhtml</p></body></html>\r\n"
)
VALID_EML = (
b"From: [email protected]\r\nTo: [email protected]\r\nSubject: s\r\n"
b"Content-Type: text/plain\r\n\r\nhello eml\r\n"
)
# Extensions for which a minimal valid document can be constructed with no
# external binary (pandoc / LibreOffice / tesseract). The pandoc-backed
# formats (.rst/.org/.rtf/.epub/.odt) and .xls are deliberately excluded --
# building a valid fixture for them needs a binary or a writer library that
# is not a dependency, and their presence in the registry is already gated.
VALID_BY_EXTENSION: dict[str, bytes] = {
".txt": b"hello txt\n",
".md": b"# hello md\n",
".markdown": b"# hello markdown\n",
".csv": b"a,b\n1,hello csv\n",
".tsv": b"a\tb\n1\thello tsv\n",
".json": b'{"t": "hello json"}',
".yaml": b"t: hello yaml\n",
".yml": b"t: hello yml\n",
".xml": b"<?xml version='1.0'?><r><a>hello xml</a></r>",
".html": b"<html><body><p>hello html</p></body></html>",
".htm": b"<html><body><p>hello htm</p></body></html>",
".eml": VALID_EML,
".ipynb": VALID_NOTEBOOK,
".mhtml": VALID_MHTML,
".mht": VALID_MHTML,
".pdf": VALID_PDF,
}
def _valid_docx() -> bytes:
docx = pytest.importorskip("docx")
document = docx.Document()
document.add_paragraph("hello docx")
buf = io.BytesIO()
document.save(buf)
return buf.getvalue()
def _valid_xlsx() -> bytes:
openpyxl = pytest.importorskip("openpyxl")
workbook = openpyxl.Workbook()
workbook.active.append(["hello xlsx"])
buf = io.BytesIO()
workbook.save(buf)
return buf.getvalue()
def _valid_pptx() -> bytes:
pptx = pytest.importorskip("pptx")
presentation = pptx.Presentation()
slide = presentation.slides.add_slide(presentation.slide_layouts[5])
slide.shapes.title.text = "hello pptx"
buf = io.BytesIO()
presentation.save(buf)
return buf.getvalue()
def _encrypted_pdf() -> bytes:
pypdf = pytest.importorskip("pypdf")
reader = pypdf.PdfReader(io.BytesIO(VALID_PDF))
writer = pypdf.PdfWriter()
for page in reader.pages:
writer.add_page(page)
writer.encrypt("correct horse battery staple")
buf = io.BytesIO()
writer.write(buf)
return buf.getvalue()
@pytest.fixture(autouse=True)
def _quiet_loguru():
"""Loaders log every rejection via ``logger.exception``.
Extraction of ~30 hostile fixtures otherwise emits megabytes of
tracebacks and makes a real failure impossible to find. Scoped with
disable/enable rather than ``logger.remove()`` so sibling test modules
in the same process keep their handlers.
"""
logger.disable("local_deep_research")
try:
yield
finally:
logger.enable("local_deep_research")
# --------------------------------------------------------------------------
# 1. Malformed / truncated / zero-byte input for every advertised format
# --------------------------------------------------------------------------
class TestHostileBytesAreHandledNotCrashes:
"""``extract_text_from_bytes`` is the route-facing contract.
``rag.upload_to_collection`` calls it inside a threadpool worker and
treats a falsy result as "could not extract text". Anything it lets
escape becomes an unhandled exception in that worker.
"""
@pytest.mark.parametrize("extension", sorted(get_supported_extensions()))
def test_zero_byte_file_never_escapes_as_an_exception(self, extension):
result = extract_text_from_bytes(b"", extension, f"empty{extension}")
# None (extraction failed, cleanly) or a str (possibly empty) are
# the only two shapes the caller knows how to handle.
assert result is None or isinstance(result, str), (
f"{extension}: zero-byte upload produced {type(result)!r}"
)
@pytest.mark.parametrize("extension", sorted(get_supported_extensions()))
def test_binary_junk_never_escapes_as_an_exception(self, extension):
junk = b"\x00\x01\x02\xff\xfe not a real file \x00" * 4
result = extract_text_from_bytes(junk, extension, f"junk{extension}")
assert result is None or isinstance(result, str), (
f"{extension}: junk upload produced {type(result)!r}"
)
@pytest.mark.parametrize(
("extension", "content"), sorted(VALID_BY_EXTENSION.items())
)
def test_positive_control_valid_document_extracts_text(
self, extension, content
):
"""Pairs with the two rejection tests above.
Without this, "zero bytes was handled" would also pass if the loader
were broken for every input of that type.
"""
text = extract_text_from_bytes(content, extension, f"ok{extension}")
assert text, f"{extension}: valid document extracted nothing"
assert "hello" in text.lower(), (
f"{extension}: extracted text lost the payload -- {text[:120]!r}"
)
class TestZeroByteAndTruncatedPdf:
def test_zero_byte_pdf_raises_empty_file_error(self):
pypdf_errors = pytest.importorskip("pypdf.errors")
with pytest.raises(pypdf_errors.EmptyFileError):
load_from_bytes(b"", ".pdf", "empty.pdf")
assert extract_text_from_bytes(b"", ".pdf", "empty.pdf") is None
def test_truncated_pdf_raises_pdf_stream_error(self):
pypdf_errors = pytest.importorskip("pypdf.errors")
truncated = VALID_PDF[: len(VALID_PDF) // 2]
with pytest.raises(pypdf_errors.PdfStreamError):
load_from_bytes(truncated, ".pdf", "cut.pdf")
assert extract_text_from_bytes(truncated, ".pdf", "cut.pdf") is None
def test_positive_control_intact_pdf_extracts(self):
assert extract_text_from_bytes(VALID_PDF, ".pdf", "ok.pdf") == (
"Hello LDR"
)
class TestStructuredTextParseFailuresAreSoftFailures:
"""JSON/YAML fall back to raw content rather than raising."""
def test_malformed_json_yields_raw_content_with_parse_error_flag(self):
broken = b'{"a": 1,,,'
documents = load_from_bytes(broken, ".json", "broken.json")
assert len(documents) == 1
assert documents[0].metadata["parse_error"] is True
assert documents[0].page_content == broken.decode()
def test_positive_control_valid_json_extracts_paths_and_values(self):
documents = load_from_bytes(b'{"t": "hello json"}', ".json", "ok.json")
assert documents[0].page_content == "t: hello json"
assert "parse_error" not in documents[0].metadata
def test_malformed_yaml_yields_raw_content_with_parse_error_flag(self):
broken = b"a: [1, 2\nb: {unclosed\n"
documents = load_from_bytes(broken, ".yaml", "broken.yaml")
assert len(documents) == 1
assert documents[0].metadata["parse_error"] is True
assert documents[0].page_content == broken.decode()
def test_positive_control_valid_yaml_round_trips(self):
documents = load_from_bytes(b"t: hello yaml\n", ".yaml", "ok.yaml")
assert documents[0].page_content == "t: hello yaml\n"
assert "parse_error" not in documents[0].metadata
@pytest.mark.parametrize("extension", [".json", ".yaml", ".yml"])
def test_undecodable_bytes_are_refused_and_handled(self, extension):
# The loaders detect the encoding the way text uploads do, so bytes that
# decode into no text are refused before any parsing happens.
undecodable = b"\xff\xfe\x00\x01not utf-8"
with pytest.raises(ValueError, match="no recognized text encoding"):
load_from_bytes(undecodable, extension, f"bad{extension}")
assert (
extract_text_from_bytes(undecodable, extension, f"bad{extension}")
is None
)
# --------------------------------------------------------------------------
# 2. Extension vs. magic bytes -- the gate is NOT applied on every path
# --------------------------------------------------------------------------
class TestPdfMagicByteGateConsistency:
"""Three PDF paths, three different answers for the same bytes.
* ``FileUploadValidator.validate_mime_type`` (used by
``POST /api/upload/pdf``) requires ``%PDF`` at offset 0.
* ``rag._try_pdf_upgrade`` (collection upload, *existing*-document
branch) requires ``%PDF`` at offset 0.
* The collection upload's *new*-document branch applies no magic-byte
check at all: it derives ``file_type`` from the filename suffix, hands
the bytes to ``extract_text_from_bytes``, and (with
``pdf_storage=database``) stores them as a PDF blob on the strength of
the extension alone.
``POLYGLOT_PDF`` is the wedge: pypdf parses it, the byte-prefix checks
reject it.
"""
def test_validator_rejects_polyglot_on_magic_bytes(self):
ok, error = FileUploadValidator.validate_mime_type(
"x.pdf", POLYGLOT_PDF
)
assert ok is False
assert "signature mismatch" in error
def test_positive_control_validator_accepts_real_pdf(self):
assert FileUploadValidator.validate_mime_type("x.pdf", VALID_PDF) == (
True,
None,
)
def test_structure_check_alone_would_accept_the_polyglot(self):
"""The magic-byte check is the *only* thing rejecting it.
pdfplumber happily parses the polyglot, so removing or weakening
``validate_mime_type`` would silently open this hole -- the deeper
"structure validation" step does not close it.
"""
assert FileUploadValidator.validate_pdf_structure(
"x.pdf", POLYGLOT_PDF
) == (True, None)
def test_extension_only_check_rejects_pdf_bytes_under_another_name(self):
ok, error = FileUploadValidator.validate_mime_type(
"notpdf.txt", VALID_PDF
)
assert ok is False
assert "Only PDF files allowed" in error
def test_collection_extraction_path_accepts_what_the_gate_rejects(self):
"""No magic-byte pre-check exists before ``extract_text_from_bytes``.
Same bytes that ``validate_mime_type`` rejects above extract
successfully here, which is how they reach the ``store_pdf_in_db``
branch in ``rag._upload_to_collection_sync``.
"""
assert (
extract_text_from_bytes(POLYGLOT_PDF, ".pdf", "x.pdf")
== "Hello LDR"
)
def test_pdf_upgrade_helper_does_enforce_the_magic_bytes(self):
"""The *upgrade* branch of the same route disagrees with extraction.
Exercises the real production helper with a stub storage manager;
the helper's only content-based decision is the ``%PDF`` prefix.
"""
from local_deep_research.web.routers.rag import _try_pdf_upgrade
class _StubStorage:
def __init__(self):
self.calls = 0
def upgrade_to_pdf(self, **_kwargs):
self.calls += 1
return True
refused_storage = _StubStorage()
refused = _try_pdf_upgrade(
db_session=None,
document=None,
file_content=POLYGLOT_PDF,
filename="x.pdf",
pdf_storage="database",
pdf_storage_manager=refused_storage,
)
assert refused is False
assert refused_storage.calls == 0, (
"polyglot reached the storage manager despite the %PDF gate"
)
# Positive control: real PDF bytes do pass the same gate.
accepted_storage = _StubStorage()
accepted = _try_pdf_upgrade(
db_session=None,
document=None,
file_content=VALID_PDF,
filename="x.pdf",
pdf_storage="database",
pdf_storage_manager=accepted_storage,
)
assert accepted is True
assert accepted_storage.calls == 1
def test_content_is_never_sniffed_the_suffix_picks_the_loader(self):
"""A PDF renamed ``.txt`` is indexed as its own source text.
Documented so the asymmetry above is read correctly: the collection
route is extension-driven by design (it indexes ~27 formats), which
is exactly why the PDF-specific gate has to be re-applied at every
PDF-specific decision point rather than assumed.
"""
text = extract_text_from_bytes(VALID_PDF, ".txt", "x.txt")
assert text is not None
# Raw PDF *source*, not a rendered page: the object graph and the
# stream delimiters survive, which no PDF parser would ever emit.
assert text.startswith("%PDF-1.4")
assert "/Type /Catalog" in text
assert "endstream" in text
class TestExtensionNormalisationCannotEscapeTheRegistry:
"""``load_from_bytes`` puts the extension into a tempfile suffix."""
@pytest.mark.parametrize(
"extension",
[
"../../../etc/passwd",
"/etc/passwd",
".exe",
"pdf\x00.txt",
"",
".pdf.exe",
],
)
def test_unregistered_extension_rejected_before_tempfile_creation(
self, extension
):
before = set(glob.glob(UPLOAD_TEMP_GLOB))
with pytest.raises(ValueError, match="Unsupported file extension"):
load_from_bytes(b"payload", extension, "f")
assert not is_extension_supported(extension)
after = set(glob.glob(UPLOAD_TEMP_GLOB))
assert after == before, "a temp file was created for a rejected ext"
@pytest.mark.parametrize("extension", [".PDF", "PDF", ".Pdf", "pdf"])
def test_positive_control_case_and_dot_variants_are_normalised(
self, extension
):
"""Pairs with the rejection test: normalisation still works.
These reach the real loader (the PDF text comes back), proving the
ValueError above is a registry decision and not a normalisation
accident that rejects everything.
"""
assert (
extract_text_from_bytes(VALID_PDF, extension, "x.pdf")
== "Hello LDR"
)
def test_temp_files_are_removed_even_when_the_loader_raises(self):
before = set(glob.glob(UPLOAD_TEMP_GLOB))
for _ in range(3):
assert (
extract_text_from_bytes(b"\x00\xff junk", ".pdf", "b.pdf")
is None
)
assert extract_text_from_bytes(b"", ".docx", "b.docx") is None
after = set(glob.glob(UPLOAD_TEMP_GLOB))
assert after - before == set(), (
f"leaked temp files: {sorted(after - before)}"
)
@pytest.mark.parametrize(
("raw_name", "forbidden"),
[
("../../../etc/passwd", "/"),
("..\\..\\windows\\system32", "\\"),
("<script>alert(1)</script>.txt", "<"),
],
)
def test_original_filename_metadata_is_sanitised(self, raw_name, forbidden):
documents = load_from_bytes(b"hello", ".txt", raw_name)
stored = documents[0].metadata["original_filename"]
assert forbidden not in stored, (
f"unsanitised filename reached metadata: {stored!r}"
)
# --------------------------------------------------------------------------
# 3. Resource bounds
# --------------------------------------------------------------------------
class TestDeclaredSizeBoundsExistAndFire:
"""Asserted against *declared* sizes -- nothing large is allocated."""
def test_oversized_declared_content_length_is_rejected(self):
four_gb = 4 * 1024 * 1024 * 1024
ok, error = FileUploadValidator.validate_file_size(four_gb, None)
assert ok is False
assert "too large" in error.lower()
def test_positive_control_small_upload_passes_the_same_check(self):
assert FileUploadValidator.validate_file_size(1024, b"x" * 16) == (
True,
None,
)
def test_configured_caps_are_far_above_a_single_worker_budget(self):
"""Documents the actual ceiling the single worker must survive.
Not a style assertion: ``rag.upload_to_collection`` computes
``MAX_TOTAL_UPLOAD_SIZE = MAX_FILES_PER_UPLOAD * MAX_FILE_SIZE`` from
exactly these two constants and buffers every accepted file into
memory before handing them to the threadpool. If these numbers
change, the amplification measurements below change meaning with
them.
"""
if os.environ.get("LDR_SECURITY_UPLOAD_MAX_FILE_SIZE_MB"):
pytest.skip("per-file cap overridden by env in this environment")
per_file_gb = FileUploadValidator.MAX_FILE_SIZE / (1024**3)
assert per_file_gb == pytest.approx(3.0), (
f"per-file cap moved to {per_file_gb}GB"
)
assert FileUploadValidator.MAX_FILES_PER_REQUEST == 200
# The product is the collection route's request ceiling.
total_gb = (
FileUploadValidator.MAX_FILE_SIZE
* FileUploadValidator.MAX_FILES_PER_REQUEST
) / (1024**3)
assert total_gb == pytest.approx(600.0)
def test_extracted_text_is_never_truncated(self):
"""The only bound on extracted text is the *input* size cap.
``load_from_bytes`` joins every document's ``page_content`` with no
ceiling, so the amplification factors measured below multiply
straight through the 3 GB per-file cap. Demonstrated on a 2 MB
payload -- small enough to be harmless, large enough that any
plausible truncation limit would have fired.
"""
payload = b"A" * 2_000_000
text = extract_text_from_bytes(payload, ".txt", "big.txt")
assert text is not None
assert len(text) == 2_000_000
class TestNestingAndAmplificationBounds:
"""All fixtures below are < 50 KB in and < 3 MB out."""
def test_json_recursion_limit_is_reached_before_the_size_cap(self):
"""``extract_strings_from_json`` recurses with no depth guard.
``json.loads`` accepts a nesting depth that the custom extractor
cannot walk, so a ~2.4 KB upload reliably raises RecursionError
inside the extraction worker. It is *handled* (RecursionError is an
Exception subclass, so ``extract_text_from_bytes`` returns None),
but nothing in the loader bounds the depth itself.
"""
deep = b"[" * 1200 + b'"leaf"' + b"]" * 1200
assert len(deep) < 3000
json.loads(deep) # the stdlib parser is fine with this depth
with pytest.raises(RecursionError):
load_from_bytes(deep, ".json", "deep.json")
assert extract_text_from_bytes(deep, ".json", "deep.json") is None
def test_positive_control_moderately_nested_json_still_loads(self):
shallow = b"[" * 100 + b'"leaf"' + b"]" * 100
text = extract_text_from_bytes(shallow, ".json", "ok.json")
assert text is not None
assert text.endswith(": leaf")
def test_json_path_prefixing_amplifies_superlinearly(self):
"""Each leaf is emitted as ``<full json path>: <value>``.
Path length grows with nesting depth, so output grows as
depth x leaves while input grows as depth + leaves. Measured on
two tiny fixtures; the ratio must be seen to *increase* with depth,
which is what makes it unbounded rather than a constant overhead.
"""
def nested(depth, leaves):
obj = ["x"] * leaves
for _ in range(depth):
obj = [obj]
return json.dumps(obj).encode()
shallow = nested(10, 100)
deep = nested(400, 100)
assert len(shallow) < 1024 and len(deep) < 2048
shallow_out = extract_text_from_bytes(shallow, ".json", "s.json")
deep_out = extract_text_from_bytes(deep, ".json", "d.json")
shallow_ratio = len(shallow_out) / len(shallow)
deep_ratio = len(deep_out) / len(deep)
assert deep_ratio > shallow_ratio * 5, (
f"expected superlinear growth, got {shallow_ratio:.1f} -> "
f"{deep_ratio:.1f}"
)
assert deep_ratio > 50, (
f"a 1.3 KB JSON expanded {deep_ratio:.1f}x with no cap"
)
def test_yaml_deep_nesting_raises_recursion_error_and_is_handled(self):
deep = b"[" * 2000 + b"]" * 2000
with pytest.raises(RecursionError):
load_from_bytes(deep, ".yaml", "deep.yaml")
assert extract_text_from_bytes(deep, ".yaml", "deep.yaml") is None
def test_yaml_alias_bomb_does_not_amplify(self):
"""Good news, pinned so a loader change cannot lose it.
PyYAML's ``safe_load`` shares one object per anchor and ``yaml.dump``
re-emits anchors instead of expanding them, so the classic
billion-laughs shape stays O(input) end to end here. A future switch
to a deep-copying or alias-flattening dump would break this.
"""
def bomb(levels, fan):
lines = ["a0: &a0 [" + ",".join(["'x'"] * fan) + "]"]
for level in range(1, levels):
refs = ",".join([f"*a{level - 1}"] * fan)
lines.append(f"a{level}: &a{level} [{refs}]")
return ("\n".join(lines) + "\n").encode()
payload = bomb(6, 10) # 10**6 nodes if expanded
assert len(payload) < 1024
text = extract_text_from_bytes(payload, ".yaml", "bomb.yaml")
assert text is not None
assert len(text) < len(payload) * 10, (
f"alias expansion amplified {len(text) / len(payload):.1f}x"
)
def test_xml_entity_amplification_is_capped_inside_the_bounded_child(
self, tmp_path
):
"""lxml's amplification cap still refuses the bomb, now in the child.
``.xml`` is parsed by the bounded child process
(``ISOLATED_FINAL_EXTENSIONS``), so the ``XMLSyntaxError`` lxml
raises no longer reaches the caller: the child reports a parser
failure and the caller sees the generic refusal. The first assertion
pins the cap itself on the registered loader class, so this test
still fails if the parser stops refusing the bomb; the second pins
that the refusal crosses the boundary as a plain refusal rather than
a limit, memory, busy or missing-NLTK-data error.
"""
etree = pytest.importorskip("lxml.etree")
from local_deep_research.security.isolated_loader import (
IsolatedLoaderError,
)
def entity_bomb(levels, fan):
entities = ['<!ENTITY e0 "xxxxxxxxxx">']
for level in range(1, levels):
body = f"&e{level - 1};" * fan
entities.append(f'<!ENTITY e{level} "{body}">')
return (
'<?xml version="1.0"?>'
f"<!DOCTYPE r [{''.join(entities)}]>"
f"<r>&e{levels - 1};</r>"
).encode()
payload = entity_bomb(6, 10)
assert len(payload) < 1024
loader_info = loader_registry.get_loader_class_for_extension(".xml")
assert loader_info is not None
loader_class, loader_kwargs = loader_info
bomb_path = tmp_path / "bomb.xml"
bomb_path.write_bytes(payload)
with pytest.raises(etree.XMLSyntaxError, match="(?i)amplification"):
loader_class(str(bomb_path), **loader_kwargs).load()
with pytest.raises(IsolatedLoaderError) as excinfo:
load_from_bytes(payload, ".xml", "bomb.xml")
assert type(excinfo.value) is IsolatedLoaderError
assert "refused the document" in str(excinfo.value)
assert extract_text_from_bytes(payload, ".xml", "bomb.xml") is None
def test_positive_control_ordinary_xml_still_parses(self):
assert (
extract_text_from_bytes(
b"<?xml version='1.0'?><r><a>hello xml</a></r>",
".xml",
"ok.xml",
)
== "hello xml"
)
def test_ooxml_decompression_has_no_output_cap(self):
"""Zip-backed formats decompress with no declared-size ceiling.
Bounded on purpose: a 2 MB payload, which compresses to tens of KB.
The point is the *ratio* and the absence of any check, not the
absolute size -- a real zip bomb reaches far higher ratios with the
same code path.
"""
docx_module = pytest.importorskip("docx")
payload = "A" * 2_000_000
document = docx_module.Document()
document.add_paragraph(payload)
buf = io.BytesIO()
document.save(buf)
compressed = buf.getvalue()
started = time.monotonic()
text = extract_text_from_bytes(compressed, ".docx", "big.docx")
elapsed = time.monotonic() - started
assert text is not None
assert len(text) >= 2_000_000
ratio = len(text) / len(compressed)
assert ratio > 20, (
f"expected decompression amplification, measured {ratio:.1f}x"
)
assert elapsed < 30, "docx extraction unexpectedly slow"
# --------------------------------------------------------------------------
# 4. Encrypted / password-protected documents
# --------------------------------------------------------------------------
class TestEncryptedDocuments:
def test_encrypted_pdf_fails_structure_validation(self):
encrypted = _encrypted_pdf()
# Magic bytes still match -- encryption is invisible at the header.
assert FileUploadValidator.validate_mime_type("e.pdf", encrypted) == (
True,
None,
)
ok, error = FileUploadValidator.validate_pdf_structure(
"e.pdf", encrypted
)
assert ok is False
assert "corrupted" in error.lower()
def test_encrypted_pdf_extraction_service_returns_failure_dict(self):
result = PDFExtractionService.extract_text_and_metadata(
_encrypted_pdf(), "e.pdf"
)
assert result["success"] is False
assert result["text"] == ""
assert result["error"] == "Failed to extract text from PDF"
def test_positive_control_unencrypted_pdf_extracts_with_page_count(self):
result = PDFExtractionService.extract_text_and_metadata(
VALID_PDF, "g.pdf"
)
assert result["success"] is True
assert result["pages"] == 1
assert result["text"] == "Hello LDR"
def test_encrypted_pdf_loader_raises_file_not_decrypted(self):
pypdf_errors = pytest.importorskip("pypdf.errors")
encrypted = _encrypted_pdf()
with pytest.raises(pypdf_errors.FileNotDecryptedError):
load_from_bytes(encrypted, ".pdf", "e.pdf")
assert extract_text_from_bytes(encrypted, ".pdf", "e.pdf") is None
def test_encrypted_xls_is_mapped_to_a_clear_value_error(self, monkeypatch):
"""``XLSLoader`` translates xlrd's encryption failure.
The translation is production logic in ``xls_loader.py``; the
monkeypatch only stands in for a real encrypted BIFF workbook (no
writer for that format is a dependency).
"""
from local_deep_research.document_loaders import xls_loader
def _raise_encrypted(*_args, **_kwargs):
raise ValueError("Workbook is encrypted")
monkeypatch.setattr(xls_loader.pd, "read_excel", _raise_encrypted)
loader = xls_loader.XLSLoader("/nonexistent/book.xls")
with pytest.raises(ValueError, match="encrypted and cannot be read"):
loader.load()
def test_non_encryption_xls_errors_propagate_unchanged(self, monkeypatch):
"""Control for the mapping above -- it must not swallow everything."""
from local_deep_research.document_loaders import xls_loader
class _CorruptWorkbook(RuntimeError):
pass
def _raise_corrupt(*_args, **_kwargs):
raise _CorruptWorkbook("Expected BOF record; found b'\\x00'")
monkeypatch.setattr(xls_loader.pd, "read_excel", _raise_corrupt)
loader = xls_loader.XLSLoader("/nonexistent/book.xls")
with pytest.raises(_CorruptWorkbook):
loader.load()
# --------------------------------------------------------------------------
# 5. Locale / timezone independence
# --------------------------------------------------------------------------
class TestExtractionIsEnvironmentIndependent:
@pytest.mark.parametrize(
("extension", "content"),
[
(".csv", b"name,when,amount\nrow,2024-03-05,1234.5\n"),
(".json", b'{"when": "2024-03-05T13:45:00Z", "amount": 1234.5}'),
(".yaml", b"when: 2024-03-05 13:45:00\namount: 1234.5\n"),
(".eml", VALID_EML),
(".pdf", VALID_PDF),
],
)
def test_output_is_identical_under_two_timezones(
self, extension, content, monkeypatch
):
outputs = []
for timezone_name in ("UTC", "Pacific/Kiritimati"):
monkeypatch.setenv("TZ", timezone_name)
time.tzset()
outputs.append(
extract_text_from_bytes(content, extension, f"f{extension}")
)
monkeypatch.delenv("TZ", raising=False)
time.tzset()
assert outputs[0] is not None, f"{extension}: nothing extracted"
assert outputs[0] == outputs[1], (
f"{extension}: extraction is timezone-dependent\n"
f"UTC: {outputs[0]!r}\nKiritimati: {outputs[1]!r}"
)
def test_spreadsheet_dates_render_identically_across_timezones(
self, monkeypatch
):
openpyxl = pytest.importorskip("openpyxl")
if ".xlsx" not in get_supported_extensions():
pytest.skip(".xlsx is not registered in this environment")
import datetime
workbook = openpyxl.Workbook()
sheet = workbook.active
sheet.append(["date", datetime.datetime(2024, 3, 5, 13, 45)])
sheet.append(["float", 1234.5])
buf = io.BytesIO()
workbook.save(buf)
content = buf.getvalue()
outputs = []
for timezone_name in ("UTC", "Pacific/Kiritimati"):
monkeypatch.setenv("TZ", timezone_name)
time.tzset()
outputs.append(extract_text_from_bytes(content, ".xlsx", "d.xlsx"))
monkeypatch.delenv("TZ", raising=False)
time.tzset()
assert outputs[0] and "2024-03-05" in outputs[0]
assert outputs[0] == outputs[1]
def test_loader_modules_never_read_the_wall_clock_or_locale(self):
"""A static property: no clock/locale input can leak into output."""
offenders = {}
for module_path in sorted(LOADER_PKG_DIR.glob("*.py")):
source = module_path.read_text(encoding="utf-8")
hits = re.findall(
r"\b(?:locale\.\w+|datetime\.now|utcnow|time\.localtime"
r"|strftime|setlocale)\b",
source,
)
if hits:
offenders[module_path.name] = sorted(set(hits))
assert offenders == {}, f"loaders read clock/locale state: {offenders}"
# --------------------------------------------------------------------------
# 6. No loader shells out or evaluates document content
# --------------------------------------------------------------------------
class TestLoadersDoNotExecuteDocumentContent:
def test_no_loader_module_imports_or_calls_an_executor(self):
offenders = {}
pattern = re.compile(
r"\b(?:import\s+subprocess|from\s+subprocess|os\.system"
r"|os\.popen|subprocess\.\w+|\beval\(|\bexec\(|__import__\("
r"|import\s+pickle|pickle\.loads|yaml\.load\(|marshal\.loads)"
)
for module_path in sorted(LOADER_PKG_DIR.glob("*.py")):
source = module_path.read_text(encoding="utf-8")
hits = pattern.findall(source)
if hits:
offenders[module_path.name] = sorted(set(hits))
assert offenders == {}, f"executor usage in loaders: {offenders}"
def test_yaml_loader_uses_safe_load_only(self):
source = (LOADER_PKG_DIR / "yaml_loader.py").read_text(encoding="utf-8")
assert "yaml.safe_load" in source
assert re.search(r"yaml\.load\s*\(", source) is None
assert "yaml.unsafe_load" not in source
assert "yaml.full_load" not in source
@pytest.mark.parametrize(
("extension", "payload", "expected_inert_text"),
[
(
".yaml",
b"!!python/object/apply:os.system ['touch ldr_pwn']\n",
"os.system",
),
(
".yml",
b"!!python/object/apply:subprocess.getoutput ['id']\n",
"subprocess.getoutput",
),
(
".ipynb",
json.dumps(
{
"cells": [
{
"cell_type": "code",
"source": [
"import os\n",
"os.system('touch ldr_pwn')\n",
],
"outputs": [],
"metadata": {},
}
],
"metadata": {},
"nbformat": 4,
"nbformat_minor": 5,
}
).encode(),
"os.system",
),
],
)
def test_executable_payloads_are_treated_as_inert_text(
self, extension, payload, expected_inert_text, monkeypatch, tmp_path
):
# Run from an empty cwd so a relative `touch ldr_pwn` would land
# somewhere observable rather than in the repo.
monkeypatch.chdir(tmp_path)
result = extract_text_from_bytes(payload, extension, f"p{extension}")
assert list(tmp_path.iterdir()) == [], (
f"{extension}: payload created files: "
f"{[p.name for p in tmp_path.iterdir()]}"
)
assert not (TEMP_DIR / "ldr_pwn").exists()
# The payload survives as *text*, which is the correct outcome: it
# was indexed, not interpreted.
assert result is not None
assert expected_inert_text in result
def test_python_object_tag_is_a_parse_error_not_a_construction(self):
payload = b"!!python/object/apply:os.system ['echo hi']\n"
documents = load_from_bytes(payload, ".yaml", "p.yaml")
assert len(documents) == 1
# safe_load refuses the tag -> the loader's YAMLError branch fires.
assert documents[0].metadata["parse_error"] is True
assert documents[0].page_content == payload.decode()
def test_positive_control_benign_yaml_is_actually_constructed(self):
"""Pairs with the tag test: safe_load is not refusing everything."""
documents = load_from_bytes(b"t: hello yaml\n", ".yaml", "g.yaml")
assert "parse_error" not in documents[0].metadata
assert documents[0].page_content == "t: hello yaml\n"
def test_external_xml_entities_do_not_leak_local_files(self):
payload = (
b'<?xml version="1.0"?>'
b'<!DOCTYPE r [<!ENTITY x SYSTEM "file:///etc/passwd">]>'
b"<r>&x;</r>"
)
text = extract_text_from_bytes(payload, ".xml", "xxe.xml")
assert text is not None
assert "root:" not in text
assert "/bin/" not in text
def test_shell_out_formats_are_only_advertised_with_their_binary(self):
"""``.doc``/``.ppt`` conversion runs LibreOffice on the upload.
``unstructured`` shells out to ``soffice`` for the legacy OLE
formats. The registry must not advertise them unless that binary is
actually present, or an upload is accepted and then handed to a
converter that does not exist.
"""
has_libreoffice = bool(
shutil.which("soffice") or shutil.which("libreoffice")
)
registered = set(get_supported_extensions())
shell_out_formats = {".doc", ".ppt"} & registered
if shell_out_formats and not has_libreoffice:
pytest.fail(
f"{sorted(shell_out_formats)} advertised without a "
"LibreOffice binary on PATH"
)
assert loader_registry.HAS_LIBREOFFICE == has_libreoffice
# --------------------------------------------------------------------------
# 7. Advertised-but-unparseable formats (issue #4414 class)
# --------------------------------------------------------------------------
class TestAdvertisedFormatsHaveTheirParser:
"""``get_supported_extensions`` is user-facing.
``rag.py`` serves it to the UI as the accepted-formats list, and
``is_extension_supported`` gates the upload. An extension listed there
whose parser dependency is missing produces a misleading
"Could not extract text" instead of "unsupported format" -- the exact
failure mode the registry's capability flags were added to prevent.
"""
@pytest.mark.parametrize(
("extension", "content"),
# TOML and ENEX have dedicated extraction controls below.
sorted(VALID_BY_EXTENSION.items()),
)
def test_no_advertised_format_fails_on_a_missing_parser_dependency(
self, extension, content
):
if extension not in get_supported_extensions():
pytest.skip(f"{extension} is not registered in this environment")
try:
documents = load_from_bytes(content, extension, f"g{extension}")
except (ImportError, ModuleNotFoundError) as exc:
pytest.fail(
f"{extension} is advertised by get_supported_extensions() "
f"but its parser dependency is missing: {exc}"
)
assert documents, f"{extension}: advertised but extracted nothing"
@pytest.mark.parametrize(
("extension", "builder"),
[
(".docx", _valid_docx),
(".xlsx", _valid_xlsx),
(".pptx", _valid_pptx),
],
)
def test_ooxml_formats_are_gated_on_their_real_runtime_dependency(
self, extension, builder
):
if extension not in get_supported_extensions():
pytest.skip(f"{extension} is not registered in this environment")
documents = load_from_bytes(builder(), extension, f"g{extension}")
assert documents
assert "hello" in documents[0].page_content.lower()
def test_advertised_toml_can_actually_be_parsed(self):
assert ".toml" in get_supported_extensions()
documents = load_from_bytes(b'a = "hello toml"\n', ".toml", "g.toml")
assert documents
assert "hello toml" in documents[0].page_content
def test_advertised_enex_can_actually_be_parsed(self):
assert ".enex" in get_supported_extensions()
enex = (
b'<?xml version="1.0" encoding="UTF-8"?>'
b'<en-export export-date="20240305T134500Z">'
b"<note><title>t</title>"
b"<content><![CDATA[<en-note>hello enex</en-note>]]></content>"
b"<created>20240305T134500Z</created></note></en-export>"
)
documents = load_from_bytes(enex, ".enex", "g.enex")
assert documents
assert "hello enex" in documents[0].page_content