Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

50.9 KB
Raw Blame
"""Resource-boundary regressions for uploaded document parsing."""

import io
import json
import os
import signal
import subprocess
import sys
import threading
import time
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path

import pytest
from langchain_core.documents import Document

from local_deep_research.document_loaders import bytes_loader
from local_deep_research.security import isolated_loader, isolated_worker


def test_child_with_no_output_is_killed_at_deadline():
    started = time.monotonic()
    with pytest.raises(isolated_loader.IsolatedLoaderError, match="timed out"):
        isolated_loader._run_capped(
            [sys.executable, "-c", "import time; time.sleep(30)"], 0.1
        )
    assert time.monotonic() - started < 5


def test_spawn_failure_does_not_fall_back_to_unbounded_parse(monkeypatch):
    def unavailable(*args, **kwargs):
        raise OSError("process creation unavailable")

    monkeypatch.setattr(isolated_loader.subprocess, "Popen", unavailable)
    with pytest.raises(
        isolated_loader.IsolatedLoaderError, match="could not start"
    ):
        isolated_loader._run_capped([sys.executable, "-c", "pass"], 1)


def test_invalid_process_configuration_is_a_parser_failure(monkeypatch):
    def unavailable(*args, **kwargs):
        raise ValueError("invalid process configuration")

    monkeypatch.setattr(isolated_loader.subprocess, "Popen", unavailable)
    with pytest.raises(
        isolated_loader.IsolatedLoaderError, match="could not start"
    ):
        isolated_loader._run_capped([sys.executable, "-c", "pass"], 1)


def test_parent_rejects_unbounded_child_output(monkeypatch):
    monkeypatch.setattr(isolated_loader, "MAX_PROTOCOL_BYTES", 1_024)
    with pytest.raises(
        isolated_loader.IsolatedLoaderError, match="output exceeded"
    ):
        isolated_loader._run_capped(
            [sys.executable, "-c", "import os; os.write(1, b'A' * 2048)"], 5
        )


def test_timeout_returns_while_another_writer_keeps_pipe_open(monkeypatch):
    # Simulate an inherited writer that outlives the parser. The test owns
    # that writer and closes it in finally, so even the broken implementation
    # is cleaned up. No detached process is needed.
    read_fd, write_fd = os.pipe()
    popen = subprocess.Popen
    children = []

    def shared_pipe(*args, **kwargs):
        kwargs["stdout"] = write_fd
        child = popen(*args, **kwargs)
        child.stdout = os.fdopen(
            read_fd, "rb", buffering=kwargs.get("bufsize", -1)
        )
        children.append(child)
        return child

    monkeypatch.setattr(isolated_loader.subprocess, "Popen", shared_pipe)
    with ThreadPoolExecutor(max_workers=1) as executor:
        future = executor.submit(
            isolated_loader._run_capped,
            [sys.executable, "-c", "pass"],
            0.1,
        )
        try:
            with pytest.raises(
                isolated_loader.IsolatedLoaderError, match="timed out"
            ):
                # Cleanup must finish before the test releases the remaining
                # writer, not wait for EOF while closing a buffered reader.
                future.result(timeout=5)
        finally:
            os.close(write_fd)
            if not children:
                os.close(read_fd)

    assert len(children) == 1
    assert children[0].poll() is not None
    assert children[0].stdout.closed


def test_small_over_limit_output_is_refused_before_eof(monkeypatch):
    monkeypatch.setattr(isolated_loader, "MAX_PROTOCOL_BYTES", 1_024)
    # More than the output limit, less than one buffered read. The child
    # keeps its pipe open, as a parser waiting during shutdown might do.
    with pytest.raises(
        isolated_loader.IsolatedLoaderError, match="output exceeded"
    ):
        isolated_loader._run_capped(
            [
                sys.executable,
                "-c",
                "import os, time; os.write(1, b'A' * 2048); time.sleep(30)",
            ],
            2,
        )


def test_nonblocking_setup_failure_kills_and_closes_child(monkeypatch):
    popen = subprocess.Popen
    children = []

    def record_child(*args, **kwargs):
        child = popen(*args, **kwargs)
        children.append(child)
        return child

    def unavailable(*args):
        raise OSError("nonblocking pipes unavailable")

    monkeypatch.setattr(isolated_loader.subprocess, "Popen", record_child)
    monkeypatch.setattr(isolated_loader.os, "set_blocking", unavailable)
    with pytest.raises(
        isolated_loader.IsolatedLoaderError, match="pipe.*configured"
    ):
        isolated_loader._run_capped(
            [sys.executable, "-c", "import time; time.sleep(30)"], 1
        )
    assert len(children) == 1
    assert children[0].poll() is not None
    assert children[0].stdout.closed


def test_exact_output_limit_is_returned_whole(monkeypatch):
    monkeypatch.setattr(isolated_loader, "MAX_PROTOCOL_BYTES", 100_000)
    expected = b"a" * 50_000 + b"b" * 50_000
    output = isolated_loader._run_capped(
        [
            sys.executable,
            "-c",
            "import sys; "
            "sys.stdout.buffer.write(b'a' * 50000); "
            "sys.stdout.buffer.flush(); "
            "sys.stdout.buffer.write(b'b' * 50000)",
        ],
        5,
    )
    assert output == expected


def test_parser_capacity_rejects_burst_before_spawn(monkeypatch):
    slots = threading.BoundedSemaphore(1)
    slots.acquire()
    monkeypatch.setattr(isolated_loader, "_parser_slots", slots)

    def forbidden(*args, **kwargs):
        raise AssertionError("saturated parser launched a child")

    monkeypatch.setattr(isolated_loader, "_run_capped", forbidden)
    with pytest.raises(isolated_loader.IsolatedLoaderError, match="capacity"):
        isolated_loader.load_documents_isolated("unused", ".docx")


def test_parser_capacity_is_held_until_result_validation_finishes(monkeypatch):
    slots = threading.BoundedSemaphore(1)
    monkeypatch.setattr(isolated_loader, "_parser_slots", slots)
    output = (
        b'{"status":"ok","documents":[{"page_content":"text","metadata":{}}]}'
    )
    monkeypatch.setattr(isolated_loader, "_run_capped", lambda *args: output)
    entered = threading.Event()
    release = threading.Event()
    main_thread = threading.current_thread()
    loads = json.loads

    def held_decode(data, *args, **kwargs):
        if data == output and threading.current_thread() is not main_thread:
            entered.set()
            assert release.wait(5), "test did not release result validation"
        return loads(data, *args, **kwargs)

    monkeypatch.setattr(isolated_loader.json, "loads", held_decode)
    with ThreadPoolExecutor(max_workers=1) as executor:
        future = executor.submit(
            isolated_loader.load_documents_isolated, "unused", ".docx"
        )
        try:
            assert entered.wait(5), "result validation did not start"
            with pytest.raises(
                isolated_loader.IsolatedLoaderError, match="capacity"
            ):
                isolated_loader.load_documents_isolated("unused", ".docx")
        finally:
            release.set()
        assert future.result(timeout=5)[0].page_content == "text"

    # The completed parse returns its slot for the next caller.
    assert (
        isolated_loader.load_documents_isolated("unused", ".docx")[
            0
        ].page_content
        == "text"
    )


@pytest.mark.parametrize(
    "invalid_output",
    [b"not JSON", b'{"status":"ok","documents":[{"page_content":42}]}'],
)
def test_invalid_result_releases_parser_capacity(monkeypatch, invalid_output):
    monkeypatch.setattr(
        isolated_loader, "_parser_slots", threading.BoundedSemaphore(1)
    )
    outputs = iter([invalid_output, b'{"status":"ok","documents":[]}'])
    monkeypatch.setattr(
        isolated_loader, "_run_capped", lambda *args: next(outputs)
    )

    with pytest.raises(isolated_loader.IsolatedLoaderError, match="invalid"):
        isolated_loader.load_documents_isolated("unused", ".docx")
    assert isolated_loader.load_documents_isolated("unused", ".docx") == []


@pytest.mark.skipif(sys.platform != "linux", reason="Linux RLIMIT_DATA only")
def test_worker_data_limit_refuses_new_allocation():
    script = """
from pathlib import Path
from local_deep_research.security.isolated_worker import _cap_data_memory
status = Path('/proc/self/status').read_text()
current = int(next(line.split()[1] for line in status.splitlines()
                   if line.startswith('VmData:'))) * 1024
_cap_data_memory(current + 32 * 1024 * 1024)
try:
    data = bytearray(128 * 1024 * 1024)
except MemoryError:
    print('bounded')
else:
    raise SystemExit('allocation escaped data limit')
"""
    result = subprocess.run(
        [sys.executable, "-c", script],
        check=False,
        capture_output=True,
        text=True,
        timeout=10,
    )
    assert result.returncode == 0, result.stderr
    assert result.stdout.strip() == "bounded"


def test_worker_refuses_too_many_documents_before_writing(monkeypatch):
    class Loader:
        def __init__(self, path, **kwargs):
            self.path = path

        def load(self):
            return [Document(page_content="text") for _ in range(3)]

    from local_deep_research.document_loaders import loader_registry

    monkeypatch.setattr(isolated_loader, "MAX_DOCUMENTS", 2)
    monkeypatch.setattr(
        loader_registry,
        "get_loader_class_for_extension",
        lambda extension: (Loader, {}),
    )
    assert isolated_worker._result("unused", ".docx") == {
        "status": "error",
        "kind": "limit",
    }


def test_worker_counts_metadata_in_its_output_budget(monkeypatch):
    class Loader:
        def __init__(self, path, **kwargs):
            pass

        def load(self):
            return [
                Document(page_content="small", metadata={"blob": "A" * 2_000})
            ]

    from local_deep_research.document_loaders import loader_registry

    monkeypatch.setattr(isolated_loader, "MAX_PROTOCOL_BYTES", 1_024)
    monkeypatch.setattr(
        loader_registry,
        "get_loader_class_for_extension",
        lambda extension: (Loader, {}),
    )
    assert isolated_worker._result("unused", ".docx") == {
        "status": "error",
        "kind": "limit",
    }


def test_xlsx_worker_reapplies_drawing_guard_before_loader(monkeypatch):
    from local_deep_research.document_loaders import (
        loader_registry,
        openpyxl_hardening,
    )

    monkeypatch.setattr(
        openpyxl_hardening, "disable_openpyxl_drawing_reads", lambda: False
    )

    def forbidden(extension):
        raise AssertionError("spreadsheet loader ran without drawing guard")

    monkeypatch.setattr(
        loader_registry, "get_loader_class_for_extension", forbidden
    )
    assert isolated_worker._result("unused", ".xlsx") == {
        "status": "error",
        "kind": "guard",
    }


def test_in_process_text_upload_has_no_child_text_limit():
    # The character limit belongs to the bounded child parse. Plain text,
    # CSV and the other formats parsed in this process are joined whole,
    # as before the child existed: CSV rows become "header: value" lines,
    # so a CSV upload of a few MB can pass this many characters.
    text = "a" * (isolated_loader.MAX_TEXT_CHARS + 1)
    extracted = bytes_loader.extract_text_from_bytes(
        text.encode(), ".txt", "large.txt"
    )
    assert extracted is not None
    assert len(extracted) == len(text)
    assert extracted == text


def test_legacy_conversion_and_zip_guard_precede_isolated_loader(monkeypatch):
    events: list[object] = []
    monkeypatch.setattr(
        bytes_loader, "is_extension_supported", lambda ext: True
    )
    monkeypatch.setattr(
        bytes_loader,
        "convert_legacy_office_bytes",
        lambda content, ext: events.append(("convert", ext)) or b"converted",
    )
    monkeypatch.setattr(
        bytes_loader,
        "validate_zip_container",
        lambda content, ext: events.append(("guard", content, ext)),
    )
    monkeypatch.setattr(
        bytes_loader,
        "get_loader_class_for_extension",
        lambda ext: (object, {}),
    )

    def isolated(path, ext):
        assert Path(path).read_bytes() == b"converted"
        events.append(("isolated", ext))
        return [Document(page_content="safe", metadata={})]

    monkeypatch.setattr(bytes_loader, "load_documents_isolated", isolated)
    documents = bytes_loader.load_from_bytes(b"ole", ".doc", "sample.doc")
    assert documents[0].page_content == "safe"
    assert events == [
        ("convert", ".doc"),
        ("guard", b"converted", ".docx"),
        ("isolated", ".docx"),
    ]


# Real-worker and limit pins. Each limit below is enforced on both sides of
# the pipe or in the real child, so a test that patched the child out could
# not notice either half going missing.

DOCX_TEXT = "Bounded parser round trip."
MHTML = (
    b"MIME-Version: 1.0\r\nContent-Type: text/html\r\n\r\n"
    b"<html><body><p>hello mhtml</p></body></html>\r\n"
)


def _docx_bytes() -> bytes:
    docx = pytest.importorskip("docx")
    document = docx.Document()
    document.add_paragraph(DOCX_TEXT)
    buffer = io.BytesIO()
    document.save(buffer)
    return buffer.getvalue()


def _worker_output(*texts: str) -> bytes:
    return json.dumps(
        {
            "status": "ok",
            "documents": [
                {"page_content": text, "metadata": {}} for text in texts
            ],
        }
    ).encode()


def _stub_registry(monkeypatch, loader_class):
    from local_deep_research.document_loaders import loader_registry

    monkeypatch.setattr(
        loader_registry,
        "get_loader_class_for_extension",
        lambda extension: (loader_class, {}),
    )


def _worker_argv(path: str, *, cpu_seconds: int | None = None) -> str:
    # A script line giving the worker the argv load_documents_isolated
    # passes. Each script runs as a direct child of this test process, so
    # this process's ID is its parent's.
    if cpu_seconds is None:
        cpu_seconds = isolated_loader.CHILD_CPU_SECONDS
    return (
        f"sys.argv = ['worker', {path!r}, '.docx', "
        f"{str(isolated_loader.MAX_CHILD_DATA_BYTES)!r}, "
        f"{str(cpu_seconds)!r}, {str(os.getpid())!r}]"
    )


def test_real_docx_round_trips_through_the_real_worker():
    # Nothing is patched: the ZIP guard, the routing decision, the real child
    # and its registry loader, the pipe protocol and parent validation.
    documents = bytes_loader.load_from_bytes(
        _docx_bytes(), ".docx", "round-trip.docx"
    )
    assert [doc.page_content for doc in documents] == [DOCX_TEXT]
    assert documents[0].metadata["original_filename"] == "round-trip.docx"


def test_real_worker_parses_without_importing_the_ml_stack(tmp_path):
    # langchain_text_splitters imports sentence-transformers, transformers
    # and torch. The worker never splits text, so no parse should pay for
    # them in time or memory.
    source = tmp_path / "plain.docx"
    source.write_bytes(_docx_bytes())
    report = tmp_path / "loaded.json"
    script = "\n".join(
        [
            "import json, sys",
            "from local_deep_research.security import isolated_worker",
            _worker_argv(str(source)),
            "code = isolated_worker.main()",
            "names = ('torch', 'transformers', 'sentence_transformers')",
            "loaded = [n for n in names if sys.modules.get(n) is not None]",
            f"with open({str(report)!r}, 'w') as handle:",
            "    handle.write(json.dumps(loaded))",
            "raise SystemExit(code)",
        ]
    )
    result = subprocess.run(
        [sys.executable, "-c", script],
        check=False,
        capture_output=True,
        timeout=150,
    )
    assert result.returncode == 0, result.stderr[-2000:]
    payload = json.loads(result.stdout)
    assert payload["status"] == "ok", payload
    assert [r["page_content"] for r in payload["documents"]] == [DOCX_TEXT]
    assert json.loads(report.read_text()) == []


@pytest.mark.skipif(sys.platform != "linux", reason="Linux RLIMIT_DATA only")
@pytest.mark.parametrize("headroom_mb", [16, 96], ids=["bounded", "healthy"])
def test_real_worker_has_live_data_limit(headroom_mb):
    # Exercise main() and its real kernel limit, replacing only parser work
    # with a predictable allocation. Native parser imports can wrap memory
    # failures in unrelated exception types, so they cannot prove which cap
    # stopped the child. Real DOCX parsing is covered independently above.
    script = "\n".join(
        [
            "import sys",
            "from contextlib import nullcontext",
            "from pathlib import Path",
            "from local_deep_research.security import isolated_worker",
            "status = Path('/proc/self/status').read_text()",
            "current = int(next(line.split()[1] for line in status.splitlines()",
            "                   if line.startswith('VmData:'))) * 1024",
            "def parse(path, extension):",
            "    data = bytearray(64 * 1024 * 1024)",
            "    return {'status': 'ok', 'documents': [",
            "        {'page_content': str(len(data)), 'metadata': {}}]}",
            "isolated_worker._result = parse",
            "isolated_worker._nltk_regex_budget = lambda ext: nullcontext()",
            _worker_argv("unused"),
            f"sys.argv[3] = str(current + {headroom_mb} * 1024 * 1024)",
            "raise SystemExit(isolated_worker.main())",
        ]
    )
    result = subprocess.run(
        [sys.executable, "-c", script],
        check=False,
        capture_output=True,
        timeout=30,
    )
    assert result.returncode == 0, result.stderr[-2000:]
    if headroom_mb == 16:
        assert json.loads(result.stdout) == {
            "status": "error",
            "kind": "memory",
        }
        with pytest.raises(isolated_loader.ParserMemoryLimitError):
            isolated_loader._documents_from_output(result.stdout)
    else:
        documents = isolated_loader._documents_from_output(result.stdout)
        assert [doc.page_content for doc in documents] == [
            str(64 * 1024 * 1024)
        ]


def test_worker_reports_memory_exhaustion_as_its_own_kind(tmp_path):
    # A MemoryError from the loader is the data limit, not an unreadable
    # document ("parser") or another limit ("limit"), and main() reports it
    # with a result encoded before the limit applied.
    script = "\n".join(
        [
            "import os, sys",
            "from local_deep_research.security import isolated_worker",
            "def parse(path, extension):",
            "    raise MemoryError",
            "isolated_worker._result = parse",
            _worker_argv("unused"),
            "raise SystemExit(isolated_worker.main())",
        ]
    )
    result = subprocess.run(
        [sys.executable, "-c", script],
        check=False,
        capture_output=True,
        timeout=60,
    )
    assert result.returncode == 0, result.stderr[-2000:]
    assert json.loads(result.stdout) == {"status": "error", "kind": "memory"}
    with pytest.raises(isolated_loader.ParserMemoryLimitError):
        isolated_loader._documents_from_output(result.stdout)


def test_worker_lets_a_loader_memory_error_reach_main(monkeypatch):
    class Loader:
        def __init__(self, path, **kwargs):
            pass

        def load(self):
            raise MemoryError

    _stub_registry(monkeypatch, Loader)
    with pytest.raises(MemoryError):
        isolated_worker._result("unused", ".docx")


@pytest.mark.parametrize(
    "output",
    [
        b"[" * 200_000 + b"]" * 200_000,
        (
            b'{"status":"ok","documents":[{"page_content":"x","metadata":'
            b'{"deep":' + b"[" * 200_000 + b"]" * 200_000 + b"}}]}"
        ),
    ],
    ids=["top-level", "inside-metadata"],
)
def test_parent_refuses_deeply_nested_output(output):
    # The structure scan refuses these before json.loads would recurse;
    # the depth-only and RecursionError cases are pinned separately below.
    assert len(output) < isolated_loader.MAX_PROTOCOL_BYTES
    with pytest.raises(isolated_loader.IsolatedLoaderError, match="invalid"):
        isolated_loader._documents_from_output(output)


def test_result_depth_is_bounded_before_json_decode(monkeypatch):
    # One level past the depth limit, and far under the collection and
    # token caps, so only the depth check can refuse it.
    depth = isolated_loader._MAX_JSON_DEPTH + 1
    output = b"[" * depth + b"]" * depth
    assert depth < 2 + 2 * isolated_loader.MAX_DOCUMENTS

    def unexpected_decode(*args, **kwargs):
        raise AssertionError("over-deep structure reached JSON decoding")

    monkeypatch.setattr(isolated_loader.json, "loads", unexpected_decode)
    with pytest.raises(
        isolated_loader.IsolatedLoaderError, match="invalid documents"
    ):
        isolated_loader._documents_from_output(output)


def test_json_recursion_error_is_a_parser_failure(monkeypatch):
    # The depth scan normally stops nesting long before the decoder's own
    # limit, so reach the RecursionError branch directly.
    def too_deep(*args, **kwargs):
        raise RecursionError("maximum recursion depth exceeded")

    monkeypatch.setattr(isolated_loader.json, "loads", too_deep)
    with pytest.raises(
        isolated_loader.IsolatedLoaderError, match="invalid output"
    ) as excinfo:
        isolated_loader._documents_from_output(b'{"status":"ok"}')
    assert isinstance(excinfo.value.__cause__, RecursionError)


@pytest.mark.parametrize(
    "output",
    [
        b"[[],[],[],[],[],[],[],[]]",
        b'{"a":0,"b":0,"c":0,"d":0,"e":0,"f":0,"g":0,"h":0}',
    ],
    ids=["collections", "fields"],
)
def test_result_structure_is_bounded_before_json_decode(monkeypatch, output):
    monkeypatch.setattr(isolated_loader, "MAX_DOCUMENTS", 1)
    monkeypatch.setattr(isolated_loader, "MAX_METADATA_FIELDS", 0)

    def unexpected_decode(*args, **kwargs):
        raise AssertionError("over-limit structure reached JSON decoding")

    monkeypatch.setattr(isolated_loader.json, "loads", unexpected_decode)
    with pytest.raises(isolated_loader.IsolatedLoaderError, match="invalid"):
        isolated_loader._documents_from_output(output)


def test_result_scan_respects_quoted_and_escaped_syntax():
    # An odd number of escaped quotes ends the string early for a scan
    # that ignores backslashes, exposing "]}" as structure. A string ending
    # in an escaped backslash keeps a scan that treats every quote after a
    # backslash as escaped inside the string, so later quoted brackets
    # flip to structure. Either mistake refuses this valid output.
    texts = [
        'Quoted "text", backslash \\ and braces [{]}:,',
        'one quote " then ]}',
        "ends in a backslash \\",
        "[{]}",
    ]
    documents = isolated_loader._documents_from_output(_worker_output(*texts))
    assert [document.page_content for document in documents] == texts


@pytest.mark.parametrize("encoding", ["utf-16-le", "utf-32-le"])
def test_result_requires_the_workers_utf8_encoding(encoding):
    output = '{"status":"ok","documents":[]}'.encode(encoding)
    with pytest.raises(isolated_loader.IsolatedLoaderError, match="invalid"):
        isolated_loader._documents_from_output(output)


def test_json_value_error_is_a_parser_failure(monkeypatch):
    def invalid_integer(*args, **kwargs):
        raise ValueError("integer conversion limit")

    monkeypatch.setattr(isolated_loader.json, "loads", invalid_integer)
    with pytest.raises(isolated_loader.IsolatedLoaderError, match="invalid"):
        isolated_loader._documents_from_output(b'{"status":"ok"}')


def test_metadata_field_budget_is_cumulative_in_parent_and_worker(monkeypatch):
    monkeypatch.setattr(isolated_loader, "MAX_METADATA_FIELDS", 2)
    records = [
        Document(page_content="a", metadata={"first": "1"}),
        Document(page_content="b", metadata={"second": "2"}),
    ]

    class Loader:
        def __init__(self, path, **kwargs):
            pass

        def load(self):
            return records

    _stub_registry(monkeypatch, Loader)
    result = isolated_worker._result("unused", ".docx")
    assert result["status"] == "ok"
    assert (
        len(isolated_loader._documents_from_output(json.dumps(result).encode()))
        == 2
    )
    records.append(Document(page_content="c", metadata={"third": "3"}))
    with pytest.raises(isolated_loader.IsolatedLoaderError, match="exceeded"):
        isolated_loader._documents_from_output(
            json.dumps(
                {
                    "status": "ok",
                    "documents": [record.model_dump() for record in records],
                }
            ).encode()
        )
    assert isolated_worker._result("unused", ".docx") == {
        "status": "error",
        "kind": "limit",
    }


@pytest.mark.parametrize(
    ("message", "kind", "exception"),
    [
        ("No module named docx", "ImportError", ImportError),
        (
            "library: cannot allocate memory",
            "memory",
            isolated_loader.ParserMemoryLimitError,
        ),
        (
            "library: failed to map segment from shared object",
            "resource",
            isolated_loader.ParserResourceError,
        ),
    ],
)
def test_worker_distinguishes_dependency_and_mapping_failures(
    monkeypatch, message, kind, exception
):
    class Loader:
        def __init__(self, path, **kwargs):
            pass

        def load(self):
            raise ImportError(message)

    _stub_registry(monkeypatch, Loader)
    result = isolated_worker._result("unused", ".docx")
    assert result == {"status": "error", "kind": kind}
    with pytest.raises(exception):
        isolated_loader._documents_from_output(json.dumps(result).encode())


@pytest.mark.parametrize(
    "value",
    [1, 1.5, None, True, ["a"], {"a": "b"}],
    ids=["int", "float", "null", "bool", "list", "object"],
)
def test_parent_refuses_non_string_metadata_values(value):
    def output(metadata_value):
        return json.dumps(
            {
                "status": "ok",
                "documents": [
                    {"page_content": "x", "metadata": {"k": metadata_value}}
                ],
            }
        ).encode()

    documents = isolated_loader._documents_from_output(output("string"))
    assert documents[0].metadata == {"k": "string"}
    with pytest.raises(
        isolated_loader.IsolatedLoaderError, match="invalid documents"
    ):
        isolated_loader._documents_from_output(output(value))


def test_documented_limits_match_the_code():
    # docs/library-and-rag.md and the changelog state these numbers; a
    # change to either side must change the other.
    assert isolated_loader.PARSE_TIMEOUT_SECONDS == 90
    assert isolated_loader.MAX_TEXT_CHARS == 10_000_000
    assert isolated_loader.MAX_PROTOCOL_BYTES == 40_000_000  # 40 MB
    assert isolated_loader.MAX_DOCUMENTS == 10_000
    assert isolated_loader.MAX_CHILD_DATA_BYTES == 3 * 1024**3  # 3 GiB
    assert isolated_loader.CHILD_CPU_SECONDS == 120
    assert isolated_loader.MAX_CONCURRENT_PARSERS == 2
    root = Path(__file__).resolve().parents[2]
    for doc in (
        root / "docs" / "library-and-rag.md",
        root / "changelog.d" / "7147.security.md",
    ):
        text = " ".join(doc.read_text().split())
        for claim in (
            "90-second deadline",
            "10-million-character text limit",
            "10,000-document limit",
            "40 MB",
            "soft 3 GiB data-memory limit",
            "120",
        ):
            assert claim in text, f"{doc.name} no longer says {claim!r}"


def test_parent_refuses_cumulative_text_over_limit(monkeypatch):
    monkeypatch.setattr(isolated_loader, "MAX_TEXT_CHARS", 10)
    # The limit counts text across documents as they are joined, and
    # exactly the limit passes: "abcd\n\nefgh" is ten characters.
    documents = isolated_loader._documents_from_output(
        _worker_output("abcd", "efgh")
    )
    assert [doc.page_content for doc in documents] == ["abcd", "efgh"]
    with pytest.raises(
        isolated_loader.IsolatedLoaderError, match="output exceeded"
    ):
        isolated_loader._documents_from_output(_worker_output("abcd", "efghi"))


def test_parent_counts_join_separators_against_the_text_limit(monkeypatch):
    # extract_text_from_bytes no longer caps its join, so the child's limit
    # must include the separators: "abcd\n\nef" is eight characters, and an
    # empty part adds none.
    monkeypatch.setattr(isolated_loader, "MAX_TEXT_CHARS", 8)
    documents = isolated_loader._documents_from_output(
        _worker_output("abcd", "", "ef")
    )
    assert [doc.page_content for doc in documents] == ["abcd", "", "ef"]
    with pytest.raises(
        isolated_loader.IsolatedLoaderError, match="output exceeded"
    ):
        isolated_loader._documents_from_output(_worker_output("abcd", "efg"))


def test_oversized_bounded_parse_is_still_refused(monkeypatch):
    # Real limits, through the upload entry point. Two halves whose join is
    # two characters over the limit are refused; at exactly the limit the
    # joined text is returned.
    half = isolated_loader.MAX_TEXT_CHARS // 2
    outputs = iter(
        [
            _worker_output("a" * half, "b" * half),
            _worker_output("a" * half, "b" * (half - 2)),
        ]
    )
    monkeypatch.setattr(
        isolated_loader, "_run_capped", lambda *args: next(outputs)
    )
    over = bytes_loader.extract_text_from_bytes(
        _docx_bytes(), ".docx", "over.docx"
    )
    assert over is None
    exact = bytes_loader.extract_text_from_bytes(
        _docx_bytes(), ".docx", "exact.docx"
    )
    assert exact is not None
    assert len(exact) == isolated_loader.MAX_TEXT_CHARS


def test_parent_refuses_more_than_max_documents(monkeypatch):
    monkeypatch.setattr(isolated_loader, "MAX_DOCUMENTS", 2)
    output = _worker_output("a", "b")
    assert len(isolated_loader._documents_from_output(output)) == 2
    with pytest.raises(
        isolated_loader.IsolatedLoaderError, match="invalid documents"
    ):
        isolated_loader._documents_from_output(_worker_output("a", "b", "c"))


def test_worker_dependency_error_maps_to_importerror_at_the_parent(
    monkeypatch,
):
    class Loader:
        def __init__(self, path, **kwargs):
            pass

        def load(self):
            raise ModuleNotFoundError("No module named 'docx'")

    _stub_registry(monkeypatch, Loader)
    result = isolated_worker._result("unused", ".docx")
    assert result == {"status": "error", "kind": "ImportError"}
    # load_from_bytes logs an ImportError as a missing parser dependency,
    # not as a document the parser refused.
    with pytest.raises(ImportError, match="dependency unavailable"):
        isolated_loader._documents_from_output(json.dumps(result).encode())


def test_worker_refuses_text_over_its_limit(monkeypatch):
    # The worker counts the joined text as the parent does: "abcd\n\nef" is
    # exactly the limit, the empty part adds no separator, and one more
    # character is over it.
    texts = ["abcd", "", "ef"]

    class Loader:
        def __init__(self, path, **kwargs):
            pass

        def load(self):
            return [Document(page_content=text) for text in texts]

    _stub_registry(monkeypatch, Loader)
    monkeypatch.setattr(isolated_loader, "MAX_TEXT_CHARS", 8)
    assert isolated_worker._result("unused", ".docx")["status"] == "ok"
    texts[-1] = "efg"
    assert isolated_worker._result("unused", ".docx") == {
        "status": "error",
        "kind": "limit",
    }


def test_worker_protocol_survives_native_stdout_writes():
    # Native parser code can write straight to fd 1. Only the JSON result
    # may reach the protocol pipe.
    script = "\n".join(
        [
            "import os, sys",
            "from local_deep_research.security import isolated_worker",
            "def noisy(path, extension):",
            "    print('python stdout noise', flush=True)",
            "    os.write(1, b'native stdout noise')",
            "    return {'status': 'ok', 'documents': []}",
            "isolated_worker._result = noisy",
            _worker_argv("unused"),
            "raise SystemExit(isolated_worker.main())",
        ]
    )
    result = subprocess.run(
        [sys.executable, "-c", script],
        check=False,
        capture_output=True,
        timeout=60,
    )
    assert result.returncode == 0, result.stderr[-2000:]
    assert json.loads(result.stdout) == {"status": "ok", "documents": []}


def _marked_process_alive(pid: int, marker: bytes) -> bool:
    """Whether ``pid`` is still the marked helper and not yet a zombie."""
    proc = Path(f"/proc/{pid}")
    try:
        if marker not in (proc / "cmdline").read_bytes():
            return False
        state = (proc / "stat").read_text().rsplit(")", 1)[1].split()[0]
    except OSError:
        return False
    return state not in {"Z", "X"}


@pytest.mark.skipif(
    sys.platform != "linux", reason="reads /proc for the helper's state"
)
def test_timeout_kills_the_parsers_descendants(tmp_path):
    # Parsers can start converter or OCR helpers. A timeout must stop the
    # child's whole process group, not only the child.
    pid_file = tmp_path / "helper.pid"
    marker = "time.sleep(61)"
    script = "\n".join(
        [
            "import subprocess, sys, time",
            "helper = subprocess.Popen("
            f"[sys.executable, '-c', 'import time; {marker}'])",
            f"with open({str(pid_file)!r}, 'w') as handle:",
            "    handle.write(str(helper.pid))",
            "time.sleep(60)",
        ]
    )
    with pytest.raises(isolated_loader.IsolatedLoaderError, match="timed out"):
        isolated_loader._run_capped([sys.executable, "-c", script], 3)
    pid = int(pid_file.read_text())
    try:
        deadline = time.monotonic() + 5
        while (
            _marked_process_alive(pid, marker.encode())
            and time.monotonic() < deadline
        ):
            time.sleep(0.05)
        assert not _marked_process_alive(pid, marker.encode())
    finally:
        if _marked_process_alive(pid, marker.encode()):
            os.kill(pid, signal.SIGKILL)


def test_parser_child_gets_its_limits_and_this_process_id(monkeypatch):
    # The child cannot rely on its parent's deadline if the parent dies, so
    # the parent passes the child's own CPU-time limit, a little above that
    # deadline, and its process ID, which the child checks.
    commands = []

    def record(command, timeout_s):
        commands.append(command)
        return _worker_output()

    monkeypatch.setattr(isolated_loader, "_run_capped", record)
    isolated_loader.load_documents_isolated("unused", ".docx")
    (command,) = commands
    assert command[command.index("-m") + 2 :] == [
        "unused",
        ".docx",
        str(isolated_loader.MAX_CHILD_DATA_BYTES),
        str(isolated_loader.CHILD_CPU_SECONDS),
        str(os.getpid()),
    ]
    assert (
        isolated_loader.PARSE_TIMEOUT_SECONDS
        < isolated_loader.CHILD_CPU_SECONDS
        <= 2 * isolated_loader.PARSE_TIMEOUT_SECONDS
    )


@pytest.mark.skipif(os.name != "posix", reason="RLIMIT_CPU is POSIX-only")
def test_parser_child_ends_a_parse_past_its_cpu_time_limit():
    # The real main() applies the CPU-time limit from its argv before the
    # parse runs. A parse that keeps a CPU busy past it gets SIGXCPU, with
    # no parent needed to stop it. One second keeps the test short.
    script = "\n".join(
        [
            "import sys",
            "from local_deep_research.security import isolated_worker",
            "def spin(path, extension):",
            "    while True:",
            "        pass",
            "isolated_worker._result = spin",
            _worker_argv("unused", cpu_seconds=1),
            "raise SystemExit(isolated_worker.main())",
        ]
    )
    result = subprocess.run(
        [sys.executable, "-c", script],
        check=False,
        capture_output=True,
        timeout=60,
    )
    assert result.returncode == -signal.SIGXCPU, result.stderr[-2000:]


@pytest.mark.skipif(
    sys.platform != "linux", reason="PR_SET_PDEATHSIG and /proc are Linux-only"
)
def test_parser_child_is_killed_when_its_parent_dies(tmp_path):
    # A server that crashed or was killed no longer stops its parses at the
    # deadline, and after a restart its parser slots do not count them. A
    # stand-in parent starts the real worker main() the way the server does,
    # passing its own process ID, and is then killed mid-parse.
    pid_file = tmp_path / "worker.pid"
    marker = "parse that outlives its parent"
    worker = "\n".join(
        [
            "import os, sys, time",
            "from local_deep_research.security import isolated_worker",
            "def parse(path, extension):",
            f"    with open({str(pid_file)!r} + '.tmp', 'w') as handle:",
            "        handle.write(str(os.getpid()))",
            f"    os.replace({str(pid_file)!r} + '.tmp', {str(pid_file)!r})",
            "    time.sleep(60)",
            "    return {'status': 'ok', 'documents': []}",
            "isolated_worker._result = parse",
            f"sys.argv = ['worker', {marker!r}, '.docx', "
            f"{str(isolated_loader.MAX_CHILD_DATA_BYTES)!r}, "
            f"{str(isolated_loader.CHILD_CPU_SECONDS)!r}, sys.argv[1]]",
            "raise SystemExit(isolated_worker.main())",
        ]
    )
    parent = subprocess.Popen(
        [
            sys.executable,
            "-c",
            "import os, subprocess, sys, time\n"
            "subprocess.Popen("
            f"[sys.executable, '-c', {worker!r}, str(os.getpid())])\n"
            "time.sleep(60)\n",
        ]
    )
    pid = None
    try:
        deadline = time.monotonic() + 30
        while not pid_file.exists() and time.monotonic() < deadline:
            time.sleep(0.05)
        assert pid_file.exists(), "the worker never started its parse"
        pid = int(pid_file.read_text())
        assert _marked_process_alive(pid, marker.encode())
        parent.kill()
        parent.wait(timeout=5)
        deadline = time.monotonic() + 5
        while (
            _marked_process_alive(pid, marker.encode())
            and time.monotonic() < deadline
        ):
            time.sleep(0.05)
        assert not _marked_process_alive(pid, marker.encode())
    finally:
        if parent.poll() is None:
            parent.kill()
            parent.wait(timeout=5)
        if pid is not None and _marked_process_alive(pid, marker.encode()):
            os.kill(pid, signal.SIGKILL)


@pytest.mark.skipif(
    sys.platform != "linux", reason="the check accompanies PR_SET_PDEATHSIG"
)
def test_parser_child_does_not_parse_once_its_parent_has_exited(tmp_path):
    # A parent that exits before the child asks for the parent-death signal
    # never triggers it; the child then has another parent. It must refuse
    # instead of parsing for a server that is gone. Here the ID it is given
    # is its own, which is never its parent's.
    parsed = tmp_path / "parsed"
    script = "\n".join(
        [
            "import os, sys",
            "from local_deep_research.security import isolated_worker",
            "def parse(path, extension):",
            f"    open({str(parsed)!r}, 'w').close()",
            "    return {'status': 'ok', 'documents': []}",
            "isolated_worker._result = parse",
            "sys.argv = ['worker', 'unused', '.docx', "
            f"{str(isolated_loader.MAX_CHILD_DATA_BYTES)!r}, "
            f"{str(isolated_loader.CHILD_CPU_SECONDS)!r}, str(os.getpid())]",
            "raise SystemExit(isolated_worker.main())",
        ]
    )
    result = subprocess.run(
        [sys.executable, "-c", script],
        check=False,
        capture_output=True,
        timeout=60,
    )
    assert result.returncode == 0, result.stderr[-2000:]
    assert json.loads(result.stdout) == {"status": "error", "kind": "limit"}
    assert not parsed.exists()


def test_parser_child_sends_no_unstructured_usage_request(monkeypatch):
    # Importing unstructured.logger sends a usage request to
    # packages.unstructured.io unless SCARF_NO_ANALYTICS or DO_NOT_TRACK is
    # exactly "true", and every parse imports it in a fresh child. Start
    # from a server environment whose opt-out does not work ("1" is not
    # "true"). The child's requests.get only records, so nothing is sent
    # even when the opt-out is missing.
    monkeypatch.setenv("SCARF_NO_ANALYTICS", "1")
    monkeypatch.delenv("DO_NOT_TRACK", raising=False)
    script = "\n".join(
        [
            "import json, os, requests",
            "calls = []",
            "requests.get = lambda *args, **kwargs: calls.append(args)",
            "import unstructured.logger",
            "result = {'calls': len(calls),",
            "          'opt_out': os.environ.get('SCARF_NO_ANALYTICS')}",
            "os.write(1, json.dumps(result).encode())",
        ]
    )
    output = isolated_loader._run_capped([sys.executable, "-c", script], 60)
    assert json.loads(output) == {"calls": 0, "opt_out": "true"}


def _hide_nltk_data(directory: Path) -> list[str]:
    # Script lines that leave NLTK only an empty data directory and make
    # nltk.download record its calls, so nothing is fetched in any case.
    directory.mkdir()
    return [
        "import nltk",
        f"nltk.data.path[:] = [{str(directory)!r}]",
        "downloads = []",
        "nltk.download = lambda *args, **kwargs: downloads.append(args)",
    ]


def test_parser_child_never_downloads_nltk_data(monkeypatch, tmp_path):
    # Importing unstructured downloads NLTK data that is missing, and every
    # parse imports it in a fresh child, where that download would run under
    # the parse deadline and outside any egress policy. Start from a server
    # environment that asks for the download.
    monkeypatch.setenv("AUTO_DOWNLOAD_NLTK", "true")
    script = "\n".join(
        [
            "import json, os",
            *_hide_nltk_data(tmp_path / "nltk_data"),
            "import unstructured.nlp.tokenize",
            "result = {'downloads': len(downloads),",
            "          'auto': os.environ.get('AUTO_DOWNLOAD_NLTK')}",
            "os.write(1, json.dumps(result).encode())",
        ]
    )
    output = isolated_loader._run_capped([sys.executable, "-c", script], 60)
    assert json.loads(output) == {"downloads": 0, "auto": "false"}


def test_real_parse_without_nltk_data_reports_the_data_missing(tmp_path):
    # The real worker and registry loader in the production child
    # environment, with no NLTK data installed: a Word paragraph's text
    # classification needs it, and the parse must name that cause instead
    # of downloading the data or calling the document unreadable.
    source = tmp_path / "paragraph.docx"
    source.write_bytes(_docx_bytes())
    report = tmp_path / "downloads.json"
    script = "\n".join(
        [
            "import json, sys",
            *_hide_nltk_data(tmp_path / "nltk_data"),
            "from local_deep_research.security import isolated_worker",
            _worker_argv(str(source)),
            "code = isolated_worker.main()",
            f"with open({str(report)!r}, 'w') as handle:",
            "    handle.write(json.dumps(len(downloads)))",
            "raise SystemExit(code)",
        ]
    )
    output = isolated_loader._run_capped([sys.executable, "-c", script], 90)
    assert json.loads(report.read_text()) == 0
    with pytest.raises(isolated_loader.ParserDataMissingError):
        isolated_loader._documents_from_output(output)


@pytest.mark.parametrize(
    ("error", "kind"),
    [
        (LookupError("Resource 'punkt_tab' not found."), "nltk_data"),
        (
            LookupError("Resource 'averaged_perceptron_tagger_eng' not found."),
            "nltk_data",
        ),
        (LookupError("unknown encoding: punctuation"), "parser"),
        (KeyError("punkt_tab"), "parser"),
    ],
    ids=["tokenizer-data", "tagger-data", "other-lookup", "key-error"],
)
def test_worker_names_missing_nltk_data_only_for_nltks_error(
    monkeypatch, error, kind
):
    # nltk.data.find raises a plain LookupError naming the missing package.
    # Other lookups, KeyError included, are ordinary parser failures.
    class FailingLoader:
        def __init__(self, path, **kwargs):
            pass

        def load(self):
            raise error

    _stub_registry(monkeypatch, FailingLoader)
    assert isolated_worker._result("unused", ".docx") == {
        "status": "error",
        "kind": kind,
    }


def test_missing_nltk_data_is_not_reported_as_an_unreadable_file(monkeypatch):
    monkeypatch.setattr(
        isolated_loader,
        "_run_capped",
        lambda *args, **kwargs: b'{"status": "error", "kind": "nltk_data"}',
    )
    # Unlike a refused document, which extracts as None, this says nothing
    # about the file, so the upload route can tell the operator the fix.
    with pytest.raises(
        isolated_loader.ParserDataMissingError, match="NLTK data"
    ):
        bytes_loader.extract_text_from_bytes(
            _docx_bytes(), ".docx", "nodata.docx"
        )


def test_parser_child_starts_with_python_safe_path(monkeypatch):
    commands = []

    def record(command, timeout_s):
        commands.append(command)
        return _worker_output()

    monkeypatch.setattr(isolated_loader, "_run_capped", record)
    isolated_loader.load_documents_isolated("unused", ".docx")
    (command,) = commands
    assert command[: command.index("-m")] == [sys.executable, "-P"]


@pytest.mark.parametrize(
    "entries",
    [["ROOT"], ["ROOT", ""], ["", "ROOT"], ["ROOT", "", ""]],
    ids=["named-only", "trailing-empty", "leading-empty", "two-empty"],
)
def test_parser_child_ignores_modules_in_the_working_directory(
    monkeypatch, tmp_path, entries
):
    # "python -m" puts its working directory first on sys.path, and the
    # child inherits the server's. A package there named like LDR would
    # then run instead of the worker for every parse. An empty PYTHONPATH
    # entry, as in PYTHONPATH=src:, also stands for that directory, even
    # under -P, and puts it before the standard library: a json.py there
    # would run in place of the json module the worker imports.
    workdir = tmp_path / "workdir"
    shadow = workdir / "local_deep_research" / "security"
    shadow.mkdir(parents=True)
    (workdir / "local_deep_research" / "__init__.py").write_text("")
    (shadow / "__init__.py").write_text("")
    (shadow / "isolated_worker.py").write_text(
        "import json, os\n"
        "documents = [{'page_content': 'working directory', 'metadata': {}}]\n"
        "os.write(1, json.dumps({'status': 'ok', 'documents': documents})"
        ".encode())\n"
    )
    imported = tmp_path / "json-imported"
    (workdir / "json.py").write_text(
        f"open({str(imported)!r}, 'w').close()\n"
        "raise ImportError('json.py from the working directory')\n"
    )
    archive = tmp_path / "saved.mht"
    archive.write_bytes(MHTML)
    # The child must still find the real package, wherever this test run
    # found it; a relative PYTHONPATH would resolve under the new directory.
    package_root = str(Path(isolated_loader.__file__).resolve().parents[2])
    monkeypatch.setenv(
        "PYTHONPATH",
        os.pathsep.join(
            package_root if entry == "ROOT" else entry for entry in entries
        ),
    )
    monkeypatch.chdir(workdir)
    documents = isolated_loader.load_documents_isolated(str(archive), ".mht")
    assert [doc.page_content.strip() for doc in documents] == ["hello mhtml"]
    assert not imported.exists()


def test_default_parser_capacity_is_two(monkeypatch):
    # The other capacity tests swap in their own semaphore. This one uses
    # the module's, which the docs promise admits two parses at once.
    def forbidden(*args, **kwargs):
        raise AssertionError("parser launched beyond its capacity")

    monkeypatch.setattr(isolated_loader, "_run_capped", forbidden)
    slots = isolated_loader._parser_slots
    held = 0
    try:
        for _ in range(2):
            assert slots.acquire(blocking=False), "fewer than two slots"
            held += 1
        with pytest.raises(
            isolated_loader.IsolatedLoaderError, match="capacity"
        ):
            isolated_loader.load_documents_isolated("unused", ".docx")
    finally:
        for _ in range(held):
            slots.release()


def test_busy_parser_is_not_reported_as_an_unreadable_file(monkeypatch):
    slots = threading.BoundedSemaphore(1)
    slots.acquire()
    monkeypatch.setattr(isolated_loader, "_parser_slots", slots)

    def forbidden(*args, **kwargs):
        raise AssertionError("saturated parser launched a child")

    monkeypatch.setattr(isolated_loader, "_run_capped", forbidden)
    # A valid upload reaching a saturated boundary says nothing about the
    # file, so the caller must be able to tell it from a refused document.
    with pytest.raises(isolated_loader.ParserBusyError, match="capacity"):
        bytes_loader.extract_text_from_bytes(
            _docx_bytes(), ".docx", "busy.docx"
        )
    slots.release()

    def timed_out(*args, **kwargs):
        raise isolated_loader.IsolatedLoaderError("document parser timed out")

    # Every other boundary failure is still a per-file extraction failure.
    monkeypatch.setattr(isolated_loader, "_run_capped", timed_out)
    assert (
        bytes_loader.extract_text_from_bytes(
            _docx_bytes(), ".docx", "slow.docx"
        )
        is None
    )


@pytest.mark.parametrize("extension", [".mht", ".mhtml"])
def test_web_archives_parse_behind_the_boundary(monkeypatch, extension):
    # A saved web page's HTML parts go through the same parser stack as an
    # .html upload, so they must not be parsed in the server process.
    calls = []

    def isolated(path, ext):
        assert Path(path).read_bytes() == MHTML
        calls.append(ext)
        return [Document(page_content="archived", metadata={})]

    def in_process(*args, **kwargs):
        raise AssertionError("web archive parsed in the server process")

    monkeypatch.setattr(bytes_loader, "load_documents_isolated", isolated)
    monkeypatch.setattr(
        bytes_loader,
        "get_loader_class_for_extension",
        lambda ext: (in_process, {}),
    )
    documents = bytes_loader.load_from_bytes(
        MHTML, extension, f"saved{extension}"
    )
    assert calls == [extension]
    assert documents[0].page_content == "archived"


def test_every_office_and_html_loader_runs_behind_the_boundary():
    # Sweep the registry rather than list extensions: a newly registered
    # extension with one of these parsers must be isolated too. Legacy
    # .doc/.ppt are converted to .docx/.pptx before this decision.
    from local_deep_research.document_loaders import loader_registry
    from local_deep_research.utilities.office_conversion import (
        is_legacy_office_extension,
    )

    bounded = {
        loader_registry.UnstructuredWordDocumentLoader,
        loader_registry.UnstructuredPowerPointLoader,
        loader_registry.UnstructuredExcelLoader,
        loader_registry.UnstructuredHTMLLoader,
        loader_registry.MHTMLLoader,
        loader_registry.XLSLoader,
        loader_registry.UnstructuredMarkdownLoader,
        loader_registry.UnstructuredEmailLoader,
        loader_registry.UnstructuredXMLLoader,
        loader_registry.EverNoteLoader,
    }
    unbounded = sorted(
        ext
        for ext, entry in loader_registry.LOADER_REGISTRY.items()
        if entry["loader_class"] in bounded
        and not is_legacy_office_extension(ext)
        and ext not in isolated_loader.ISOLATED_FINAL_EXTENSIONS
    )
    assert unbounded == []