Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

4.0 KB
Raw Blame
"""Measure reference/text growth and assert the complete exported documents."""

from concurrent.futures import ThreadPoolExecutor
from dataclasses import replace
import io
import zipfile

import pytest

from local_deep_research.exporters.quarto_exporter import QuartoExporter
from .corpus import Report, perform, prepare, today, verify


@pytest.mark.parametrize("mode", ["document", "zip"])
@pytest.mark.parametrize("curve", ["references", "text"])
def test_export_growth_preserves_complete_document_and_bibliography(
    quarto_workload, entry_probe, mode, curve
):
    for size in (
        quarto_workload.settings["size"],
        quarto_workload.settings["size"] * 2,
    ):
        report = Report(size, 4) if curve == "references" else Report(4, size)
        bound = 3 if mode == "zip" else 2

        def setup():
            entry_probe.update(parses=0, parsed_characters=0)
            return prepare(report)

        def check(state, output):
            verify(state, output, report, mode)
            assert entry_probe["parses"] <= bound
            assert entry_probe["parsed_characters"] <= bound * len(
                report.content
            )
            return dict(
                references=report.references,
                paragraphs=report.paragraphs,
                **entry_probe,
            )

        quarto_workload.measure(
            f"{curve}_{mode}",
            size,
            setup,
            lambda state: perform(state[0], report, mode),
            check,
            units=len(report.content),
            unit="input_character",
            parameters=dict(
                references=report.references,
                paragraphs=report.paragraphs,
                mode=mode,
                instrumentation=True,
                corpus_sha256=report.sha256,
                input_characters=len(report.content),
                input_bytes=len(report.content.encode("utf-8")),
            ),
        )


@pytest.mark.parametrize("mode", ["document", "zip"])
def test_reused_exporter_observes_changed_reports(mode):
    exporter = QuartoExporter()
    outputs = []
    for report in [Report(2, 1), Report(3, 2), Report(2, 1)]:
        state = exporter, today(), report.content
        output = perform(exporter, report, mode)
        verify(state, output, report, mode)
        outputs.append((state, output, report))
    for state, output, report in outputs:
        verify(state, output, report, mode)


def test_shared_exporter_keeps_concurrent_report_results_separate():
    exporter = QuartoExporter()
    reports = [Report(2, 1), Report(3, 2)]
    started = today()
    with ThreadPoolExecutor(max_workers=2) as pool:
        outputs = list(pool.map(lambda r: perform(exporter, r, "zip"), reports))
    for report, output in zip(reports, outputs, strict=True):
        verify((exporter, started, report.content), output, report, "zip")


@pytest.mark.parametrize("fault", ["document", "bibliography", "member"])
def test_archive_oracle_rejects_incomplete_or_changed_results(fault):
    report = Report(2, 1)
    state = prepare(report)
    output = perform(state[0], report, "zip")
    verify(state, output, report, "zip")
    contents = io.BytesIO()
    with zipfile.ZipFile(io.BytesIO(output.content)) as original:
        with zipfile.ZipFile(contents, "w", zipfile.ZIP_DEFLATED) as changed:
            for name in original.namelist():
                if name == "references.bib" and fault == "member":
                    continue
                value = original.read(name).decode("utf-8")
                if fault == "document" and name.endswith(".qmd"):
                    value = value.replace("[@ref1]", "[@ref9999]")
                if fault == "bibliography" and name.endswith(".bib"):
                    value = value.replace("/source/1", "/source/9999")
                changed.writestr(name, value)
    corrupted = replace(output, content=contents.getvalue())
    with pytest.raises(AssertionError, match="Quarto .* mismatch"):
        verify(state, corrupted, report, "zip")