~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
test(performance): measure Quarto document and archive exports (#8043)
LearningCircuit18911c756657
diff --git a/scripts/ci/performance_pytest.py b/scripts/ci/performance_pytest.pyindex b83deb7527..37ae3b5d16 100644--- a/scripts/ci/performance_pytest.py+++ b/scripts/ci/performance_pytest.py@@ -18,6 +18,7 @@ SUITES = (("tests/performance/content_cache", "cache"),("tests/performance/citation_formatting", "citations"),("tests/performance/citation_metadata", "citenorm"),+ ("tests/performance/quarto_export", "quarto"),("tests/performance/url_processing", "urls"),("tests/performance/text_splitting", "splitting"),("tests/performance/source_rendering", "sources"),@@ -66,6 +67,15 @@ def available_suites(root):suites.append(("tests/metrics/test_token_counting_dispatch_attribution.py", None))+ if "tests/performance/quarto_export" in targets:+ suites.extend(+ (target, None)+ for target in (+ "tests/exporters/test_quarto_exporter.py",+ "tests/text_optimization/test_quarto_yaml_title_escaping.py",+ "tests/text_optimization/test_bibliography_escaping.py",+ )+ )contracts = "tests/ci/test_offline_workload_shards.py"if (root / contracts).is_file():suites.append((contracts, None))diff --git a/tests/ci/standalone/test_performance_pytest.py b/tests/ci/standalone/test_performance_pytest.pyindex d7993f3c71..7e92fbe67e 100644--- a/tests/ci/standalone/test_performance_pytest.py+++ b/tests/ci/standalone/test_performance_pytest.py@@ -21,6 +21,9 @@ class BenchmarkEvidenceTests(unittest.TestCase):self.temporary = tempfile.TemporaryDirectory()self.addCleanup(self.temporary.cleanup)self.root = Path(self.temporary.name)+ # TMPDIR may be inside a real checkout. Keep miniature checkouts+ # independent of its pytest root/configuration and JUnit prefixes.+ (self.root / "pytest.ini").write_text("[pytest]\n", encoding="utf-8")def good_report(self):return {diff --git a/tests/performance/quarto_export/README.md b/tests/performance/quarto_export/README.mdnew file mode 100644index 0000000000..2870a6d58e--- /dev/null+++ b/tests/performance/quarto_export/README.md@@ -0,0 +1,35 @@+# Quarto export measurements++Measure the public document conversion and full in-memory ZIP export while+varying bibliography size and report text length independently. The ZIP timing+includes compression and both archive writes. Corpus construction, exporter+construction, archive decompression and correctness checks are outside timing;+the export logger's sink is disabled. No Quarto/Pandoc process, network or model+is used. This measures serialization, not rendering a document to HTML or PDF.++```bash+pdm run pytest tests/performance/quarto_export -n 0 -q \+ --quarto-profile local --quarto-report /tmp/quarto-export.json+```++CI uses sizes 4/8, two measured samples and one warmup. Local uses 128/256,+seven samples and two warmups. `--quarto-{profile,size,iterations,warmups,report,+max-p95-ms}` overrides `LDR_PERF_QUARTO_*`; `LDR_PERF_PROFILE` is the shared+profile fallback. Timing ceilings are opt-in; CI always asserts correctness+and work bounds. Native thread counts should remain one for comparisons.++Every iteration compares the complete expected document, bibliography, archive+membership, filename and MIME type. The expected text is composed independently+from the corpus parameters; UTC date is checked separately. Both grouped and+individual references must resolve while unlisted numbers remain prose.+Negative controls alter document references, bibliography URLs or archive+membership. Reused and concurrently shared exporters must observe each input+without changing earlier results.++A wrapper around the real bibliography-entry parser counts calls and characters+presented to it, adding small instrumentation overhead. The baseline bounds are+two parses per document and three per ZIP, allowing future reductions. This is+a count of parser invocations, not a claim that the parser touches each character+once. Raw samples, corpus hashes and both loaded implementation hashes are+retained by the shared measurement helper and automatic CI runner. Per-character+p95 divides a whole-export statistic; it is not a character latency percentile.diff --git a/tests/performance/quarto_export/__init__.py b/tests/performance/quarto_export/__init__.pynew file mode 100644index 0000000000..ad832edccd--- /dev/null+++ b/tests/performance/quarto_export/__init__.py@@ -0,0 +1 @@+"""Offline measurements of complete Quarto document and ZIP exports."""diff --git a/tests/performance/quarto_export/conftest.py b/tests/performance/quarto_export/conftest.pynew file mode 100644index 0000000000..1a1bd419e6--- /dev/null+++ b/tests/performance/quarto_export/conftest.py@@ -0,0 +1,54 @@+from types import SimpleNamespace++import pytest++from local_deep_research.exporters import quarto_exporter+from local_deep_research.text_optimization import citation_formatter+from tests.performance._shared.pytest_workload import (+ Workload,+ add_options,+ resolve,+)++PROFILES = {+ "ci": dict(size=4, iterations=2, warmups=1),+ "local": dict(size=128, iterations=7, warmups=2),+}+++def pytest_addoption(parser):+ add_options(parser, "quarto")++[email protected](scope="session")+def quarto_workload(request):+ workload = Workload("quarto", resolve(request.config, "quarto", PROFILES))+ yield workload+ workload.finish(+ modules=(+ "local_deep_research.exporters.quarto_exporter",+ "local_deep_research.text_optimization.citation_formatter",+ )+ )+++def entry_probe(monkeypatch):+ counts = dict(parses=0, parsed_characters=0)+ original = citation_formatter.QuartoExporter._bibliography_entries++ def counted(exporter, content):+ counts["parses"] += 1+ counts["parsed_characters"] += len(content)+ return original(exporter, content)++ def noop(*args, **kwargs):+ return None++ monkeypatch.setattr(+ citation_formatter.QuartoExporter, "_bibliography_entries", counted+ )+ monkeypatch.setattr(+ quarto_exporter, "logger", SimpleNamespace(info=noop, exception=noop)+ )+ return countsdiff --git a/tests/performance/quarto_export/corpus.py b/tests/performance/quarto_export/corpus.pynew file mode 100644index 0000000000..97faea665b--- /dev/null+++ b/tests/performance/quarto_export/corpus.py@@ -0,0 +1,138 @@+"""Plain synthetic reports with complete, independently composed expectations."""++from datetime import UTC, datetime+import hashlib+import io+import re+import zipfile++from local_deep_research.exporters.base import ExportOptions+from local_deep_research.exporters.quarto_exporter import QuartoExporter++TITLE = "Synthetic report"+READER = (+ "markdown-raw_html-raw_tex-raw_attribute-native_divs-native_spans"+ "-fenced_divs-bracketed_spans-header_attributes-link_attributes"+ "-inline_code_attributes-fenced_code_attributes-yaml_metadata_block"+ "-pandoc_title_block"+)+++def today():+ return datetime.now(UTC).date().isoformat()+++def header(date):+ return f'''---+title: "{TITLE}"+author: "Local Deep Research"+date: "{date}"+from: {READER}+format:+ html:+ toc: true+ toc-depth: 3+ number-sections: true+ pdf:+ toc: true+ number-sections: true+ colorlinks: true+bibliography: references.bib+csl: apa.csl+---++'''+++class Report:+ def __init__(self, references, paragraphs):+ body, converted = [], []+ for number in range(paragraphs):+ prefix = f"Synthetic café paragraph {number}: evidence "+ body.append(+ prefix + f"[1] and [{references}, 1]; prose [999999].\n"+ )+ converted.append(+ prefix+ + f"[@ref1] and [@ref{references}, @ref1]; prose [999999].\n"+ )+ sources, converted_sources, bibliography = [], [], []+ for number in range(1, references + 1):+ title = f"Synthetic source {number}"+ url = f"https://example.test/source/{number}"+ continuation = f" {title}\n URL: {url}\n"+ sources.append(f"[{number}]" + continuation)+ converted_sources.append(f"[@ref{number}]" + continuation)+ bibliography.append(+ f"@misc{{ref{number},\n"+ f' title = "{{{title}}}",\n'+ f" url = {{{url}}},\n"+ f' howpublished = "\\url{{{url}}}",\n'+ " year = {2024},\n"+ ' note = "Accessed: \\today"\n'+ "}"+ )+ self.content = (+ f"# {TITLE}\n\n"+ + "".join(body)+ + "\n## Sources\n\n"+ + "\n".join(sources)+ )+ self.converted = (+ f"# {TITLE}\n\n"+ + "".join(converted)+ + "\n## Sources\n\n"+ + "\n".join(converted_sources)+ )+ self.bibliography = "\n\n".join(bibliography)+ self.sha256 = hashlib.sha256(self.content.encode()).hexdigest()+ self.references, self.paragraphs = references, paragraphs++ def expected_document(self, date):+ return (+ header(date)+ + self.converted+ + "\n\n**Bibliography File Required**\n\n"+ "This document requires a `references.bib` file in the same "+ "directory with the following content:\n\n```bibtex\n"+ + self.bibliography+ + "\n```\n"+ )+++def perform(exporter, report, mode):+ if mode == "zip":+ return exporter.export(report.content, ExportOptions(title=TITLE))+ return exporter._legacy.export_to_quarto(report.content, TITLE)+++def prepare(report):+ return QuartoExporter(), today(), report.content+++def verify_document(document, report, started):+ date = re.search(r'^date: "(\d{4}-\d{2}-\d{2})"$', document, re.MULTILINE)+ assert date and date.group(1) in {started, today()}, "Export date mismatch"+ assert document == report.expected_document(date.group(1)), (+ "Quarto document mismatch"+ )+++def verify(state, output, report, mode):+ _, started, original = state+ assert original == report.content+ if mode == "document":+ verify_document(output, report, started)+ return+ assert output.mimetype == "application/zip"+ assert output.filename == "Synthetic_report_quarto.zip"+ with zipfile.ZipFile(io.BytesIO(output.content)) as archive:+ assert archive.namelist() == [+ "Synthetic_report.qmd",+ "references.bib",+ ], "Quarto archive members mismatch"+ assert archive.testzip() is None+ document = archive.read("Synthetic_report.qmd").decode("utf-8")+ bibliography = archive.read("references.bib").decode("utf-8")+ verify_document(document, report, started)+ assert bibliography == report.bibliography, "Quarto bibliography mismatch"diff --git a/tests/performance/quarto_export/test_offline.py b/tests/performance/quarto_export/test_offline.pynew file mode 100644index 0000000000..af923b6c12--- /dev/null+++ b/tests/performance/quarto_export/test_offline.py@@ -0,0 +1,105 @@+"""Measure reference/text growth and assert the complete exported documents."""++from concurrent.futures import ThreadPoolExecutor+from dataclasses import replace+import io+import zipfile++import pytest++from local_deep_research.exporters.quarto_exporter import QuartoExporter+from .corpus import Report, perform, prepare, today, verify++[email protected]("mode", ["document", "zip"])[email protected]("curve", ["references", "text"])+def test_export_growth_preserves_complete_document_and_bibliography(+ quarto_workload, entry_probe, mode, curve+):+ for size in (+ quarto_workload.settings["size"],+ quarto_workload.settings["size"] * 2,+ ):+ report = Report(size, 4) if curve == "references" else Report(4, size)+ bound = 3 if mode == "zip" else 2++ def setup():+ entry_probe.update(parses=0, parsed_characters=0)+ return prepare(report)++ def check(state, output):+ verify(state, output, report, mode)+ assert entry_probe["parses"] <= bound+ assert entry_probe["parsed_characters"] <= bound * len(+ report.content+ )+ return dict(+ references=report.references,+ paragraphs=report.paragraphs,+ **entry_probe,+ )++ quarto_workload.measure(+ f"{curve}_{mode}",+ size,+ setup,+ lambda state: perform(state[0], report, mode),+ check,+ units=len(report.content),+ unit="input_character",+ parameters=dict(+ references=report.references,+ paragraphs=report.paragraphs,+ mode=mode,+ instrumentation=True,+ corpus_sha256=report.sha256,+ input_characters=len(report.content),+ input_bytes=len(report.content.encode("utf-8")),+ ),+ )++[email protected]("mode", ["document", "zip"])+def test_reused_exporter_observes_changed_reports(mode):+ exporter = QuartoExporter()+ outputs = []+ for report in [Report(2, 1), Report(3, 2), Report(2, 1)]:+ state = exporter, today(), report.content+ output = perform(exporter, report, mode)+ verify(state, output, report, mode)+ outputs.append((state, output, report))+ for state, output, report in outputs:+ verify(state, output, report, mode)+++def test_shared_exporter_keeps_concurrent_report_results_separate():+ exporter = QuartoExporter()+ reports = [Report(2, 1), Report(3, 2)]+ started = today()+ with ThreadPoolExecutor(max_workers=2) as pool:+ outputs = list(pool.map(lambda r: perform(exporter, r, "zip"), reports))+ for report, output in zip(reports, outputs, strict=True):+ verify((exporter, started, report.content), output, report, "zip")++[email protected]("fault", ["document", "bibliography", "member"])+def test_archive_oracle_rejects_incomplete_or_changed_results(fault):+ report = Report(2, 1)+ state = prepare(report)+ output = perform(state[0], report, "zip")+ verify(state, output, report, "zip")+ contents = io.BytesIO()+ with zipfile.ZipFile(io.BytesIO(output.content)) as original:+ with zipfile.ZipFile(contents, "w", zipfile.ZIP_DEFLATED) as changed:+ for name in original.namelist():+ if name == "references.bib" and fault == "member":+ continue+ value = original.read(name).decode("utf-8")+ if fault == "document" and name.endswith(".qmd"):+ value = value.replace("[@ref1]", "[@ref9999]")+ if fault == "bibliography" and name.endswith(".bib"):+ value = value.replace("/source/1", "/source/9999")+ changed.writestr(name, value)+ corrupted = replace(output, content=contents.getvalue())+ with pytest.raises(AssertionError, match="Quarto .* mismatch"):+ verify(state, corrupted, report, "zip")