Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

test(performance): measure Quarto document and archive exports (#8043)

LearningCircuit18911c756657
diff --git a/scripts/ci/performance_pytest.py b/scripts/ci/performance_pytest.py
index b83deb7527..37ae3b5d16 100644
--- a/scripts/ci/performance_pytest.py
+++ b/scripts/ci/performance_pytest.py
@@ -18,6 +18,7 @@ SUITES = (
("tests/performance/content_cache", "cache"),
("tests/performance/citation_formatting", "citations"),
("tests/performance/citation_metadata", "citenorm"),
+ ("tests/performance/quarto_export", "quarto"),
("tests/performance/url_processing", "urls"),
("tests/performance/text_splitting", "splitting"),
("tests/performance/source_rendering", "sources"),
@@ -66,6 +67,15 @@ def available_suites(root):
suites.append(
("tests/metrics/test_token_counting_dispatch_attribution.py", None)
)
+ if "tests/performance/quarto_export" in targets:
+ suites.extend(
+ (target, None)
+ for target in (
+ "tests/exporters/test_quarto_exporter.py",
+ "tests/text_optimization/test_quarto_yaml_title_escaping.py",
+ "tests/text_optimization/test_bibliography_escaping.py",
+ )
+ )
contracts = "tests/ci/test_offline_workload_shards.py"
if (root / contracts).is_file():
suites.append((contracts, None))
diff --git a/tests/ci/standalone/test_performance_pytest.py b/tests/ci/standalone/test_performance_pytest.py
index d7993f3c71..7e92fbe67e 100644
--- a/tests/ci/standalone/test_performance_pytest.py
+++ b/tests/ci/standalone/test_performance_pytest.py
@@ -21,6 +21,9 @@ class BenchmarkEvidenceTests(unittest.TestCase):
self.temporary = tempfile.TemporaryDirectory()
self.addCleanup(self.temporary.cleanup)
self.root = Path(self.temporary.name)
+ # TMPDIR may be inside a real checkout. Keep miniature checkouts
+ # independent of its pytest root/configuration and JUnit prefixes.
+ (self.root / "pytest.ini").write_text("[pytest]\n", encoding="utf-8")
def good_report(self):
return {
diff --git a/tests/performance/quarto_export/README.md b/tests/performance/quarto_export/README.md
new file mode 100644
index 0000000000..2870a6d58e
--- /dev/null
+++ b/tests/performance/quarto_export/README.md
@@ -0,0 +1,35 @@
+# Quarto export measurements
+
+Measure the public document conversion and full in-memory ZIP export while
+varying bibliography size and report text length independently. The ZIP timing
+includes compression and both archive writes. Corpus construction, exporter
+construction, archive decompression and correctness checks are outside timing;
+the export logger's sink is disabled. No Quarto/Pandoc process, network or model
+is used. This measures serialization, not rendering a document to HTML or PDF.
+
+```bash
+pdm run pytest tests/performance/quarto_export -n 0 -q \
+ --quarto-profile local --quarto-report /tmp/quarto-export.json
+```
+
+CI uses sizes 4/8, two measured samples and one warmup. Local uses 128/256,
+seven samples and two warmups. `--quarto-{profile,size,iterations,warmups,report,
+max-p95-ms}` overrides `LDR_PERF_QUARTO_*`; `LDR_PERF_PROFILE` is the shared
+profile fallback. Timing ceilings are opt-in; CI always asserts correctness
+and work bounds. Native thread counts should remain one for comparisons.
+
+Every iteration compares the complete expected document, bibliography, archive
+membership, filename and MIME type. The expected text is composed independently
+from the corpus parameters; UTC date is checked separately. Both grouped and
+individual references must resolve while unlisted numbers remain prose.
+Negative controls alter document references, bibliography URLs or archive
+membership. Reused and concurrently shared exporters must observe each input
+without changing earlier results.
+
+A wrapper around the real bibliography-entry parser counts calls and characters
+presented to it, adding small instrumentation overhead. The baseline bounds are
+two parses per document and three per ZIP, allowing future reductions. This is
+a count of parser invocations, not a claim that the parser touches each character
+once. Raw samples, corpus hashes and both loaded implementation hashes are
+retained by the shared measurement helper and automatic CI runner. Per-character
+p95 divides a whole-export statistic; it is not a character latency percentile.
diff --git a/tests/performance/quarto_export/__init__.py b/tests/performance/quarto_export/__init__.py
new file mode 100644
index 0000000000..ad832edccd
--- /dev/null
+++ b/tests/performance/quarto_export/__init__.py
@@ -0,0 +1 @@
+"""Offline measurements of complete Quarto document and ZIP exports."""
diff --git a/tests/performance/quarto_export/conftest.py b/tests/performance/quarto_export/conftest.py
new file mode 100644
index 0000000000..1a1bd419e6
--- /dev/null
+++ b/tests/performance/quarto_export/conftest.py
@@ -0,0 +1,54 @@
+from types import SimpleNamespace
+
+import pytest
+
+from local_deep_research.exporters import quarto_exporter
+from local_deep_research.text_optimization import citation_formatter
+from tests.performance._shared.pytest_workload import (
+ Workload,
+ add_options,
+ resolve,
+)
+
+PROFILES = {
+ "ci": dict(size=4, iterations=2, warmups=1),
+ "local": dict(size=128, iterations=7, warmups=2),
+}
+
+
+def pytest_addoption(parser):
+ add_options(parser, "quarto")
+
+
[email protected](scope="session")
+def quarto_workload(request):
+ workload = Workload("quarto", resolve(request.config, "quarto", PROFILES))
+ yield workload
+ workload.finish(
+ modules=(
+ "local_deep_research.exporters.quarto_exporter",
+ "local_deep_research.text_optimization.citation_formatter",
+ )
+ )
+
+
+def entry_probe(monkeypatch):
+ counts = dict(parses=0, parsed_characters=0)
+ original = citation_formatter.QuartoExporter._bibliography_entries
+
+ def counted(exporter, content):
+ counts["parses"] += 1
+ counts["parsed_characters"] += len(content)
+ return original(exporter, content)
+
+ def noop(*args, **kwargs):
+ return None
+
+ monkeypatch.setattr(
+ citation_formatter.QuartoExporter, "_bibliography_entries", counted
+ )
+ monkeypatch.setattr(
+ quarto_exporter, "logger", SimpleNamespace(info=noop, exception=noop)
+ )
+ return counts
diff --git a/tests/performance/quarto_export/corpus.py b/tests/performance/quarto_export/corpus.py
new file mode 100644
index 0000000000..97faea665b
--- /dev/null
+++ b/tests/performance/quarto_export/corpus.py
@@ -0,0 +1,138 @@
+"""Plain synthetic reports with complete, independently composed expectations."""
+
+from datetime import UTC, datetime
+import hashlib
+import io
+import re
+import zipfile
+
+from local_deep_research.exporters.base import ExportOptions
+from local_deep_research.exporters.quarto_exporter import QuartoExporter
+
+TITLE = "Synthetic report"
+READER = (
+ "markdown-raw_html-raw_tex-raw_attribute-native_divs-native_spans"
+ "-fenced_divs-bracketed_spans-header_attributes-link_attributes"
+ "-inline_code_attributes-fenced_code_attributes-yaml_metadata_block"
+ "-pandoc_title_block"
+)
+
+
+def today():
+ return datetime.now(UTC).date().isoformat()
+
+
+def header(date):
+ return f'''---
+title: "{TITLE}"
+author: "Local Deep Research"
+date: "{date}"
+from: {READER}
+format:
+ html:
+ toc: true
+ toc-depth: 3
+ number-sections: true
+ pdf:
+ toc: true
+ number-sections: true
+ colorlinks: true
+bibliography: references.bib
+csl: apa.csl
+---
+
+'''
+
+
+class Report:
+ def __init__(self, references, paragraphs):
+ body, converted = [], []
+ for number in range(paragraphs):
+ prefix = f"Synthetic café paragraph {number}: evidence "
+ body.append(
+ prefix + f"[1] and [{references}, 1]; prose [999999].\n"
+ )
+ converted.append(
+ prefix
+ + f"[@ref1] and [@ref{references}, @ref1]; prose [999999].\n"
+ )
+ sources, converted_sources, bibliography = [], [], []
+ for number in range(1, references + 1):
+ title = f"Synthetic source {number}"
+ url = f"https://example.test/source/{number}"
+ continuation = f" {title}\n URL: {url}\n"
+ sources.append(f"[{number}]" + continuation)
+ converted_sources.append(f"[@ref{number}]" + continuation)
+ bibliography.append(
+ f"@misc{{ref{number},\n"
+ f' title = "{{{title}}}",\n'
+ f" url = {{{url}}},\n"
+ f' howpublished = "\\url{{{url}}}",\n'
+ " year = {2024},\n"
+ ' note = "Accessed: \\today"\n'
+ "}"
+ )
+ self.content = (
+ f"# {TITLE}\n\n"
+ + "".join(body)
+ + "\n## Sources\n\n"
+ + "\n".join(sources)
+ )
+ self.converted = (
+ f"# {TITLE}\n\n"
+ + "".join(converted)
+ + "\n## Sources\n\n"
+ + "\n".join(converted_sources)
+ )
+ self.bibliography = "\n\n".join(bibliography)
+ self.sha256 = hashlib.sha256(self.content.encode()).hexdigest()
+ self.references, self.paragraphs = references, paragraphs
+
+ def expected_document(self, date):
+ return (
+ header(date)
+ + self.converted
+ + "\n\n**Bibliography File Required**\n\n"
+ "This document requires a `references.bib` file in the same "
+ "directory with the following content:\n\n```bibtex\n"
+ + self.bibliography
+ + "\n```\n"
+ )
+
+
+def perform(exporter, report, mode):
+ if mode == "zip":
+ return exporter.export(report.content, ExportOptions(title=TITLE))
+ return exporter._legacy.export_to_quarto(report.content, TITLE)
+
+
+def prepare(report):
+ return QuartoExporter(), today(), report.content
+
+
+def verify_document(document, report, started):
+ date = re.search(r'^date: "(\d{4}-\d{2}-\d{2})"$', document, re.MULTILINE)
+ assert date and date.group(1) in {started, today()}, "Export date mismatch"
+ assert document == report.expected_document(date.group(1)), (
+ "Quarto document mismatch"
+ )
+
+
+def verify(state, output, report, mode):
+ _, started, original = state
+ assert original == report.content
+ if mode == "document":
+ verify_document(output, report, started)
+ return
+ assert output.mimetype == "application/zip"
+ assert output.filename == "Synthetic_report_quarto.zip"
+ with zipfile.ZipFile(io.BytesIO(output.content)) as archive:
+ assert archive.namelist() == [
+ "Synthetic_report.qmd",
+ "references.bib",
+ ], "Quarto archive members mismatch"
+ assert archive.testzip() is None
+ document = archive.read("Synthetic_report.qmd").decode("utf-8")
+ bibliography = archive.read("references.bib").decode("utf-8")
+ verify_document(document, report, started)
+ assert bibliography == report.bibliography, "Quarto bibliography mismatch"
diff --git a/tests/performance/quarto_export/test_offline.py b/tests/performance/quarto_export/test_offline.py
new file mode 100644
index 0000000000..af923b6c12
--- /dev/null
+++ b/tests/performance/quarto_export/test_offline.py
@@ -0,0 +1,105 @@
+"""Measure reference/text growth and assert the complete exported documents."""
+
+from concurrent.futures import ThreadPoolExecutor
+from dataclasses import replace
+import io
+import zipfile
+
+import pytest
+
+from local_deep_research.exporters.quarto_exporter import QuartoExporter
+from .corpus import Report, perform, prepare, today, verify
+
+
[email protected]("mode", ["document", "zip"])
[email protected]("curve", ["references", "text"])
+def test_export_growth_preserves_complete_document_and_bibliography(
+ quarto_workload, entry_probe, mode, curve
+):
+ for size in (
+ quarto_workload.settings["size"],
+ quarto_workload.settings["size"] * 2,
+ ):
+ report = Report(size, 4) if curve == "references" else Report(4, size)
+ bound = 3 if mode == "zip" else 2
+
+ def setup():
+ entry_probe.update(parses=0, parsed_characters=0)
+ return prepare(report)
+
+ def check(state, output):
+ verify(state, output, report, mode)
+ assert entry_probe["parses"] <= bound
+ assert entry_probe["parsed_characters"] <= bound * len(
+ report.content
+ )
+ return dict(
+ references=report.references,
+ paragraphs=report.paragraphs,
+ **entry_probe,
+ )
+
+ quarto_workload.measure(
+ f"{curve}_{mode}",
+ size,
+ setup,
+ lambda state: perform(state[0], report, mode),
+ check,
+ units=len(report.content),
+ unit="input_character",
+ parameters=dict(
+ references=report.references,
+ paragraphs=report.paragraphs,
+ mode=mode,
+ instrumentation=True,
+ corpus_sha256=report.sha256,
+ input_characters=len(report.content),
+ input_bytes=len(report.content.encode("utf-8")),
+ ),
+ )
+
+
[email protected]("mode", ["document", "zip"])
+def test_reused_exporter_observes_changed_reports(mode):
+ exporter = QuartoExporter()
+ outputs = []
+ for report in [Report(2, 1), Report(3, 2), Report(2, 1)]:
+ state = exporter, today(), report.content
+ output = perform(exporter, report, mode)
+ verify(state, output, report, mode)
+ outputs.append((state, output, report))
+ for state, output, report in outputs:
+ verify(state, output, report, mode)
+
+
+def test_shared_exporter_keeps_concurrent_report_results_separate():
+ exporter = QuartoExporter()
+ reports = [Report(2, 1), Report(3, 2)]
+ started = today()
+ with ThreadPoolExecutor(max_workers=2) as pool:
+ outputs = list(pool.map(lambda r: perform(exporter, r, "zip"), reports))
+ for report, output in zip(reports, outputs, strict=True):
+ verify((exporter, started, report.content), output, report, "zip")
+
+
[email protected]("fault", ["document", "bibliography", "member"])
+def test_archive_oracle_rejects_incomplete_or_changed_results(fault):
+ report = Report(2, 1)
+ state = prepare(report)
+ output = perform(state[0], report, "zip")
+ verify(state, output, report, "zip")
+ contents = io.BytesIO()
+ with zipfile.ZipFile(io.BytesIO(output.content)) as original:
+ with zipfile.ZipFile(contents, "w", zipfile.ZIP_DEFLATED) as changed:
+ for name in original.namelist():
+ if name == "references.bib" and fault == "member":
+ continue
+ value = original.read(name).decode("utf-8")
+ if fault == "document" and name.endswith(".qmd"):
+ value = value.replace("[@ref1]", "[@ref9999]")
+ if fault == "bibliography" and name.endswith(".bib"):
+ value = value.replace("/source/1", "/source/9999")
+ changed.writestr(name, value)
+ corrupted = replace(output, content=contents.getvalue())
+ with pytest.raises(AssertionError, match="Quarto .* mismatch"):
+ verify(state, corrupted, report, "zip")