Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

4.9 KB
Raw Blame
"""Plain synthetic reports with complete, independently composed expectations.

The ``from:`` reader line is the one exception: it is taken from the
exporter's ``_QUARTO_READER_FORMAT`` rather than spelled out here. This
oracle first hard-coded that format, built before #8070 switched
``multiline_tables`` off in the reader (#8027). Once both landed on main,
every exported document carried ``-multiline_tables`` and no longer matched
the stale copy, although the export itself was correct. The reader format's
behaviour is pinned by the citation formatter contract tests; this
measurement checks that the document declares the exporter's real format,
while the body, header fields and bibliography stay independently composed.
"""

from datetime import UTC, datetime
import hashlib
import io
import re
import zipfile

from local_deep_research.exporters.base import ExportOptions
from local_deep_research.exporters.quarto_exporter import QuartoExporter
from local_deep_research.text_optimization.citation_formatter import (
    _QUARTO_READER_FORMAT as READER,
)

TITLE = "Synthetic report"


def today():
    return datetime.now(UTC).date().isoformat()


def header(date):
    return f'''---
title: "{TITLE}"
author: "Local Deep Research"
date: "{date}"
from: {READER}
format:
  html:
    toc: true
    toc-depth: 3
    number-sections: true
  pdf:
    toc: true
    number-sections: true
    colorlinks: true
bibliography: references.bib
csl: apa.csl
---

'''


class Report:
    def __init__(self, references, paragraphs):
        body, converted = [], []
        for number in range(paragraphs):
            prefix = f"Synthetic café paragraph {number}: evidence "
            body.append(
                prefix + f"[1] and [{references}, 1]; prose [999999].\n"
            )
            converted.append(
                prefix
                + f"[@ref1] and [@ref{references}, @ref1]; prose [999999].\n"
            )
        sources, converted_sources, bibliography = [], [], []
        for number in range(1, references + 1):
            title = f"Synthetic source {number}"
            url = f"https://example.test/source/{number}"
            continuation = f" {title}\n   URL: {url}\n"
            sources.append(f"[{number}]" + continuation)
            converted_sources.append(f"[@ref{number}]" + continuation)
            bibliography.append(
                f"@misc{{ref{number},\n"
                f'  title = "{{{title}}}",\n'
                f"  url = {{{url}}},\n"
                f'  howpublished = "\\url{{{url}}}",\n'
                "  year = {2024},\n"
                '  note = "Accessed: \\today"\n'
                "}"
            )
        self.content = (
            f"# {TITLE}\n\n"
            + "".join(body)
            + "\n## Sources\n\n"
            + "\n".join(sources)
        )
        self.converted = (
            f"# {TITLE}\n\n"
            + "".join(converted)
            + "\n## Sources\n\n"
            + "\n".join(converted_sources)
        )
        self.bibliography = "\n\n".join(bibliography)
        self.sha256 = hashlib.sha256(self.content.encode()).hexdigest()
        self.references, self.paragraphs = references, paragraphs

    def expected_document(self, date):
        return (
            header(date)
            + self.converted
            + "\n\n**Bibliography File Required**\n\n"
            "This document requires a `references.bib` file in the same "
            "directory with the following content:\n\n```bibtex\n"
            + self.bibliography
            + "\n```\n"
        )


def perform(exporter, report, mode):
    if mode == "zip":
        return exporter.export(report.content, ExportOptions(title=TITLE))
    return exporter._legacy.export_to_quarto(report.content, TITLE)


def prepare(report):
    return QuartoExporter(), today(), report.content


def verify_document(document, report, started):
    date = re.search(r'^date: "(\d{4}-\d{2}-\d{2})"$', document, re.MULTILINE)
    assert date and date.group(1) in {started, today()}, "Export date mismatch"
    assert document == report.expected_document(date.group(1)), (
        "Quarto document mismatch"
    )


def verify(state, output, report, mode):
    _, started, original = state
    assert original == report.content
    if mode == "document":
        verify_document(output, report, started)
        return
    assert output.mimetype == "application/zip"
    assert output.filename == "Synthetic_report_quarto.zip"
    with zipfile.ZipFile(io.BytesIO(output.content)) as archive:
        assert archive.namelist() == [
            "Synthetic_report.qmd",
            "references.bib",
        ], "Quarto archive members mismatch"
        assert archive.testzip() is None
        document = archive.read("Synthetic_report.qmd").decode("utf-8")
        bibliography = archive.read("references.bib").decode("utf-8")
    verify_document(document, report, started)
    assert bibliography == report.bibliography, "Quarto bibliography mismatch"