Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

13.5 KB
Raw Blame
"""Run available offline benchmarks serially and require asserted results."""

import argparse
import json
import math
import os
from pathlib import Path
import statistics
import subprocess
import sys
import tempfile

from defusedxml import ElementTree as ET


SUITES = (
    ("tests/performance/content_extraction", "extraction"),
    ("tests/performance/content_cache", "cache"),
    ("tests/performance/citation_formatting", "citations"),
    ("tests/performance/citation_metadata", "citenorm"),
    ("tests/performance/quarto_export", "quarto"),
    ("tests/performance/url_processing", "urls"),
    ("tests/performance/text_splitting", "splitting"),
    ("tests/performance/source_rendering", "sources"),
    ("tests/performance/json_extraction", "jsonparse"),
    ("tests/performance/chat_context", "chatctx"),
    ("tests/performance/token_tracking", "tokens"),
    ("tests/performance/source_persistence", "sourcepersist"),
    ("tests/performance/settings_snapshots", "settings"),
    ("tests/performance/rag_scale_smoke", None),
    ("tests/performance/database/test_connection_workload.py", None),
    ("tests/performance/http/test_request_workload.py", None),
    ("tests/performance/rag/test_storage_workload.py", None),
    ("tests/search_quality", None),
)

NATIVE_REPORTS = {
    "tests/performance/database/test_connection_workload.py": (
        "database_performance",
        (),
    ),
    "tests/performance/http/test_request_workload.py": ("http_performance", ()),
    "tests/performance/rag/test_storage_workload.py": (
        "rag_performance",
        ("flat", "hnsw"),
    ),
    "tests/search_quality": ("search_quality", ("flat", "hnsw")),
}


def available_suites(root):
    suites = [
        (target, area)
        for target, area in SUITES
        if (root / target).is_file()
        or (
            (root / target).is_dir() and any((root / target).rglob("test_*.py"))
        )
    ]
    if not suites:
        raise ValueError("No offline benchmark suites are present")
    targets = {target for target, _ in suites}
    if "tests/performance/citation_metadata" in targets:
        suites.append(("tests/utilities/test_citation_normalizer.py", None))
    if "tests/performance/rag_scale_smoke" in targets:
        suites.append(("tests/vector_stores/test_segmented_faiss.py", None))
    if "tests/performance/token_tracking" in targets:
        suites.append(
            ("tests/metrics/test_token_counting_dispatch_attribution.py", None)
        )
    if "tests/performance/quarto_export" in targets:
        suites.extend(
            (target, None)
            for target in (
                "tests/exporters/test_quarto_exporter.py",
                "tests/text_optimization/test_quarto_yaml_title_escaping.py",
                "tests/text_optimization/test_bibliography_escaping.py",
            )
        )
    contracts = "tests/ci/test_offline_workload_shards.py"
    if (root / contracts).is_file():
        suites.append((contracts, None))
    return suites


def pytest_command(root, suites, profile, output, temporary=None):
    targets = [target for target, _ in suites]
    helper = "tests/performance/test_pytest_workload.py"
    if (root / helper).is_file():
        targets.append(helper)
    resources = "tests/performance/test_resources.py"
    if (root / resources).is_file():
        targets.append(resources)
    command = [
        sys.executable,
        "-m",
        "pytest",
        *targets,
        "-q",
        "-n",
        "0",
        "--tb=short",
        "--junitxml",
        str(output / "pytest.xml"),
    ]
    if temporary is not None:
        command.extend(["--basetemp", str(temporary)])
    for _, area in suites:
        if area:
            command.extend(
                [
                    f"--{area}-profile",
                    profile,
                    f"--{area}-report",
                    str(output / f"{area}.json"),
                ]
            )
    if any(
        target == "tests/performance/rag_scale_smoke" for target, _ in suites
    ):
        command.extend(["--ragscale-profile", profile])
    return command


def require_junit(path, suites):
    cases = list(ET.parse(path).getroot().iter("testcase"))
    if not cases:
        raise ValueError("Benchmark CI collected no test cases")
    if any(
        case.find(tag) is not None
        for case in cases
        for tag in ("failure", "error", "skipped")
    ):
        raise ValueError("A benchmark assertion failed or was skipped")
    counts = {}
    for target, _ in suites:
        prefix = target.removesuffix(".py").replace("/", ".")
        count = sum(
            case.get("classname", "") == prefix
            or case.get("classname", "").startswith(prefix + ".")
            for case in cases
        )
        if count < 1:
            raise ValueError(f"No passing assertions recorded for {target}")
        counts[target] = count
    return counts


def finite_number(value):
    return (
        isinstance(value, (int, float))
        and not isinstance(value, bool)
        and math.isfinite(value)
        and value >= 0
    )


def require_samples(result, key="samples_ms", scale=1):
    if not isinstance(result, dict):
        raise ValueError("Invalid measurement")
    count, samples = result.get("count"), result.get(key)
    if (
        type(count) is not int
        or count < 1
        or not isinstance(samples, list)
        or len(samples) != count
        or not all(finite_number(value) for value in samples)
    ):
        raise ValueError("Missing or invalid measured samples")
    ordered = sorted(samples)
    expected = {
        "median_ms": statistics.median(samples) / scale,
        "p95_ms": ordered[math.ceil(0.95 * count) - 1] / scale,
    }
    if "max_ms" in result:
        expected["max_ms"] = ordered[-1] / scale
    for field, value in expected.items():
        actual = result.get(field)
        if not finite_number(actual) or not math.isclose(
            actual, value, rel_tol=1e-9, abs_tol=1e-12
        ):
            raise ValueError(
                f"Measurement summary disagrees with samples: {field}"
            )


def require_report(path, area, profile):
    report = json.loads(path.read_text(encoding="utf-8"))
    if not isinstance(report, dict):
        raise ValueError(f"Invalid benchmark report: {path.name}")
    results = report.get("results")
    settings = report.get("settings")
    if (
        report.get("schema_version") != 2
        or report.get("area") != area
        or not isinstance(settings, dict)
        or settings.get("profile") != profile
        or type(settings.get("iterations")) is not int
        or settings["iterations"] < 1
        or report.get("status") != "passed"
        or not isinstance(results, list)
        or not results
    ):
        raise ValueError(f"Missing or failed benchmark results: {path.name}")
    for result in results:
        if (
            not isinstance(result, dict)
            or result.get("status") != "passed"
            or not isinstance(result.get("checks"), dict)
            or not result["checks"]
            or result.get("count") != settings["iterations"]
        ):
            raise ValueError(f"Unverified or unmeasured case in {path.name}")
        require_samples(result, "samples_ns", 1_000_000)
        require_samples(result)
        if any(
            not math.isclose(ns / 1_000_000, ms, rel_tol=1e-9, abs_tol=1e-12)
            for ns, ms in zip(
                result["samples_ns"], result["samples_ms"], strict=True
            )
        ):
            raise ValueError("Nanosecond and millisecond samples disagree")
    return len(results)


def require_native_report(report):
    configuration = report.get("configuration")
    if (
        not isinstance(configuration, dict)
        or configuration.get("profile") != "ci"
    ):
        raise ValueError(
            "Native smoke report is missing its effective CI configuration"
        )
    if report.get("topic") == "search_quality":
        results = report.get("results")
        if (
            report.get("quality_failures") != []
            or not isinstance(results, list)
            or not results
        ):
            raise ValueError("Missing or failed retrieval-quality results")
        for result in results:
            queries = result.get("queries")
            if not isinstance(queries, list) or not queries:
                raise ValueError("Missing retrieval-quality queries")
            for query in queries:
                recall = query.get("recall_at_k")
                if (
                    not finite_number(recall)
                    or recall > 1
                    or type(query.get("returned")) is not int
                    or query["returned"] < 1
                ):
                    raise ValueError("Invalid retrieval-quality query")
        return len(results)
    measurements = report.get("measurements")
    if not isinstance(measurements, dict) or not measurements:
        raise ValueError("Missing native workload measurements")
    for measurement in measurements.values():
        require_samples(measurement)
    return len(measurements)


def collect_native_reports(temporary, suites, output, *, complete=True):
    required = {
        NATIVE_REPORTS[target][0]: NATIVE_REPORTS[target][1]
        for target, _ in suites
        if target in NATIVE_REPORTS
    }
    collected = {topic: [] for topic in required}
    candidates = set(temporary.rglob("isolated-result.json")) | set(
        temporary.rglob("http.json")
    )
    for path in sorted(candidates):
        report = json.loads(path.read_text(encoding="utf-8"))
        if not isinstance(report, dict) or report.get("topic") not in required:
            continue
        topic = report["topic"]
        destination = (
            output / "native" / f"{topic}-{len(collected[topic])}.json"
        )
        destination.parent.mkdir(parents=True, exist_ok=True)
        destination.write_text(
            json.dumps(report, indent=2) + "\n", encoding="utf-8"
        )
        count = require_native_report(report)
        collected[topic].append(
            {
                "path": str(destination),
                "source": str(path.relative_to(temporary)),
                "configuration": report["configuration"],
                "index_type": report.get("index_type"),
                "measured_cases": count,
            }
        )
    if complete:
        for topic, backends in required.items():
            if not collected[topic] or not set(backends).issubset(
                {result["index_type"] for result in collected[topic]}
            ):
                raise ValueError(
                    f"Missing native workload report or backend: {topic}"
                )
    return collected


def require_success(returncode):
    if returncode:
        raise RuntimeError(f"Benchmark pytest exited {returncode}")


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--profile", choices=("ci", "local"), default="ci")
    parser.add_argument("--output", type=Path, required=True)
    args = parser.parse_args()
    root = Path(__file__).resolve().parents[2]
    output = args.output.resolve()
    output.mkdir(parents=True, exist_ok=True)
    run_directory = Path(tempfile.mkdtemp(prefix="run-", dir=output))
    environment = dict(
        os.environ,
        LDR_PERF_PROFILE=args.profile,
        OMP_NUM_THREADS="1",
        OPENBLAS_NUM_THREADS="1",
        MKL_NUM_THREADS="1",
        NUMEXPR_NUM_THREADS="1",
        HF_HUB_OFFLINE="1",
        TRANSFORMERS_OFFLINE="1",
        TOKENIZERS_PARALLELISM="false",
        LDR_TESTING_WITH_MOCKS="false",
        PYTHONOPTIMIZE="0",
        PYTHONPATH=f"{root / 'src'}{os.pathsep}{root}",
    )
    environment["LDR_PERF_RAGSCALE_REPORT_DIR"] = str(
        run_directory / "ragscale"
    )
    summary = {
        "profile": args.profile,
        "run_directory": str(run_directory),
        "status": "running",
    }
    try:
        suites = available_suites(root)
        with tempfile.TemporaryDirectory(
            prefix=".pytest-", dir=output.parent
        ) as temporary:
            command = pytest_command(
                root, suites, args.profile, run_directory, temporary
            )
            summary.update(command=command, suites=[x[0] for x in suites])
            result = subprocess.run(
                command, cwd=root, env=environment, check=False
            )
            summary["pytest_returncode"] = result.returncode
            native = collect_native_reports(
                Path(temporary),
                suites,
                run_directory,
                complete=result.returncode == 0,
            )
            summary["native_reports"] = native
            require_success(result.returncode)
            summary["asserted_tests"] = require_junit(
                run_directory / "pytest.xml", suites
            )
            summary["measured_cases"] = {
                area: require_report(
                    run_directory / f"{area}.json", area, args.profile
                )
                for _, area in suites
                if area
            }
            summary["measured_cases"].update(
                {
                    topic: sum(record["measured_cases"] for record in records)
                    for topic, records in native.items()
                }
            )
        summary["status"] = "passed"
    except BaseException:
        summary["status"] = "failed"
        raise
    finally:
        (output / "ci-summary.json").write_text(
            json.dumps(summary, indent=2) + "\n", encoding="utf-8"
        )


if __name__ == "__main__":
    main()