~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""Run available offline benchmarks serially and require asserted results."""
import argparse
import json
import math
import os
from pathlib import Path
import statistics
import subprocess
import sys
import tempfile
from defusedxml import ElementTree as ET
SUITES = (
("tests/performance/content_extraction", "extraction"),
("tests/performance/content_cache", "cache"),
("tests/performance/citation_formatting", "citations"),
("tests/performance/citation_metadata", "citenorm"),
("tests/performance/quarto_export", "quarto"),
("tests/performance/url_processing", "urls"),
("tests/performance/text_splitting", "splitting"),
("tests/performance/source_rendering", "sources"),
("tests/performance/json_extraction", "jsonparse"),
("tests/performance/chat_context", "chatctx"),
("tests/performance/token_tracking", "tokens"),
("tests/performance/source_persistence", "sourcepersist"),
("tests/performance/settings_snapshots", "settings"),
("tests/performance/rag_scale_smoke", None),
("tests/performance/database/test_connection_workload.py", None),
("tests/performance/http/test_request_workload.py", None),
("tests/performance/rag/test_storage_workload.py", None),
("tests/search_quality", None),
)
NATIVE_REPORTS = {
"tests/performance/database/test_connection_workload.py": (
"database_performance",
(),
),
"tests/performance/http/test_request_workload.py": ("http_performance", ()),
"tests/performance/rag/test_storage_workload.py": (
"rag_performance",
("flat", "hnsw"),
),
"tests/search_quality": ("search_quality", ("flat", "hnsw")),
}
def available_suites(root):
suites = [
(target, area)
for target, area in SUITES
if (root / target).is_file()
or (
(root / target).is_dir() and any((root / target).rglob("test_*.py"))
)
]
if not suites:
raise ValueError("No offline benchmark suites are present")
targets = {target for target, _ in suites}
if "tests/performance/citation_metadata" in targets:
suites.append(("tests/utilities/test_citation_normalizer.py", None))
if "tests/performance/rag_scale_smoke" in targets:
suites.append(("tests/vector_stores/test_segmented_faiss.py", None))
if "tests/performance/token_tracking" in targets:
suites.append(
("tests/metrics/test_token_counting_dispatch_attribution.py", None)
)
if "tests/performance/quarto_export" in targets:
suites.extend(
(target, None)
for target in (
"tests/exporters/test_quarto_exporter.py",
"tests/text_optimization/test_quarto_yaml_title_escaping.py",
"tests/text_optimization/test_bibliography_escaping.py",
)
)
contracts = "tests/ci/test_offline_workload_shards.py"
if (root / contracts).is_file():
suites.append((contracts, None))
return suites
def pytest_command(root, suites, profile, output, temporary=None):
targets = [target for target, _ in suites]
helper = "tests/performance/test_pytest_workload.py"
if (root / helper).is_file():
targets.append(helper)
resources = "tests/performance/test_resources.py"
if (root / resources).is_file():
targets.append(resources)
command = [
sys.executable,
"-m",
"pytest",
*targets,
"-q",
"-n",
"0",
"--tb=short",
"--junitxml",
str(output / "pytest.xml"),
]
if temporary is not None:
command.extend(["--basetemp", str(temporary)])
for _, area in suites:
if area:
command.extend(
[
f"--{area}-profile",
profile,
f"--{area}-report",
str(output / f"{area}.json"),
]
)
if any(
target == "tests/performance/rag_scale_smoke" for target, _ in suites
):
command.extend(["--ragscale-profile", profile])
return command
def require_junit(path, suites):
cases = list(ET.parse(path).getroot().iter("testcase"))
if not cases:
raise ValueError("Benchmark CI collected no test cases")
if any(
case.find(tag) is not None
for case in cases
for tag in ("failure", "error", "skipped")
):
raise ValueError("A benchmark assertion failed or was skipped")
counts = {}
for target, _ in suites:
prefix = target.removesuffix(".py").replace("/", ".")
count = sum(
case.get("classname", "") == prefix
or case.get("classname", "").startswith(prefix + ".")
for case in cases
)
if count < 1:
raise ValueError(f"No passing assertions recorded for {target}")
counts[target] = count
return counts
def finite_number(value):
return (
isinstance(value, (int, float))
and not isinstance(value, bool)
and math.isfinite(value)
and value >= 0
)
def require_samples(result, key="samples_ms", scale=1):
if not isinstance(result, dict):
raise ValueError("Invalid measurement")
count, samples = result.get("count"), result.get(key)
if (
type(count) is not int
or count < 1
or not isinstance(samples, list)
or len(samples) != count
or not all(finite_number(value) for value in samples)
):
raise ValueError("Missing or invalid measured samples")
ordered = sorted(samples)
expected = {
"median_ms": statistics.median(samples) / scale,
"p95_ms": ordered[math.ceil(0.95 * count) - 1] / scale,
}
if "max_ms" in result:
expected["max_ms"] = ordered[-1] / scale
for field, value in expected.items():
actual = result.get(field)
if not finite_number(actual) or not math.isclose(
actual, value, rel_tol=1e-9, abs_tol=1e-12
):
raise ValueError(
f"Measurement summary disagrees with samples: {field}"
)
def require_report(path, area, profile):
report = json.loads(path.read_text(encoding="utf-8"))
if not isinstance(report, dict):
raise ValueError(f"Invalid benchmark report: {path.name}")
results = report.get("results")
settings = report.get("settings")
if (
report.get("schema_version") != 2
or report.get("area") != area
or not isinstance(settings, dict)
or settings.get("profile") != profile
or type(settings.get("iterations")) is not int
or settings["iterations"] < 1
or report.get("status") != "passed"
or not isinstance(results, list)
or not results
):
raise ValueError(f"Missing or failed benchmark results: {path.name}")
for result in results:
if (
not isinstance(result, dict)
or result.get("status") != "passed"
or not isinstance(result.get("checks"), dict)
or not result["checks"]
or result.get("count") != settings["iterations"]
):
raise ValueError(f"Unverified or unmeasured case in {path.name}")
require_samples(result, "samples_ns", 1_000_000)
require_samples(result)
if any(
not math.isclose(ns / 1_000_000, ms, rel_tol=1e-9, abs_tol=1e-12)
for ns, ms in zip(
result["samples_ns"], result["samples_ms"], strict=True
)
):
raise ValueError("Nanosecond and millisecond samples disagree")
return len(results)
def require_native_report(report):
configuration = report.get("configuration")
if (
not isinstance(configuration, dict)
or configuration.get("profile") != "ci"
):
raise ValueError(
"Native smoke report is missing its effective CI configuration"
)
if report.get("topic") == "search_quality":
results = report.get("results")
if (
report.get("quality_failures") != []
or not isinstance(results, list)
or not results
):
raise ValueError("Missing or failed retrieval-quality results")
for result in results:
queries = result.get("queries")
if not isinstance(queries, list) or not queries:
raise ValueError("Missing retrieval-quality queries")
for query in queries:
recall = query.get("recall_at_k")
if (
not finite_number(recall)
or recall > 1
or type(query.get("returned")) is not int
or query["returned"] < 1
):
raise ValueError("Invalid retrieval-quality query")
return len(results)
measurements = report.get("measurements")
if not isinstance(measurements, dict) or not measurements:
raise ValueError("Missing native workload measurements")
for measurement in measurements.values():
require_samples(measurement)
return len(measurements)
def collect_native_reports(temporary, suites, output, *, complete=True):
required = {
NATIVE_REPORTS[target][0]: NATIVE_REPORTS[target][1]
for target, _ in suites
if target in NATIVE_REPORTS
}
collected = {topic: [] for topic in required}
candidates = set(temporary.rglob("isolated-result.json")) | set(
temporary.rglob("http.json")
)
for path in sorted(candidates):
report = json.loads(path.read_text(encoding="utf-8"))
if not isinstance(report, dict) or report.get("topic") not in required:
continue
topic = report["topic"]
destination = (
output / "native" / f"{topic}-{len(collected[topic])}.json"
)
destination.parent.mkdir(parents=True, exist_ok=True)
destination.write_text(
json.dumps(report, indent=2) + "\n", encoding="utf-8"
)
count = require_native_report(report)
collected[topic].append(
{
"path": str(destination),
"source": str(path.relative_to(temporary)),
"configuration": report["configuration"],
"index_type": report.get("index_type"),
"measured_cases": count,
}
)
if complete:
for topic, backends in required.items():
if not collected[topic] or not set(backends).issubset(
{result["index_type"] for result in collected[topic]}
):
raise ValueError(
f"Missing native workload report or backend: {topic}"
)
return collected
def require_success(returncode):
if returncode:
raise RuntimeError(f"Benchmark pytest exited {returncode}")
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--profile", choices=("ci", "local"), default="ci")
parser.add_argument("--output", type=Path, required=True)
args = parser.parse_args()
root = Path(__file__).resolve().parents[2]
output = args.output.resolve()
output.mkdir(parents=True, exist_ok=True)
run_directory = Path(tempfile.mkdtemp(prefix="run-", dir=output))
environment = dict(
os.environ,
LDR_PERF_PROFILE=args.profile,
OMP_NUM_THREADS="1",
OPENBLAS_NUM_THREADS="1",
MKL_NUM_THREADS="1",
NUMEXPR_NUM_THREADS="1",
HF_HUB_OFFLINE="1",
TRANSFORMERS_OFFLINE="1",
TOKENIZERS_PARALLELISM="false",
LDR_TESTING_WITH_MOCKS="false",
PYTHONOPTIMIZE="0",
PYTHONPATH=f"{root / 'src'}{os.pathsep}{root}",
)
environment["LDR_PERF_RAGSCALE_REPORT_DIR"] = str(
run_directory / "ragscale"
)
summary = {
"profile": args.profile,
"run_directory": str(run_directory),
"status": "running",
}
try:
suites = available_suites(root)
with tempfile.TemporaryDirectory(
prefix=".pytest-", dir=output.parent
) as temporary:
command = pytest_command(
root, suites, args.profile, run_directory, temporary
)
summary.update(command=command, suites=[x[0] for x in suites])
result = subprocess.run(
command, cwd=root, env=environment, check=False
)
summary["pytest_returncode"] = result.returncode
native = collect_native_reports(
Path(temporary),
suites,
run_directory,
complete=result.returncode == 0,
)
summary["native_reports"] = native
require_success(result.returncode)
summary["asserted_tests"] = require_junit(
run_directory / "pytest.xml", suites
)
summary["measured_cases"] = {
area: require_report(
run_directory / f"{area}.json", area, args.profile
)
for _, area in suites
if area
}
summary["measured_cases"].update(
{
topic: sum(record["measured_cases"] for record in records)
for topic, records in native.items()
}
)
summary["status"] = "passed"
except BaseException:
summary["status"] = "failed"
raise
finally:
(output / "ci-summary.json").write_text(
json.dumps(summary, indent=2) + "\n", encoding="utf-8"
)
if __name__ == "__main__":
main()