~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""Reject absent, skipped and failed benchmark evidence without app imports."""
import importlib.util
from copy import deepcopy
import json
from pathlib import Path
import tempfile
import unittest
from unittest.mock import patch
SCRIPT = (
Path(__file__).resolve().parents[3] / "scripts/ci/performance_pytest.py"
)
SPEC = importlib.util.spec_from_file_location("performance_ci", SCRIPT)
CI = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(CI)
class BenchmarkEvidenceTests(unittest.TestCase):
def setUp(self):
self.temporary = tempfile.TemporaryDirectory()
self.addCleanup(self.temporary.cleanup)
self.root = Path(self.temporary.name)
# TMPDIR may be inside a real checkout. Keep miniature checkouts
# independent of its pytest root/configuration and JUnit prefixes.
(self.root / "pytest.ini").write_text("[pytest]\n", encoding="utf-8")
def good_report(self):
return {
"schema_version": 2,
"area": "chatctx",
"settings": {"profile": "ci", "iterations": 2},
"status": "passed",
"results": [
{
"status": "passed",
"checks": {"rows": 8},
"count": 2,
"samples_ns": [1_000_000, 2_000_000],
"samples_ms": [1, 2],
"median_ms": 1.5,
"p95_ms": 2,
}
],
}
def native_report(self, topic="database_performance", backend=None):
return {
"topic": topic,
"index_type": backend,
"configuration": {"profile": "ci", "iterations": 2, "rows": 8},
"measurements": {
"read": {
"count": 2,
"samples_ms": [1, 2],
"median_ms": 1.5,
"p95_ms": 2,
"max_ms": 2,
}
},
}
def mini_test(self, relative, source):
target = self.root / relative
target.parent.mkdir(parents=True, exist_ok=True)
directory = target.parent
while directory != self.root:
(directory / "__init__.py").write_text("", encoding="utf-8")
directory = directory.parent
target.write_text(source, encoding="utf-8")
return target
def run_mini(self, output):
with (
patch.object(
CI,
"__file__",
str(self.root / "scripts/ci/performance_pytest.py"),
),
patch.object(
CI.sys, "argv", ["performance_pytest", "--output", str(output)]
),
):
CI.main()
def xml(self, body):
path = self.root / "results.xml"
path.write_text(
f"<testsuites><testsuite>{body}</testsuite></testsuites>",
encoding="utf-8",
)
return path
def test_empty_checkout_is_not_a_successful_benchmark_run(self):
with self.assertRaises(ValueError):
CI.available_suites(self.root)
def test_cache_directories_do_not_create_phantom_suites_or_core_targets(
self,
):
self.mini_test(
"tests/performance/database/test_connection_workload.py", ""
)
for target in (
"content_extraction",
"token_tracking",
"rag_scale_smoke",
):
(self.root / "tests/performance" / target / "__pycache__").mkdir(
parents=True
)
self.assertEqual(
CI.available_suites(self.root),
[
(
"tests/performance/database/test_connection_workload.py",
None,
),
],
)
def test_each_selected_suite_must_record_passing_assertions(self):
path = self.xml(
'<testcase classname="tests.performance.chat_context.test_offline" name="check"/>'
)
suites = [("tests/performance/chat_context", "chatctx")]
self.assertEqual(CI.require_junit(path, suites), {suites[0][0]: 1})
with self.assertRaises(ValueError):
CI.require_junit(
path, suites + [("tests/performance/token_tracking", "tokens")]
)
def test_no_collection_skips_and_failures_are_rejected(self):
suites = [("tests/performance/chat_context", "chatctx")]
for body in [
"",
*[
'<testcase classname="tests.performance.chat_context.test_offline">'
f"<{tag}/></testcase>"
for tag in ["skipped", "failure", "error"]
],
]:
with self.subTest(body=body), self.assertRaises(ValueError):
CI.require_junit(self.xml(body), suites)
def test_failed_or_unverified_measurements_are_rejected(self):
path = self.root / "report.json"
good = self.good_report()
path.write_text(json.dumps(good), encoding="utf-8")
self.assertEqual(CI.require_report(path, "chatctx", "ci"), 1)
bad = []
for key, value in [
("status", "failed"),
("results", []),
("area", "tokens"),
("schema_version", 1),
]:
report = deepcopy(good)
report[key] = value
bad.append(report)
report = deepcopy(good)
report["settings"]["profile"] = "local"
bad.append(report)
for key, value in [
("checks", {}),
("count", 0),
("count", True),
("samples_ns", []),
("samples_ms", [1, float("nan")]),
("samples_ns", [1_000_000, float("inf")]),
("p95_ms", 1),
("median_ms", float("nan")),
]:
report = deepcopy(good)
report["results"][0][key] = value
bad.append(report)
for report in bad:
path.write_text(json.dumps(report), encoding="utf-8")
with self.subTest(report=report), self.assertRaises(ValueError):
CI.require_report(path, "chatctx", "ci")
def test_native_measurements_are_retained_and_require_both_backends(self):
temporary = self.root / "pytest"
temporary.mkdir()
output = self.root / "artifacts"
suites = [("tests/performance/rag/test_storage_workload.py", None)]
first = temporary / "flat" / "isolated-result.json"
first.parent.mkdir()
first.write_text(
json.dumps(self.native_report("rag_performance", "flat"))
)
with self.assertRaisesRegex(ValueError, "backend"):
CI.collect_native_reports(temporary, suites, output)
second = temporary / "hnsw" / "isolated-result.json"
second.parent.mkdir()
second.write_text(
json.dumps(self.native_report("rag_performance", "hnsw"))
)
result = CI.collect_native_reports(temporary, suites, output)
self.assertEqual(
{r["index_type"] for r in result["rag_performance"]},
{"flat", "hnsw"},
)
first.unlink()
second.unlink()
for record in result["rag_performance"]:
report = json.loads(Path(record["path"]).read_text())
self.assertEqual(
report["measurements"]["read"]["samples_ms"], [1, 2]
)
self.assertEqual(record["configuration"]["rows"], 8)
def test_missing_or_invalid_native_measurements_are_rejected(self):
with self.assertRaisesRegex(ValueError, "Missing native"):
CI.collect_native_reports(
self.root,
[
(
"tests/performance/database/test_connection_workload.py",
None,
)
],
self.root / "output",
)
for key, value in [("configuration", {}), ("measurements", {})]:
report = self.native_report()
report[key] = value
with self.subTest(key=key), self.assertRaises(ValueError):
CI.require_native_report(report)
def test_quality_reports_require_successful_nonempty_queries(self):
good = {
"topic": "search_quality",
"configuration": {"profile": "ci"},
"quality_failures": [],
"results": [{"queries": [{"recall_at_k": 1.0, "returned": 3}]}],
}
self.assertEqual(CI.require_native_report(good), 1)
for report in [
dict(good, quality_failures=["cosine/reopen"]),
dict(good, results=[]),
dict(good, results=[{"queries": []}]),
dict(
good,
results=[
{"queries": [{"recall_at_k": float("nan"), "returned": 3}]}
],
),
]:
with self.subTest(report=report), self.assertRaises(ValueError):
CI.require_native_report(report)
def test_existing_manual_workflow_contracts_are_also_selected(self):
target = (
self.root / "tests/performance/database/test_connection_workload.py"
)
target.parent.mkdir(parents=True)
target.touch()
contracts = self.root / "tests/ci/test_offline_workload_shards.py"
contracts.parent.mkdir()
contracts.touch()
self.assertIn(
("tests/ci/test_offline_workload_shards.py", None),
CI.available_suites(self.root),
)
def test_ci_uses_one_worker_and_explicit_profile_per_present_area(self):
command = CI.pytest_command(
self.root,
[("tests/performance/chat_context", "chatctx")],
"ci",
self.root,
)
self.assertEqual(command[command.index("-n") + 1], "0")
self.assertEqual(command[command.index("--chatctx-profile") + 1], "ci")
self.assertIn("--chatctx-report", command)
def test_real_child_does_not_silently_skip_integration_assertions(self):
target = self.mini_test(
"tests/performance/database/test_connection_workload.py",
"import json\nimport os\nimport pytest\n"
"@pytest.fixture(autouse=True)\n"
"def real_mode():\n"
' if os.environ.get("LDR_TESTING_WITH_MOCKS") != "false":\n'
' pytest.skip("integration disabled")\n'
"def test_required_assertion(tmp_path):\n"
" assert __debug__\n"
' assert os.environ["LDR_PERF_PROFILE"] == "ci"\n'
' assert os.environ["OMP_NUM_THREADS"] == "1"\n',
)
target.write_text(
target.read_text()
+ ' (tmp_path / "isolated-result.json").write_text('
+ repr(json.dumps(self.native_report()))
+ ")\n"
)
output = self.root / "results"
self.run_mini(output)
result = json.loads(
(output / "ci-summary.json").read_text(encoding="utf-8")
)
self.assertEqual(result["status"], "passed")
self.assertEqual(
result["asserted_tests"][str(target.relative_to(self.root))], 1
)
native = result["native_reports"]["database_performance"]
self.assertEqual(len(native), 1)
self.assertTrue(Path(native[0]["path"]).is_file())
self.assertFalse(
(
output.parent
/ Path(
result["command"][result["command"].index("--basetemp") + 1]
)
).exists()
)
def test_reused_output_cannot_substitute_previous_measurements(self):
target = self.mini_test(
"tests/performance/chat_context/test_measurement.py",
"import os\nfrom pathlib import Path\n"
"def test_measurement(request):\n"
" assert 2 + 2 == 4\n"
' if os.environ.get("WRITE_BENCHMARK_REPORT") == "yes":\n'
' Path(request.config.getoption("--chatctx-report")).write_text('
+ repr(json.dumps(self.good_report()))
+ ")\n",
)
(target.parent / "conftest.py").write_text(
"def pytest_addoption(parser):\n"
' parser.addoption("--chatctx-profile")\n'
' parser.addoption("--chatctx-report")\n'
)
output = self.root / "results"
with patch.dict("os.environ", {"WRITE_BENCHMARK_REPORT": "yes"}):
self.run_mini(output)
first = json.loads((output / "ci-summary.json").read_text())
with (
patch.dict("os.environ", {"WRITE_BENCHMARK_REPORT": "no"}),
self.assertRaises(FileNotFoundError),
):
self.run_mini(output)
second = json.loads((output / "ci-summary.json").read_text())
self.assertEqual(second["status"], "failed")
self.assertNotEqual(first["run_directory"], second["run_directory"])
self.assertTrue(
(Path(first["run_directory"]) / "chatctx.json").exists()
)
if __name__ == "__main__":
unittest.main()