~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""Configuration precedence and retained diagnostics for offline workloads."""
# allow: no-sut-import — tests the performance runner, not application behavior.
import json
import sys
from pathlib import Path
from types import SimpleNamespace
import pytest
from tests.performance._shared.pytest_workload import Workload, resolve
from tests.performance._shared import pytest_workload as runner
PROFILES = {
"ci": dict(size=2, iterations=1, warmups=0),
"local": dict(size=10, iterations=3, warmups=1),
}
def config(**options):
return SimpleNamespace(getoption=lambda name: options.get(name))
def test_cli_overrides_area_environment_and_global_profile(monkeypatch):
monkeypatch.setenv("LDR_PERF_PROFILE", "local")
monkeypatch.setenv("LDR_PERF_DEMO_PROFILE", "ci")
monkeypatch.setenv("LDR_PERF_DEMO_SIZE", "7")
settings = resolve(config(**{"--demo-size": "4"}), "demo", PROFILES)
assert settings["profile"] == "ci"
assert settings["size"] == 4
assert settings["iterations"] == 1
def test_empty_environment_preserves_profile_defaults(monkeypatch):
monkeypatch.setenv("LDR_PERF_PROFILE", "")
monkeypatch.setenv("LDR_PERF_DEMO_PROFILE", "")
monkeypatch.setenv("LDR_PERF_DEMO_SIZE", "")
settings = resolve(config(), "demo", PROFILES)
assert settings["profile"] == "ci"
assert settings["size"] == 2
@pytest.mark.parametrize(
"field,value",
[("size", "0"), ("iterations", "many"), ("max-p95-ms", "nan")],
)
def test_invalid_sizes_and_budgets_fail_before_running(field, value):
with pytest.raises(pytest.UsageError):
resolve(config(**{f"--demo-{field}": value}), "demo", PROFILES)
def test_failed_timing_budget_keeps_raw_samples(tmp_path, monkeypatch):
path = tmp_path / "report.json"
settings = resolve(config(**{"--demo-profile": "ci"}), "demo", PROFILES)
settings.update(report=str(path), max_p95_ms=1)
workload = Workload("demo", settings)
clock = Clock(monkeypatch)
workload.measure(
"measured",
1,
lambda: None,
lambda _: clock.advance(10_000_000),
lambda *_: {},
)
with pytest.raises(pytest.fail.Exception, match="p95 budget"):
workload.finish()
report = json.loads(path.read_text())
assert report["budget_failures"] == ["measured[1]"]
assert report["results"][0]["samples_ms"] == [10]
assert report["status"] == "failed"
def test_correctness_failure_keeps_case_diagnostics(tmp_path):
settings = resolve(config(**{"--demo-profile": "ci"}), "demo", PROFILES)
settings["report"] = str(tmp_path / "incorrect.json")
workload = Workload("demo", settings)
def verify(_state, output):
assert output == "expected"
with pytest.raises(AssertionError):
workload.measure(
"incorrect", 1, lambda: None, lambda _: "wrong", verify
)
workload.finish()
report = json.loads(Path(settings["report"]).read_text())
assert report["results"][0]["status"] == "failed"
assert report["results"][0]["failure_phase"] == "verify"
assert report["status"] == "failed"
assert len(report["results"][0]["samples_ms"]) == 1
class Clock:
def __init__(self, monkeypatch):
self.value = 0
monkeypatch.setattr(runner.time, "perf_counter_ns", lambda: self.value)
monkeypatch.setattr(
runner.time, "perf_counter", lambda: self.value / 1_000_000_000
)
def advance(self, nanoseconds):
self.value += nanoseconds
def test_setup_checks_warmups_and_prior_result_release_are_untimed(monkeypatch):
clock = Clock(monkeypatch)
settings = resolve(config(**{"--demo-profile": "ci"}), "demo", PROFILES)
settings.update(iterations=2, warmups=1)
workload = Workload("demo", settings)
released = []
durations = iter((99_000_000, 5_000_000, 7_000_000))
class Result:
def __del__(self):
clock.advance(50_000_000)
released.append(True)
def prepare():
clock.advance(100_000_000)
def operation(_state):
clock.advance(next(durations))
return Result()
def verify(_state, _output):
clock.advance(200_000_000)
return {"checked": True}
workload.measure("timed", 1, prepare, operation, verify)
assert workload.results[0]["samples_ms"] == [5, 7]
assert workload.results[0]["count"] == 2
assert len(released) == 3
def test_rates_and_scaling_use_declared_work_units(monkeypatch):
clock = Clock(monkeypatch)
workload = Workload(
"demo", resolve(config(**{"--demo-profile": "ci"}), "demo", PROFILES)
)
for size in (2, 4):
workload.measure(
"scaled",
size,
lambda: None,
lambda _: clock.advance(size * 200),
lambda *_: {},
units=size,
unit="lookup",
)
report = workload.finish()
assert [point["median_ns_per_unit"] for point in report["results"]] == [
200,
200,
]
assert [point["units_per_second"] for point in report["results"]] == [
5_000_000,
5_000_000,
]
assert report["observed_scaling"][0]["normalized_latency_ratio"] == 1
def test_missing_package_metadata_preserves_measurements(tmp_path, monkeypatch):
clock = Clock(monkeypatch)
settings = resolve(config(**{"--demo-profile": "ci"}), "demo", PROFILES)
settings["report"] = str(tmp_path / "report.json")
workload = Workload("demo", settings)
workload.measure(
"measured", 1, lambda: None, lambda _: clock.advance(100), lambda *_: {}
)
def unavailable(_name):
raise runner.PackageNotFoundError
monkeypatch.setattr(runner, "version", unavailable)
workload.finish(("optional-dependency",))
report = json.loads(Path(settings["report"]).read_text())
assert report["environment"]["packages"] == {"optional-dependency": None}
assert report["results"][0]["samples_ns"] == [100]
def test_untracked_loaded_code_is_not_stamped_with_harness_revision(
tmp_path, monkeypatch
):
implementation = tmp_path / "implementation.py"
implementation.write_text("value = 42\n")
monkeypatch.setitem(
sys.modules,
"workload_demo_implementation",
SimpleNamespace(__file__=str(implementation)),
)
workload = Workload(
"demo", resolve(config(**{"--demo-profile": "ci"}), "demo", PROFILES)
)
report = workload.finish(modules=("workload_demo_implementation",))
loaded = report["environment"]["loaded_sources"][
"workload_demo_implementation"
]
assert loaded["path"] == str(implementation)
assert loaded["revision"] is None
assert len(loaded["sha256"]) == 64
assert report["environment"]["source_revision"] is None