~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""Small pytest workloads with explicit profiles and optional timing budgets."""
from dataclasses import dataclass
import hashlib
from importlib.metadata import PackageNotFoundError, version
import json
import math
import os
from pathlib import Path
import platform
import statistics
import subprocess
import sys
import time
import pytest
def add_options(parser, area):
group = parser.getgroup(f"{area} performance")
for field in (
"profile",
"size",
"iterations",
"warmups",
"report",
"max-p95-ms",
):
group.addoption(f"--{area}-{field}", default=None)
def resolve(config, area, profiles):
def value(field, default):
cli = config.getoption(f"--{area}-{field}")
key = field.replace("-", "_").upper()
if cli is not None:
return cli
environment = os.environ.get(f"LDR_PERF_{area.upper()}_{key}")
return environment if environment not in (None, "") else default
profile = value("profile", os.environ.get("LDR_PERF_PROFILE") or "ci")
if profile not in profiles:
raise pytest.UsageError("Performance profile must be ci or local")
result = {"profile": profile}
for field in ("size", "iterations", "warmups"):
try:
number = int(value(field, profiles[profile][field]))
except (TypeError, ValueError):
raise pytest.UsageError(
f"{area} {field} must be an integer"
) from None
if number < (0 if field == "warmups" else 1):
raise pytest.UsageError(f"Invalid {area} {field}")
result[field] = number
result["report"] = value("report", None)
limit = value("max-p95-ms", None)
if limit is not None:
try:
limit = float(limit)
except (TypeError, ValueError):
raise pytest.UsageError("Timing budget must be numeric") from None
if not math.isfinite(limit) or limit <= 0:
raise pytest.UsageError("Timing budget must be positive and finite")
result["max_p95_ms"] = limit
return result
@dataclass
class Workload:
area: str
settings: dict
def __post_init__(self):
self.results = []
def measure(
self,
name,
size,
prepare,
operation,
verify,
*,
units=1,
unit="batch",
parameters=None,
):
"""Time only the operation, excluding setup, checks and prior outputs."""
if not isinstance(units, int) or isinstance(units, bool) or units < 1:
raise ValueError("Work units must be a positive integer")
samples = []
result = dict(
case=name,
size=size,
unit=unit,
units=units,
parameters=parameters or {},
samples_ns=samples,
samples_ms=[],
status="running",
)
self.results.append(result)
phase = "prepare"
try:
total = self.settings["warmups"] + self.settings["iterations"]
for number in range(total):
state = output = None
try:
phase = "prepare"
state = prepare()
phase = "operation"
begun = time.perf_counter_ns()
output = operation(state)
elapsed = time.perf_counter_ns() - begun
if elapsed < 0:
raise ValueError("Negative benchmark duration")
if number >= self.settings["warmups"]:
samples.append(elapsed)
result["samples_ms"].append(elapsed / 1_000_000)
phase = "verify"
result["checks"] = verify(state, output)
finally:
# Replacing last iteration's result inside the next
# timed assignment charged its destruction to that call.
output = state = None
except BaseException as error:
result.update(
status="failed",
error_type=type(error).__name__,
failure_phase=phase,
)
raise
ordered = sorted(samples)
median = statistics.median(samples)
p95 = ordered[math.ceil(0.95 * len(ordered)) - 1]
result.update(
status="passed",
count=len(samples),
median_ms=median / 1_000_000,
p95_ms=p95 / 1_000_000,
median_ns_per_unit=median / units,
p95_ns_per_unit=p95 / units,
units_per_second=units * 1_000_000_000 / median if median else None,
)
def finish(self, packages=(), *, modules=()):
root = Path(__file__).resolve().parents[3]
def git(*arguments, cwd=root):
try:
completed = subprocess.run(
["git", *arguments],
cwd=cwd,
capture_output=True,
text=True,
)
except OSError:
return None
return (
completed.stdout.strip() if completed.returncode == 0 else None
)
revision, dirty = git("rev-parse", "HEAD"), git("status", "--porcelain")
loaded_sources = {}
for name in modules:
filename = getattr(sys.modules.get(name), "__file__", None)
path = Path(filename).resolve() if filename else None
tracked = (
path is not None
and git(
"ls-files",
"--error-unmatch",
"--",
str(path),
cwd=path.parent,
)
is not None
)
try:
digest = (
hashlib.sha256(path.read_bytes()).hexdigest()
if path
else None
)
except OSError:
digest = None
source_dirty = (
git("status", "--porcelain", cwd=path.parent)
if tracked
else None
)
loaded_sources[name] = dict(
path=str(path) if path else None,
revision=git("rev-parse", "HEAD", cwd=path.parent)
if tracked
else None,
dirty=bool(source_dirty) if source_dirty is not None else None,
sha256=digest,
)
source_revisions = {
value["revision"] for value in loaded_sources.values()
}
versions = {}
for name in packages:
try:
versions[name] = version(name)
except PackageNotFoundError:
versions[name] = None
budget = self.settings["max_p95_ms"]
failures = [
f"{result['case']}[{result['size']}]"
for result in self.results
if budget is not None and result.get("p95_ms", 0) > budget
]
curves = {}
for result in self.results:
if result["status"] == "passed":
curves.setdefault((result["case"], result["unit"]), []).append(
result
)
scaling = []
for (name, unit), points in curves.items():
points.sort(key=lambda point: point["size"])
for smaller, larger in zip(points, points[1:]):
if (
larger["size"] > smaller["size"]
and smaller["median_ms"] > 0
):
latency = larger["median_ms"] / smaller["median_ms"]
work = larger["units"] / smaller["units"]
scaling.append(
dict(
case=name,
unit=unit,
sizes=[smaller["size"], larger["size"]],
latency_ratio=latency,
work_ratio=work,
normalized_latency_ratio=latency / work,
)
)
passed = bool(self.results) and all(
result["status"] == "passed" for result in self.results
)
report = dict(
schema_version=2,
area=self.area,
status="passed" if passed and not failures else "failed",
settings=self.settings,
results=self.results,
observed_scaling=scaling,
budget_failures=failures,
environment=dict(
python=platform.python_version(),
platform=platform.platform(),
processor=platform.processor(),
cpu_count=os.cpu_count(),
native_threads={
name: os.environ.get(name)
for name in (
"OMP_NUM_THREADS",
"OPENBLAS_NUM_THREADS",
"MKL_NUM_THREADS",
)
},
pytest_worker=os.environ.get("PYTEST_XDIST_WORKER"),
timing_scope="operation in pytest process; preparation, verification and result release excluded",
rate_scope="batch duration divided by declared work; per-unit p95 is not an individual-request quantile",
harness_revision=revision,
harness_dirty=bool(dirty) if dirty is not None else None,
source_revision=next(iter(source_revisions))
if len(source_revisions) == 1
else None,
source_dirty=any(
value["dirty"] for value in loaded_sources.values()
)
if loaded_sources
and all(
value["dirty"] is not None
for value in loaded_sources.values()
)
else None,
loaded_sources=loaded_sources,
packages=versions,
),
)
if self.settings["report"]:
path = Path(self.settings["report"])
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(
json.dumps(report, indent=2) + "\n", encoding="utf-8"
)
if failures:
pytest.fail("Explicit p95 budget exceeded: " + ", ".join(failures))
return report