Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

6.6 KB
Raw Blame
"""Configuration precedence and retained diagnostics for offline workloads."""
# allow: no-sut-import — tests the performance runner, not application behavior.

import json
import sys
from pathlib import Path
from types import SimpleNamespace

import pytest

from tests.performance._shared.pytest_workload import Workload, resolve
from tests.performance._shared import pytest_workload as runner

PROFILES = {
    "ci": dict(size=2, iterations=1, warmups=0),
    "local": dict(size=10, iterations=3, warmups=1),
}


def config(**options):
    return SimpleNamespace(getoption=lambda name: options.get(name))


def test_cli_overrides_area_environment_and_global_profile(monkeypatch):
    monkeypatch.setenv("LDR_PERF_PROFILE", "local")
    monkeypatch.setenv("LDR_PERF_DEMO_PROFILE", "ci")
    monkeypatch.setenv("LDR_PERF_DEMO_SIZE", "7")
    settings = resolve(config(**{"--demo-size": "4"}), "demo", PROFILES)
    assert settings["profile"] == "ci"
    assert settings["size"] == 4
    assert settings["iterations"] == 1


def test_empty_environment_preserves_profile_defaults(monkeypatch):
    monkeypatch.setenv("LDR_PERF_PROFILE", "")
    monkeypatch.setenv("LDR_PERF_DEMO_PROFILE", "")
    monkeypatch.setenv("LDR_PERF_DEMO_SIZE", "")
    settings = resolve(config(), "demo", PROFILES)
    assert settings["profile"] == "ci"
    assert settings["size"] == 2


@pytest.mark.parametrize(
    "field,value",
    [("size", "0"), ("iterations", "many"), ("max-p95-ms", "nan")],
)
def test_invalid_sizes_and_budgets_fail_before_running(field, value):
    with pytest.raises(pytest.UsageError):
        resolve(config(**{f"--demo-{field}": value}), "demo", PROFILES)


def test_failed_timing_budget_keeps_raw_samples(tmp_path, monkeypatch):
    path = tmp_path / "report.json"
    settings = resolve(config(**{"--demo-profile": "ci"}), "demo", PROFILES)
    settings.update(report=str(path), max_p95_ms=1)
    workload = Workload("demo", settings)
    clock = Clock(monkeypatch)
    workload.measure(
        "measured",
        1,
        lambda: None,
        lambda _: clock.advance(10_000_000),
        lambda *_: {},
    )
    with pytest.raises(pytest.fail.Exception, match="p95 budget"):
        workload.finish()
    report = json.loads(path.read_text())
    assert report["budget_failures"] == ["measured[1]"]
    assert report["results"][0]["samples_ms"] == [10]
    assert report["status"] == "failed"


def test_correctness_failure_keeps_case_diagnostics(tmp_path):
    settings = resolve(config(**{"--demo-profile": "ci"}), "demo", PROFILES)
    settings["report"] = str(tmp_path / "incorrect.json")
    workload = Workload("demo", settings)

    def verify(_state, output):
        assert output == "expected"

    with pytest.raises(AssertionError):
        workload.measure(
            "incorrect", 1, lambda: None, lambda _: "wrong", verify
        )
    workload.finish()
    report = json.loads(Path(settings["report"]).read_text())
    assert report["results"][0]["status"] == "failed"
    assert report["results"][0]["failure_phase"] == "verify"
    assert report["status"] == "failed"
    assert len(report["results"][0]["samples_ms"]) == 1


class Clock:
    def __init__(self, monkeypatch):
        self.value = 0
        monkeypatch.setattr(runner.time, "perf_counter_ns", lambda: self.value)
        monkeypatch.setattr(
            runner.time, "perf_counter", lambda: self.value / 1_000_000_000
        )

    def advance(self, nanoseconds):
        self.value += nanoseconds


def test_setup_checks_warmups_and_prior_result_release_are_untimed(monkeypatch):
    clock = Clock(monkeypatch)
    settings = resolve(config(**{"--demo-profile": "ci"}), "demo", PROFILES)
    settings.update(iterations=2, warmups=1)
    workload = Workload("demo", settings)
    released = []
    durations = iter((99_000_000, 5_000_000, 7_000_000))

    class Result:
        def __del__(self):
            clock.advance(50_000_000)
            released.append(True)

    def prepare():
        clock.advance(100_000_000)

    def operation(_state):
        clock.advance(next(durations))
        return Result()

    def verify(_state, _output):
        clock.advance(200_000_000)
        return {"checked": True}

    workload.measure("timed", 1, prepare, operation, verify)
    assert workload.results[0]["samples_ms"] == [5, 7]
    assert workload.results[0]["count"] == 2
    assert len(released) == 3


def test_rates_and_scaling_use_declared_work_units(monkeypatch):
    clock = Clock(monkeypatch)
    workload = Workload(
        "demo", resolve(config(**{"--demo-profile": "ci"}), "demo", PROFILES)
    )
    for size in (2, 4):
        workload.measure(
            "scaled",
            size,
            lambda: None,
            lambda _: clock.advance(size * 200),
            lambda *_: {},
            units=size,
            unit="lookup",
        )
    report = workload.finish()
    assert [point["median_ns_per_unit"] for point in report["results"]] == [
        200,
        200,
    ]
    assert [point["units_per_second"] for point in report["results"]] == [
        5_000_000,
        5_000_000,
    ]
    assert report["observed_scaling"][0]["normalized_latency_ratio"] == 1


def test_missing_package_metadata_preserves_measurements(tmp_path, monkeypatch):
    clock = Clock(monkeypatch)
    settings = resolve(config(**{"--demo-profile": "ci"}), "demo", PROFILES)
    settings["report"] = str(tmp_path / "report.json")
    workload = Workload("demo", settings)
    workload.measure(
        "measured", 1, lambda: None, lambda _: clock.advance(100), lambda *_: {}
    )

    def unavailable(_name):
        raise runner.PackageNotFoundError

    monkeypatch.setattr(runner, "version", unavailable)
    workload.finish(("optional-dependency",))
    report = json.loads(Path(settings["report"]).read_text())
    assert report["environment"]["packages"] == {"optional-dependency": None}
    assert report["results"][0]["samples_ns"] == [100]


def test_untracked_loaded_code_is_not_stamped_with_harness_revision(
    tmp_path, monkeypatch
):
    implementation = tmp_path / "implementation.py"
    implementation.write_text("value = 42\n")
    monkeypatch.setitem(
        sys.modules,
        "workload_demo_implementation",
        SimpleNamespace(__file__=str(implementation)),
    )
    workload = Workload(
        "demo", resolve(config(**{"--demo-profile": "ci"}), "demo", PROFILES)
    )
    report = workload.finish(modules=("workload_demo_implementation",))
    loaded = report["environment"]["loaded_sources"][
        "workload_demo_implementation"
    ]
    assert loaded["path"] == str(implementation)
    assert loaded["revision"] is None
    assert len(loaded["sha256"]) == 64
    assert report["environment"]["source_revision"] is None