Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

10.3 KB
Raw Blame
"""Small pytest workloads with explicit profiles and optional timing budgets."""

from dataclasses import dataclass
import hashlib
from importlib.metadata import PackageNotFoundError, version
import json
import math
import os
from pathlib import Path
import platform
import statistics
import subprocess
import sys
import time

import pytest


def add_options(parser, area):
    group = parser.getgroup(f"{area} performance")
    for field in (
        "profile",
        "size",
        "iterations",
        "warmups",
        "report",
        "max-p95-ms",
    ):
        group.addoption(f"--{area}-{field}", default=None)


def resolve(config, area, profiles):
    def value(field, default):
        cli = config.getoption(f"--{area}-{field}")
        key = field.replace("-", "_").upper()
        if cli is not None:
            return cli
        environment = os.environ.get(f"LDR_PERF_{area.upper()}_{key}")
        return environment if environment not in (None, "") else default

    profile = value("profile", os.environ.get("LDR_PERF_PROFILE") or "ci")
    if profile not in profiles:
        raise pytest.UsageError("Performance profile must be ci or local")
    result = {"profile": profile}
    for field in ("size", "iterations", "warmups"):
        try:
            number = int(value(field, profiles[profile][field]))
        except (TypeError, ValueError):
            raise pytest.UsageError(
                f"{area} {field} must be an integer"
            ) from None
        if number < (0 if field == "warmups" else 1):
            raise pytest.UsageError(f"Invalid {area} {field}")
        result[field] = number
    result["report"] = value("report", None)
    limit = value("max-p95-ms", None)
    if limit is not None:
        try:
            limit = float(limit)
        except (TypeError, ValueError):
            raise pytest.UsageError("Timing budget must be numeric") from None
        if not math.isfinite(limit) or limit <= 0:
            raise pytest.UsageError("Timing budget must be positive and finite")
    result["max_p95_ms"] = limit
    return result


@dataclass
class Workload:
    area: str
    settings: dict

    def __post_init__(self):
        self.results = []

    def measure(
        self,
        name,
        size,
        prepare,
        operation,
        verify,
        *,
        units=1,
        unit="batch",
        parameters=None,
    ):
        """Time only the operation, excluding setup, checks and prior outputs."""
        if not isinstance(units, int) or isinstance(units, bool) or units < 1:
            raise ValueError("Work units must be a positive integer")
        samples = []
        result = dict(
            case=name,
            size=size,
            unit=unit,
            units=units,
            parameters=parameters or {},
            samples_ns=samples,
            samples_ms=[],
            status="running",
        )
        self.results.append(result)
        phase = "prepare"
        try:
            total = self.settings["warmups"] + self.settings["iterations"]
            for number in range(total):
                state = output = None
                try:
                    phase = "prepare"
                    state = prepare()
                    phase = "operation"
                    begun = time.perf_counter_ns()
                    output = operation(state)
                    elapsed = time.perf_counter_ns() - begun
                    if elapsed < 0:
                        raise ValueError("Negative benchmark duration")
                    if number >= self.settings["warmups"]:
                        samples.append(elapsed)
                        result["samples_ms"].append(elapsed / 1_000_000)
                    phase = "verify"
                    result["checks"] = verify(state, output)
                finally:
                    # Replacing last iteration's result inside the next
                    # timed assignment charged its destruction to that call.
                    output = state = None
        except BaseException as error:
            result.update(
                status="failed",
                error_type=type(error).__name__,
                failure_phase=phase,
            )
            raise
        ordered = sorted(samples)
        median = statistics.median(samples)
        p95 = ordered[math.ceil(0.95 * len(ordered)) - 1]
        result.update(
            status="passed",
            count=len(samples),
            median_ms=median / 1_000_000,
            p95_ms=p95 / 1_000_000,
            median_ns_per_unit=median / units,
            p95_ns_per_unit=p95 / units,
            units_per_second=units * 1_000_000_000 / median if median else None,
        )

    def finish(self, packages=(), *, modules=()):
        root = Path(__file__).resolve().parents[3]

        def git(*arguments, cwd=root):
            try:
                completed = subprocess.run(
                    ["git", *arguments],
                    cwd=cwd,
                    capture_output=True,
                    text=True,
                )
            except OSError:
                return None
            return (
                completed.stdout.strip() if completed.returncode == 0 else None
            )

        revision, dirty = git("rev-parse", "HEAD"), git("status", "--porcelain")
        loaded_sources = {}
        for name in modules:
            filename = getattr(sys.modules.get(name), "__file__", None)
            path = Path(filename).resolve() if filename else None
            tracked = (
                path is not None
                and git(
                    "ls-files",
                    "--error-unmatch",
                    "--",
                    str(path),
                    cwd=path.parent,
                )
                is not None
            )
            try:
                digest = (
                    hashlib.sha256(path.read_bytes()).hexdigest()
                    if path
                    else None
                )
            except OSError:
                digest = None
            source_dirty = (
                git("status", "--porcelain", cwd=path.parent)
                if tracked
                else None
            )
            loaded_sources[name] = dict(
                path=str(path) if path else None,
                revision=git("rev-parse", "HEAD", cwd=path.parent)
                if tracked
                else None,
                dirty=bool(source_dirty) if source_dirty is not None else None,
                sha256=digest,
            )
        source_revisions = {
            value["revision"] for value in loaded_sources.values()
        }
        versions = {}
        for name in packages:
            try:
                versions[name] = version(name)
            except PackageNotFoundError:
                versions[name] = None
        budget = self.settings["max_p95_ms"]
        failures = [
            f"{result['case']}[{result['size']}]"
            for result in self.results
            if budget is not None and result.get("p95_ms", 0) > budget
        ]
        curves = {}
        for result in self.results:
            if result["status"] == "passed":
                curves.setdefault((result["case"], result["unit"]), []).append(
                    result
                )
        scaling = []
        for (name, unit), points in curves.items():
            points.sort(key=lambda point: point["size"])
            for smaller, larger in zip(points, points[1:]):
                if (
                    larger["size"] > smaller["size"]
                    and smaller["median_ms"] > 0
                ):
                    latency = larger["median_ms"] / smaller["median_ms"]
                    work = larger["units"] / smaller["units"]
                    scaling.append(
                        dict(
                            case=name,
                            unit=unit,
                            sizes=[smaller["size"], larger["size"]],
                            latency_ratio=latency,
                            work_ratio=work,
                            normalized_latency_ratio=latency / work,
                        )
                    )
        passed = bool(self.results) and all(
            result["status"] == "passed" for result in self.results
        )
        report = dict(
            schema_version=2,
            area=self.area,
            status="passed" if passed and not failures else "failed",
            settings=self.settings,
            results=self.results,
            observed_scaling=scaling,
            budget_failures=failures,
            environment=dict(
                python=platform.python_version(),
                platform=platform.platform(),
                processor=platform.processor(),
                cpu_count=os.cpu_count(),
                native_threads={
                    name: os.environ.get(name)
                    for name in (
                        "OMP_NUM_THREADS",
                        "OPENBLAS_NUM_THREADS",
                        "MKL_NUM_THREADS",
                    )
                },
                pytest_worker=os.environ.get("PYTEST_XDIST_WORKER"),
                timing_scope="operation in pytest process; preparation, verification and result release excluded",
                rate_scope="batch duration divided by declared work; per-unit p95 is not an individual-request quantile",
                harness_revision=revision,
                harness_dirty=bool(dirty) if dirty is not None else None,
                source_revision=next(iter(source_revisions))
                if len(source_revisions) == 1
                else None,
                source_dirty=any(
                    value["dirty"] for value in loaded_sources.values()
                )
                if loaded_sources
                and all(
                    value["dirty"] is not None
                    for value in loaded_sources.values()
                )
                else None,
                loaded_sources=loaded_sources,
                packages=versions,
            ),
        )
        if self.settings["report"]:
            path = Path(self.settings["report"])
            path.parent.mkdir(parents=True, exist_ok=True)
            path.write_text(
                json.dumps(report, indent=2) + "\n", encoding="utf-8"
            )
        if failures:
            pytest.fail("Explicit p95 budget exceeded: " + ", ".join(failures))
        return report