Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++
23.5 KB
Raw Blame
"""Running one engine configuration and measuring it, for the tuner.

A *session* is what the tuner needs from the outside world: start an engine with these arguments and environment, run the
fixed request set, report numbers, stop. The real session (`WorkerSession`) starts Strata's own worker (``serve.server``)
on a **copy** of the user's run configuration and talks to it over HTTP, so the engine, the queueing and the prompt
handling are exactly the ones the user gets. The numbers come from the engine's own clock (the ``timings`` block of the
reply), not from this process's stopwatch. The stub session in the tests implements the same two methods.

Never invents a number: an engine that does not start, answers nothing or answers too little raises `StartFailed` or
returns a result whose ``status`` says why, and the tuner records that as the candidate's result.
"""

from __future__ import annotations

import contextlib
import json
import os
import random
import re
import secrets
import socket
import statistics
import subprocess
import sys
import tempfile
import threading
import time
import urllib.error
import urllib.request
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, Callable, Iterator, Protocol

from strata_control import loopback, tune_machine

# --------------------------------------------------------------------------- the contract


class StartFailed(RuntimeError):
    """The engine did not come up. ``status``: FAIL_BOOT | OOM | UNSUPPORTED | TIMEOUT."""

    def __init__(self, status: str, reason: str) -> None:
        super().__init__(reason)
        self.status = status
        self.reason = reason


@dataclass
class RequestSet:
    """The fixed work every run does, so two configurations are compared on the same requests."""

    warmups: int = 1
    decodes: int = 3
    decode_tokens: int = 200
    pp_tokens: int = 4096
    long_tokens: int = 0              # 0 = no long prompt


@dataclass
class RunResult:
    status: str = "PASS"              # PASS | FAIL_BOOT | OOM | UNSUPPORTED | TIMEOUT | NO_TOKENS
    reason: str = ""
    load_s: float | None = None
    decode_tok_s: float | None = None          # median of the measured decodes
    decode_runs: list[float] = field(default_factory=list)
    pp_tok_s: float | None = None              # the warm prompt (the second one; the first is cold)
    pp_cold_tok_s: float | None = None
    ttft_s: float | None = None
    pp_long_tok_s: float | None = None
    rss_peak_mib: float | None = None
    vram_used_mib: float | None = None         # this run's total graphics memory in use (all cards, absolute)
    vram_added_mib: float | None = None        # ... minus what was in use before the engine started
    min_avail_mib: float | None = None         # lowest free system RAM seen while the engine ran
    facts: dict[str, Any] = field(default_factory=dict)
    wall_s: float = 0.0

    def to_dict(self) -> dict[str, Any]:
        return dict(self.__dict__)


class Engine(Protocol):
    facts: dict[str, Any]

    def measure(self, spec: RequestSet) -> RunResult: ...

    def decode_once(self, tune: dict[str, Any] | None = None, tokens: int = 200) -> float | None: ...


class Session(Protocol):
    def engine(self, args: list[str], env: dict[str, str | None], pre_start: Callable[[], None] | None = None) -> contextlib.AbstractContextManager[Engine]: ...

    def system(self) -> dict[str, Any]: ...


# --------------------------------------------------------------------------- the engine log

_FACTS = [
    ("slots", re.compile(r"expert cache (\d+) slots"), int),
    ("workers", re.compile(r"(\d+) (?:expert-)?pool workers"), int),
    ("cpu_threads", re.compile(r"(\d+) CPU threads"), int),
    ("pcie_gbs", re.compile(r"PCIe probe: ([\d.]+) GB/s"), float),
    ("pcie_frac", re.compile(r"-> pcie_frac ([\d.]+)"), float),
    ("chunk", re.compile(r"prompt chunk auto: (\d+) tokens"), int),
    ("avail_gib", re.compile(r"available ([\d.]+) GiB \(MemAvailable"), float),
    ("loaded_gib", re.compile(r"loaded ([\d.]+) GiB at"), float),
]


def parse_facts(text: str) -> dict[str, Any]:
    """What the engine's own log says about the setup it chose (the last value of each fact wins)."""
    facts: dict[str, Any] = {}
    for key, rx, cast in _FACTS:
        found = rx.findall(text)
        if found:
            try:
                facts[key] = cast(found[-1])
            except ValueError:
                pass
    if "workers" not in facts and "cpu_threads" in facts:
        facts["workers"] = facts["cpu_threads"]
    low = text.lower()
    # the engine's low-RAM resident mode (--resident-experts) says so in its log; "resident experts" alone is the GPU cache's own wording
    facts["resident_ram"] = bool(re.search(r"resident ram mode|resident complement|resident [\d.]+ gib, pinned|low-ram mode", low))
    facts["hybrid_experts"] = "hybrid experts" in low or "hybrid:" in low
    facts["unknown_argument"] = bool(re.search(r"unknown argument|unrecognized (?:argument|option)", low))
    return facts


_CUDA_BUSY = re.compile(r"failed to initialize cuda|no cuda-capable device|cuda-capable device\(s\) is/are busy|"
                        r"cuda driver version is insufficient|hipinit.{0,40}(?:fail|error)|no hip-capable device", re.I)


def failure_kind(text: str, returncode: int | None) -> tuple[str, str]:
    """Classify an engine that ended before it was ready: (status, one plain line). The engine's informational
    `memory guard: limit ... available ...` line is not a failure and never decides the status (#3502)."""
    low = text.lower()
    lines = [ln.strip() for ln in text.splitlines()
             if re.search(r"unknown argument|unrecognized|invalid value|out of memory|cannot|failed|error|refus|killed", ln, re.I)
             and not re.search(r"memory guard: (?:limit|the available)", ln, re.I)]
    reason = " | ".join(lines[:3])[:300]
    if re.search(r"unknown argument|unrecognized (argument|option)", low):
        return "UNSUPPORTED", reason or "the engine does not accept one of the settings"
    if _CUDA_BUSY.search(text):
        return "FAIL_BOOT", ("the graphics card could not be opened by the measuring engine; it is probably in use by the "
                             "loaded model. Unload the model and tune again (" + reason + ")")[:400]
    if re.search(r"out of memory|memory guard: (?:refus|not enough|too)|cannot allocate|std::bad_alloc|\boom\b|(?:^|\n)killed(?:\r?\n|$)", low) or returncode in (-9, 137, 3221226505):
        return "OOM", reason or "the engine ran out of memory"
    return "FAIL_BOOT", reason or f"the engine stopped while starting (exit {returncode})"


# --------------------------------------------------------------------------- prompts

_SENTENCES = (
    "The harbour town woke slowly, with fishing boats knocking against the wooden piers.",
    "A small team measured how long each step of the process took and wrote the numbers in a notebook.",
    "Rain fell on the old railway station while a clerk sorted letters by hand.",
    "The recipe asked for two cups of flour, a pinch of salt and patience.",
    "In the library, a quiet reader turned pages while the clock in the hall struck three.",
    "Engineers compared the two bridges and noted how each handled wind and weight.",
    "The orchard produced more apples than the village could eat, so the surplus went to the city market.",
    "She packed a map, a flask of tea and a spare pair of boots for the long walk north.",
    "Each morning the baker lit the oven before dawn and left the shop door unlocked for the milkman.",
    "A careful gardener keeps a record of when each seed is planted and when it first sprouts.",
    "The museum's new wing displayed instruments that once guided ships across open water.",
    "Traffic slowed near the tunnel entrance, and the radio suggested taking the coast road instead.",
    "Two neighbours argued kindly over whose tomato plants had grown taller that summer.",
    "The orchestra tuned together, a single note spreading through the hall until the conductor raised a hand.",
    "He wrote down every question he had about the machine before opening its case.",
    "A river of lanterns floated past the bridge as the festival reached its final evening.",
)


def make_prompt(tokens: int, salt: str, chars_per_token: float = 4.4) -> str:
    """A prompt of about ``tokens`` tokens that no earlier run has seen: its first line is the salt, so a cached prefix is
    never reused, and the rest is shuffled prose."""
    rng = random.Random(salt)
    target = int(tokens * chars_per_token)
    parts = [f"Reference {salt}."]
    size = len(parts[0])
    while size < target:
        s = rng.choice(_SENTENCES)
        parts.append(s)
        size += len(s) + 1
    parts.append("Now answer in one short sentence: what is the main subject of the text above?")
    return " ".join(parts)


DECODE_PROMPTS = (
    "Write a long, detailed story about a lighthouse keeper who finds a message in a bottle. Keep going without stopping.",
    "Explain step by step how a bicycle works, covering the frame, wheels, gears, chain and brakes in depth.",
)


# --------------------------------------------------------------------------- a flock for the box locks

class FileLock:
    """An exclusive advisory lock on a file, held around each engine run (the lent boxes' /data/gpu.lock rule)."""

    def __init__(self, path: str | os.PathLike[str], wait_s: float = 5400.0, log: Callable[[str], None] = lambda s: None) -> None:
        self.path = str(path)
        self.wait_s = wait_s
        self.log = log
        self._fh = None

    def __enter__(self) -> "FileLock":
        Path(self.path).parent.mkdir(parents=True, exist_ok=True)
        self._fh = open(self.path, "a+")
        if os.name != "nt" and threading.current_thread() is threading.main_thread():
            return self._blocking()
        return self._polling()

    def _blocking(self) -> "FileLock":
        """A blocking flock(2), queued in the kernel like `flock -x` (so the other streams' waiters are served in order and
        a polling tuner is not starved). The wait limit is a timer, not a loop."""
        import fcntl
        import signal

        def on_alarm(signum, frame):  # noqa: ANN001
            raise TimeoutError

        try:
            fcntl.flock(self._fh.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
            return self
        except OSError:
            self.log(f"waiting for the lock {self.path}")
        old = signal.signal(signal.SIGALRM, on_alarm)
        signal.setitimer(signal.ITIMER_REAL, self.wait_s)
        try:
            fcntl.flock(self._fh.fileno(), fcntl.LOCK_EX)
            return self
        except TimeoutError:
            self._fh.close()
            raise StartFailed("TIMEOUT", f"the lock {self.path} was not free within {int(self.wait_s)} s") from None
        finally:
            signal.setitimer(signal.ITIMER_REAL, 0)
            signal.signal(signal.SIGALRM, old)

    def _polling(self) -> "FileLock":
        deadline = time.time() + self.wait_s
        told = False
        while True:
            try:
                if os.name == "nt":
                    import msvcrt
                    self._fh.seek(0)
                    msvcrt.locking(self._fh.fileno(), msvcrt.LK_NBLCK, 1)
                else:
                    import fcntl
                    fcntl.flock(self._fh.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
                return self
            except OSError:
                if time.time() > deadline:
                    self._fh.close()
                    raise StartFailed("TIMEOUT", f"the lock {self.path} was not free within {int(self.wait_s)} s")
                if not told:
                    self.log(f"waiting for the lock {self.path}")
                    told = True
                time.sleep(2.0)

    def __exit__(self, *exc: object) -> None:
        try:
            if os.name == "nt":
                import msvcrt
                self._fh.seek(0)
                msvcrt.locking(self._fh.fileno(), msvcrt.LK_UNLCK, 1)
            else:
                import fcntl
                fcntl.flock(self._fh.fileno(), fcntl.LOCK_UN)
        except OSError:
            pass
        finally:
            if self._fh:
                self._fh.close()


# --------------------------------------------------------------------------- the real session

def _free_port() -> int:
    with socket.socket() as s:
        s.bind(("127.0.0.1", 0))
        return s.getsockname()[1]


class WorkerEngine:
    """One running worker and the requests that measure it."""

    def __init__(self, session: "WorkerSession", proc: subprocess.Popen, port: int, key: str, log_files: list[Path],
                 load_s: float, vram_before: int | None, log: Callable[[str], None]) -> None:
        self.s = session
        self.proc = proc
        self.port = port
        self.key = key
        self.log_files = log_files
        self.load_s = load_s
        self.vram_before = vram_before
        self.log = log
        self.model = "strata-local"
        self.min_avail: float | None = None
        self.facts: dict[str, Any] = parse_facts(self.log_text())
        self.salt_n = 0
        self._model_id()

    # ---- helpers
    def log_text(self) -> str:
        out = []
        for f in self.log_files:
            try:
                out.append(f.read_text(encoding="utf-8", errors="replace"))
            except OSError:
                pass
        return "\n".join(out)

    def _model_id(self) -> None:
        try:
            req = urllib.request.Request(f"http://127.0.0.1:{self.port}/v1/models", headers=self._h())
            with loopback.urlopen(req, timeout=10) as r:
                data = json.loads(r.read())
            self.model = ((data.get("data") or [{}])[0]).get("id") or self.model
        except Exception:  # noqa: BLE001
            pass

    def _h(self) -> dict[str, str]:
        return {"content-type": "application/json", "authorization": f"Bearer {self.key}"}

    def _sample_mem(self) -> None:
        a = tune_machine.memory().get("available_mib")
        if a is not None:
            self.min_avail = a if self.min_avail is None else min(self.min_avail, a)

    def chat(self, prompt: str, max_tokens: int, tune: dict[str, Any] | None = None, timeout: float = 1800.0) -> dict[str, Any]:
        body: dict[str, Any] = {"model": self.model, "stream": False, "max_tokens": max_tokens, "temperature": 0.0,
                                "messages": [{"role": "user", "content": prompt}], "strata_checkpoint": False}
        if tune:
            body["strata_tune"] = tune
        req = urllib.request.Request(f"http://127.0.0.1:{self.port}/v1/chat/completions",
                                     data=json.dumps(body).encode(), headers=self._h(), method="POST")
        try:
            with loopback.urlopen(req, timeout=timeout) as r:
                data = json.loads(r.read())
        except urllib.error.HTTPError as e:
            raise StartFailed("NO_TOKENS", f"the worker refused the request ({e.code}): {e.read()[:200]!r}") from None
        except (urllib.error.URLError, OSError, ValueError) as e:
            code = self.proc.poll()
            status, why = failure_kind(self.log_text()[-4000:], code) if code is not None else ("TIMEOUT", str(e))
            raise StartFailed(status, f"the engine did not answer: {why}") from None
        self._sample_mem()
        return data

    # ---- measurements
    def decode_once(self, tune: dict[str, Any] | None = None, tokens: int = 200) -> float | None:
        for prompt in DECODE_PROMPTS:
            data = self.chat(prompt, tokens, tune)
            t = data.get("timings") or {}
            n, rate = t.get("predicted_n"), t.get("predicted_per_second")
            if n and rate and n >= max(32, tokens // 2):
                return float(rate)
        return None

    def prompt_once(self, tokens: int) -> dict[str, float | None]:
        self.salt_n += 1
        salt = secrets.token_hex(6)
        text = make_prompt(tokens, salt, self.s.chars_per_token)
        data = self.chat(text, 8)
        t = data.get("timings") or {}
        usage = data.get("usage") or {}
        pt = usage.get("prompt_tokens")
        if isinstance(pt, int) and pt > 64 and not self.s.calibrated:
            self.s.chars_per_token = max(2.0, min(8.0, self.s.chars_per_token * tokens / pt))
            self.s.calibrated = True
        rate = t.get("prompt_per_second")
        # a prompt that was partly cached is not a prompt-speed measurement
        if t.get("cache_n") and t.get("prompt_n") and t["cache_n"] > 0.1 * (t["prompt_n"] + t["cache_n"]):
            rate = None
        ms = t.get("prompt_ms")
        return {"rate": float(rate) if rate else None, "ttft_s": float(ms) / 1000.0 if ms else None,
                "tokens": pt if isinstance(pt, int) else None}

    def measure(self, spec: RequestSet) -> RunResult:
        t0 = time.time()
        res = RunResult(load_s=self.load_s, facts=dict(self.facts))
        for _ in range(spec.warmups):
            self.decode_once(None, 64)
        runs: list[float] = []
        for _ in range(spec.decodes):
            r = self.decode_once(None, spec.decode_tokens)
            if r:
                runs.append(r)
        res.decode_runs = [round(x, 3) for x in runs]
        if not runs:
            res.status, res.reason = "NO_TOKENS", "the engine produced no measurable decode"
        else:
            res.decode_tok_s = round(statistics.median(runs), 3)
        if spec.pp_tokens:
            cold = self.prompt_once(spec.pp_tokens)
            warm = self.prompt_once(spec.pp_tokens)
            res.pp_cold_tok_s = cold["rate"]
            res.pp_tok_s, res.ttft_s = warm["rate"], warm["ttft_s"]
        if spec.long_tokens:
            res.pp_long_tok_s = self.prompt_once(spec.long_tokens)["rate"]
        res.rss_peak_mib = tune_engine_peak(self.proc.pid)
        used = tune_machine.gpu_used_mib()
        if used is not None:
            res.vram_used_mib = used
            if self.vram_before is not None:
                res.vram_added_mib = used - self.vram_before
        res.min_avail_mib = self.min_avail
        res.facts = dict(parse_facts(self.log_text()))
        res.wall_s = round(time.time() - t0, 1)
        return res


def tune_engine_peak(pid: int) -> float | None:
    try:
        from strata_control.tune_engine import process_tree_peak_mib
        return process_tree_peak_mib(pid)
    except Exception:  # noqa: BLE001
        return None


class WorkerSession:
    """Starts ``serve.server`` on a copy of the run configuration for each configuration the tuner asks for."""

    def __init__(self, *, base_config: dict[str, Any], python: str | os.PathLike[str] | None = None,
                 serve_dir: str | os.PathLike[str], workdir: str | os.PathLike[str] | None = None,
                 boot_timeout_s: float = 900.0, lock: Callable[[], contextlib.AbstractContextManager] | None = None,
                 log: Callable[[str], None] = lambda s: None, extra_env: dict[str, str] | None = None) -> None:
        self.base = json.loads(json.dumps(base_config))
        self.python = str(python or sys.executable)
        self.serve_dir = Path(serve_dir)
        self.workdir = Path(workdir) if workdir else Path(tempfile.mkdtemp(prefix="strata-tune-"))
        self.workdir.mkdir(parents=True, exist_ok=True)
        self.boot_timeout_s = boot_timeout_s
        self.lock = lock
        self.log = log
        self.extra_env = dict(extra_env or {})
        self.chars_per_token = 4.4
        self.calibrated = False
        self.runs = 0

    def system(self) -> dict[str, Any]:
        return {"memory": tune_machine.memory(), "gpus": tune_machine.gpus(), "cpu": tune_machine.cpu()}

    @contextlib.contextmanager
    def engine(self, args: list[str], env: dict[str, str | None], pre_start: Callable[[], None] | None = None) -> Iterator[WorkerEngine]:
        with contextlib.ExitStack() as stack:
            if self.lock is not None:
                stack.enter_context(self.lock())
            if pre_start is not None:
                pre_start()                      # the memory and quiet checks run once the lock is ours, right before the start
            self.runs += 1
            n = self.runs
            cfg = json.loads(json.dumps(self.base))
            cfg["args"] = list(args)
            cenv = dict(cfg.get("env") or {})
            for k, v in env.items():
                if v is None:
                    cenv.pop(k, None)
                else:
                    cenv[k] = str(v)
            cfg["env"] = cenv
            key = secrets.token_hex(16)
            port = _free_port()
            engine_log = self.workdir / f"engine-{n}.log"
            worker_log = self.workdir / f"worker-{n}.log"
            engine_log.write_text("")
            cfg.update({"api_key": key, "port": port, "host": "127.0.0.1", "open_browser": False, "log": str(engine_log)})
            cfg_path = self.workdir / f"run-{n}.json"
            cfg_path.write_text(json.dumps(cfg, indent=1), encoding="utf-8")
            spawn_env = dict(os.environ)
            spawn_env.update(self.extra_env)
            for k, v in env.items():
                if v is None:
                    spawn_env.pop(k, None)
            spawn_env["PYTHONUNBUFFERED"] = "1"
            spawn_env["STRATA_NO_PRESET"] = "1"          # measure the configuration as given, not a saved tuning on top
            before = tune_machine.gpu_used_mib()
            wl = open(worker_log, "w")
            argv = [self.python, "-m", "serve.server", "--engine", "strata", "--config", str(cfg_path),
                    "--host", "127.0.0.1", "--port", str(port)]
            t0 = time.time()
            try:
                from serve import lifeline
                proc = lifeline.popen(argv, "worker", cwd=str(self.serve_dir), env=spawn_env, stdout=wl,
                                      stderr=subprocess.STDOUT)
            except OSError as e:
                wl.close()
                raise StartFailed("FAIL_BOOT", f"the worker could not be started: {e}") from None
            try:
                deadline = t0 + self.boot_timeout_s
                ready = False
                while time.time() < deadline:
                    if proc.poll() is not None:
                        text = (engine_log.read_text(errors="replace") if engine_log.exists() else "") + "\n" + \
                               worker_log.read_text(errors="replace")
                        status, why = failure_kind(text, proc.returncode)
                        raise StartFailed(status, why)
                    try:
                        req = urllib.request.Request(f"http://127.0.0.1:{port}/health",
                                                     headers={"authorization": f"Bearer {key}"})
                        with loopback.urlopen(req, timeout=2) as r:
                            if r.status == 200:
                                ready = True
                                break
                    except (urllib.error.URLError, OSError):
                        time.sleep(0.5)
                if not ready:
                    raise StartFailed("TIMEOUT", f"the engine was not ready within {int(self.boot_timeout_s)} s")
                eng = WorkerEngine(self, proc, port, key, [engine_log, worker_log], time.time() - t0, before, self.log)
                yield eng
            finally:
                try:
                    if proc.poll() is None:
                        proc.terminate()
                        try:
                            proc.wait(timeout=60)
                        except subprocess.TimeoutExpired:
                            proc.kill()
                            proc.wait(timeout=30)
                finally:
                    wl.close()
                    # the engine gives its graphics memory back a moment after it exits
                    time.sleep(3.0)