Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

14.5 KB
Raw Blame
import json
import math
import os
import queue
import secrets
import subprocess
import sys
import time
from contextlib import redirect_stdout
from pathlib import Path
from typing import Annotated

import typer
from bench import BenchRequest, BenchResponse, BenchSampling
from common import InferenceEngine, run_loop
from mach import get_memory_counters

PROJECT_DIR = Path(__file__).resolve().parent
SPLASH_DIR = PROJECT_DIR / "deps" / "splash"
BUILD_DIR = SPLASH_DIR / "build"
SPLASH_BINARY = BUILD_DIR / "splash"
MEMORY_SAMPLE_INTERVAL = 0.1


def get_model_path(model: str | Path) -> Path:
    from splash.install import models, paths, upstream

    path = Path(model).expanduser()
    if path.is_dir():
        root = path.resolve()
    else:
        # Benchmark requests contain text only; skip vision weights and warmup.
        selection = models.Selection.of(paths.MODELS, str(model), language_only=True)
        upstream.prepare(selection)
        root = selection.link.resolve()

    for name in ("target", "draft", "tokenizer"):
        if not (root / name).is_dir():
            raise ValueError(f"Expected a prepared Splash model directory with {name}/: {root}")

    return root


def sampling_options(sampling: BenchSampling | None) -> dict[str, float | int]:
    # Match Frontend's float32 and native top-k limits for raw prompts too.
    minimum = float.fromhex("0x1p-149")
    sampling = sampling or BenchSampling()

    temperature = sampling.temp if sampling.temp is not None else 1.0
    if not math.isfinite(temperature) or not 0 <= temperature <= 2 or 0 < temperature < minimum:
        raise ValueError("Splash requires temp in [0, 2]")

    top_p = sampling.top_p if sampling.top_p is not None else 0.95
    if not math.isfinite(top_p) or not minimum <= top_p <= 1:
        raise ValueError("Splash requires top_p in (0, 1]")

    top_k = sampling.top_k if sampling.top_k is not None else 20
    if not 1 <= top_k <= 32:
        raise ValueError("Splash requires top_k in [1, 32]")

    if sampling.min_p not in (None, 0.0):
        raise ValueError("Splash does not support nonzero min_p")

    return {"temperature": temperature, "top_p": top_p, "top_k": top_k}


class SplashEngine(InferenceEngine):
    def __init__(self, model: str | Path):
        sys.path.insert(0, str(SPLASH_DIR.parent))
        sys.path.insert(0, str(SPLASH_DIR))
        from splash.install import assembly, models, paths

        self._processes = models.processes
        self._processes.run([str(paths.BINARY), "device-check"], check=True)

        from splash.server import protocol as wire
        from splash.server.backend import Job, NativeBackend
        from splash.server.chat_templates import ChatTemplates
        from splash.server.constraints import ConstraintFactory, validate_tokenizer
        from splash.server.frontend import Frontend
        from splash.server.metrics import metrics_dict, timings_dict
        from splash.server.runtime import MultiplexedRuntime
        from splash.server.thinking import ThinkingCodec
        from transformers import AutoTokenizer

        self._wire = wire
        self._job_type = Job
        self._metrics = metrics_dict
        self._timings = timings_dict
        self.closed = False
        self.process = None
        self.runtime = None
        self.backend = None
        root = get_model_path(model)
        family = assembly.family(root)
        if family is None:
            raise ValueError(f"Splash cannot determine the prepared model's family: {root}")
        self.tokenizer = AutoTokenizer.from_pretrained(
            root / "tokenizer",
            local_files_only=True,
            trust_remote_code=False,
            fix_mistral_regex=True,
        )
        validate_tokenizer(self.tokenizer, family.chat_format)
        chat_templates = ChatTemplates(self.tokenizer)
        self._command = [str(paths.BINARY), "serve-native", str(root), "auto", "auto"]
        try:
            self.runtime = MultiplexedRuntime(
                self._command,
                process_factory=self._start_process,
                startup_timeout=600.0,
                pending_limit=1,
                eager_start=False,
            )
            self.backend = NativeBackend(self.runtime, self.tokenizer, request_logger=lambda record: None)
            # Each run explicitly waits for readiness. Disable the server's
            # background recovery, whose stdout logs would corrupt JSONL output.
            self.runtime.on_engine_failure = None
            if not self.runtime.wait_ready():
                raise RuntimeError("Splash did not become ready after native warmup")

            ready = self.runtime.readiness
            if ready is None:
                raise RuntimeError("Splash did not announce its context window")

            self.frontend = Frontend(
                self.tokenizer,
                self.backend,
                str(model),
                ready.max_context_tokens,
                1800.0,
                ready.max_concurrent_requests,
                constraint_factory=ConstraintFactory(self.tokenizer, family.chat_format),
                chat_templates=chat_templates,
                chat_format=family.chat_format,
                thinking_codec=ThinkingCodec(),
                vision=ready.vision,
            )
            # Fail before accepting requests if child-process accounting is unavailable.
            self._get_memory_counters()
        except BaseException:
            self.close()
            raise

    def close(self) -> None:
        if self.closed:
            return

        self.closed = True
        try:
            if self.backend is not None:
                self.backend.close()
            elif self.runtime is not None:
                self.runtime.close()
        finally:
            self.process = None

    def _start_process(self) -> subprocess.Popen:
        self.process = self._processes.Popen(
            self._command,
            stdin=subprocess.PIPE,
            stdout=subprocess.PIPE,
            stderr=None,
            bufsize=0,
        )
        return self.process

    def execute(self, request: BenchRequest) -> list[BenchResponse]:
        if self.closed:
            raise RuntimeError("Engine is closed")

        num_runs = request.num_runs if request.num_runs is not None else 1
        if num_runs < 1:
            raise ValueError("num_runs must be 1 or greater")

        if request.speculative_depth not in (None, 7):
            raise ValueError("Splash uses a fixed speculative_depth of 7; it cannot disable or resize drafting")

        if request.max_tokens is not None and request.max_tokens < 0:
            raise ValueError("max_tokens must be 0 or greater")

        body: dict = sampling_options(request.sampling)
        if request.max_tokens:
            body["max_tokens"] = request.max_tokens
        if request.prompt_text is None:
            if request.prompt_chat is None:
                raise ValueError("prompt_text and prompt_chat are None")
            body["messages"] = [message.model_dump(mode="json", exclude_none=True) for message in request.prompt_chat]
            body.update(request.model_dump(mode="json", include={"tools", "tool_choice"}))

        return [self._run(self._prepare(request, body)) for _ in range(num_runs)]

    def _prepare(self, request: BenchRequest, body: dict):
        deadline = self.frontend.request_deadline(body, time.monotonic())
        if request.prompt_text is None:
            return self.frontend.prepare(body, deadline=deadline)

        # Submit raw tokens to the same native runtime without a chat template.
        tokens = self.frontend.tokenize({"content": request.prompt_text, "add_special": True}, deadline=deadline)
        if not tokens:
            raise ValueError("Input prompt is empty")
        # Native Splash requires a positive, context-bounded output budget.
        # Its EOS handling still stops generation before this limit.
        max_tokens = body.get("max_tokens", self.frontend.max_context - len(tokens))
        if max_tokens < 1 or len(tokens) + max_tokens > self.frontend.max_context:
            raise ValueError("prompt and max_tokens exceed the context window")

        return self._job_type(
            request_id=next(self.frontend.ids),
            prompt_tokens=tokens,
            max_new_tokens=max_tokens,
            seed=secrets.randbits(64),
            sampling=self._wire.SamplingParameters(
                temperature=body["temperature"],
                top_p=body["top_p"],
                top_k=body["top_k"],
            ),
            deadline=deadline,
        )

    def _status(self) -> dict:
        # NativeBackend.status() may return cached telemetry on a timeout.
        # A benchmark needs fresh snapshots around its single request.
        event = self.runtime.status()
        snapshot = json.loads(event.json)
        if not isinstance(snapshot, dict) or snapshot.get("schema_version") != self._wire.STATUS_SCHEMA_VERSION:
            raise RuntimeError("Splash status does not match the current schema")
        return snapshot

    def _get_memory_counters(self):
        process = self.process
        if process is None or process.poll() is not None:
            raise RuntimeError("Splash process is not running")
        memory = get_memory_counters()
        if process.pid == os.getpid():
            return memory
        child = get_memory_counters(pid=process.pid)
        # Include Python's tokenizer/frontend as well as the native engine,
        # matching the scope of the other in-process benchmark adapters.
        memory.phys_footprint += child.phys_footprint
        memory.resident_size += child.resident_size
        memory.graphics_total += child.graphics_total
        return memory

    def _run(self, job) -> BenchResponse:
        if not self.runtime.wait_ready():
            raise RuntimeError("Splash runtime is not ready")

        process = self.process
        restarts = self.runtime.restart_count
        before = self._status()
        memory = self._get_memory_counters()
        next_memory_sample = time.monotonic() + MEMORY_SAMPLE_INTERVAL
        chunks = []
        submitted = False
        terminal = False

        try:
            self.backend.submit(job)
            submitted = True

            while True:
                remaining = job.deadline - time.monotonic()
                if remaining <= 0:
                    raise TimeoutError("Splash request timed out")

                try:
                    kind, value = job.events.get(timeout=min(remaining, 0.1))
                except queue.Empty:
                    kind, value = None, None

                if kind in ("done", "error"):
                    terminal = True
                if kind == "error":
                    raise value
                if self.process is not process or self.runtime.restart_count != restarts:
                    raise RuntimeError("Splash restarted during the benchmark")

                # Detokenization can emit several text fragments per native
                # batch. Keep Mach queries off that per-fragment hot path.
                now = time.monotonic()
                if kind in ("start", "done") or now >= next_memory_sample:
                    current = self._get_memory_counters()
                    if current.graphics_total > memory.graphics_total:
                        memory = current
                    next_memory_sample = now + MEMORY_SAMPLE_INTERVAL
                if kind == "text":
                    chunks.append(value)
                elif kind == "done":
                    result = value
                    if result.reason == "cancelled":
                        raise RuntimeError("Splash request was cancelled")
                    break
                elif kind not in (None, "start", "progress"):
                    raise RuntimeError(f"Unexpected Splash backend event: {kind}")
        except BaseException as error:
            if not submitted:
                self.close()
            elif not terminal:
                self._cancel_and_drain(job, timed_out=isinstance(error, TimeoutError))
            raise

        after = self._status()
        if self.process is not process or self.runtime.restart_count != restarts:
            raise RuntimeError("Splash restarted during the benchmark")

        timings = self._timings(result)
        latency = self._metrics(result)["request_latency"]
        decode_batches = after["scheduler"]["decode_batches"] - before["scheduler"]["decode_batches"]
        decode_tokens = after["metrics"]["decode_output_tokens"] - before["metrics"]["decode_output_tokens"]
        if decode_batches < 0 or decode_tokens < 0:
            raise RuntimeError("Splash native counters moved backwards")

        return BenchResponse(
            text="".join(chunks),
            tokens_count=result.completion_tokens,
            time_to_first_token=latency.get("ttft_ms", result.request_wall_ms) / 1000.0,
            prompt_tps=timings["prompt_per_second"],
            decode_tps=timings["predicted_per_second"],
            tokens_per_forward_pass=(
                decode_tokens / decode_batches if decode_batches else float(result.completion_tokens > 0)
            ),
            duration=result.request_wall_ms / 1000.0,
            memory_phys_footprint=memory.phys_footprint,
            memory_resident=memory.resident_size,
            memory_graphics_total=memory.graphics_total,
        )

    def _cancel_and_drain(self, job, *, timed_out: bool = False) -> None:
        # Let upstream cancellation retire the admitted work before the next
        # request takes its counter baseline. Close if the engine cannot drain.
        try:
            self.backend.cancel(job, timed_out=timed_out)
            deadline = time.monotonic() + 155.0
            while True:
                remaining = deadline - time.monotonic()
                if remaining <= 0:
                    raise TimeoutError("Splash cancellation timed out")

                kind, _ = job.events.get(timeout=remaining)
                if kind in ("done", "error"):
                    return
        except BaseException:
            self.close()
            raise


def run(model: Annotated[str, typer.Option("-m", "--model")]) -> None:
    if sys.platform != "ios" and not SPLASH_BINARY.is_file():
        subprocess.run([str(PROJECT_DIR / "bootstrap.sh")], check=True, stdout=sys.stderr)

    engine: SplashEngine
    with redirect_stdout(sys.stderr):
        engine = SplashEngine(model)
    try:
        run_loop(engine)
    finally:
        with redirect_stdout(sys.stderr):
            engine.close()


def main() -> None:
    typer.run(run)


if __name__ == "__main__":
    main()