Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

7.6 KB
Raw Blame
import json
import os
import sys
import time
from collections.abc import Callable
from contextlib import redirect_stdout
from functools import partial
from pathlib import Path
from typing import Annotated

import typer
from bench import BenchRequest, BenchResponse
from common import InferenceEngine, get_model_path, get_tokenized_prompt, run_loop
from mach import MemoryCounters, get_memory_counters
from mtplx.generation import GenerationOutput, VerifyStrategy, generate_ar, generate_mtpk
from mtplx.profiles import apply_profile_env, runtime_env_overrides_from_contract
from mtplx.runtime import MTPLXRuntime, load
from mtplx.sampling import SamplerConfig


class MTPLXEngine(InferenceEngine):
    runtime: MTPLXRuntime
    default_speculative_depth: int
    mtp_history_policy: str

    def __init__(self, model: str | Path):
        super().__init__()
        local_path = Path(model).expanduser()
        model_path = str(local_path) if local_path.is_dir() else get_model_path(model)
        default_depth = self._configure_profile(model_path)
        self.runtime = load(model_path)
        self.default_speculative_depth = default_depth if self.runtime.mtp_enabled else 0
        self.mtp_history_policy = os.environ.get("MTPLX_MTP_HISTORY_POLICY", "committed")

    def _configure_profile(self, model_path: str) -> int:
        metadata_path = Path(model_path) / "mtplx_runtime.json"
        metadata = json.loads(metadata_path.read_text()) if metadata_path.is_file() else {}
        config_path = Path(model_path) / "config.json"
        config = json.loads(config_path.read_text()) if config_path.is_file() else {}
        model_type = config.get("text_config", config).get("model_type")
        # Capture-commit inspects Qwen3-Next-style layers; other architectures
        # (e.g. Qwen Flash-Next) must retain the portable batched verifier.
        capture_verify = model_type in {
            "qwen3_next",
            "qwen3_5",
            "qwen3_5_text",
            "qwen3_5_moe",
            "qwen3_5_moe_text",
            "qwen3_5_mtp",
        }
        self.verify_strategy: VerifyStrategy = "capture_commit" if capture_verify else "batched"
        self.verify_core = "linear-gdn-from-conv-tape" if capture_verify else "stock"
        runtime_env = runtime_env_overrides_from_contract(metadata)
        if not capture_verify:
            runtime_env["MTPLX_SKIP_VERIFY_SNAPSHOT"] = "0"

        apply_profile_env(
            metadata.get("recommended_profile") or "sustained",
            runtime_env_overrides=runtime_env,
        )

        depth: int | None = metadata.get("recommended_mtp_depth")
        if depth is None:
            depth = metadata.get("mtp_depth_default")

        return int(depth if depth is not None else 3)

    def execute(self, request: BenchRequest) -> list[BenchResponse]:
        num_runs = request.num_runs if request.num_runs is not None else 1
        if num_runs < 1:
            raise ValueError("num_runs must be 1 or greater")

        if request.max_tokens is not None and request.max_tokens < 0:
            raise ValueError("max_tokens must be 0 or greater")

        depth = request.speculative_depth
        if depth is None:
            depth = self.default_speculative_depth

        if depth < 0:
            raise ValueError("speculative_depth must be 0 or greater")

        if depth and not self.runtime.mtp_enabled:
            raise ValueError("Speculative decoding requires a model with matching MTP weights")

        prompt: list[int] = get_tokenized_prompt(request, self.runtime.tokenizer)

        sampler = SamplerConfig()
        if request.sampling is not None:
            if request.sampling.min_p not in (None, 0.0):
                raise ValueError("MTPLX does not support min_p sampling")
            sampler = SamplerConfig(
                temperature=request.sampling.temp if request.sampling.temp is not None else sampler.temperature,
                top_p=request.sampling.top_p if request.sampling.top_p is not None else sampler.top_p,
                top_k=request.sampling.top_k if request.sampling.top_k is not None else sampler.top_k,
            )

        tokenizer = self.runtime.tokenizer
        stop_token_ids: set[int] = set(getattr(tokenizer, "eos_token_ids", None) or ())
        eos_token_id = getattr(tokenizer, "eos_token_id", None)
        if eos_token_id is not None:
            stop_token_ids.add(eos_token_id)
        if not request.max_tokens and not stop_token_ids:
            raise ValueError("Unlimited generation requires an EOS token")

        decoder = (
            partial(
                generate_mtpk,
                speculative_depth=depth,
                mtp_history_policy=self.mtp_history_policy,
                # Pair the profile's snapshot-free path with the CLI's verifier.
                verify_strategy=self.verify_strategy,
                verify_core=self.verify_core,
            )
            if depth
            else generate_ar
        )
        generate = partial(
            decoder,
            self.runtime,
            prompt,
            max_tokens=request.max_tokens or sys.maxsize,
            sampler=sampler,
            stop_token_ids=stop_token_ids,
        )
        return [self._run(generate, speculative=depth > 0) for _ in range(num_runs)]

    def _run(self, generate: Callable[..., GenerationOutput], *, speculative: bool) -> BenchResponse:
        # prepare variables
        time_first_token: float = -1.0
        mem_counters_max: MemoryCounters = get_memory_counters()

        def update_memory(*_progress: object) -> None:
            nonlocal mem_counters_max
            counters = get_memory_counters()
            if counters.graphics_total > mem_counters_max.graphics_total:
                mem_counters_max = counters

        def token_callback(token_ids: list[int]) -> None:
            nonlocal time_first_token
            if token_ids and time_first_token < 0.0:
                time_first_token = time.perf_counter()
            update_memory()

        # create and run inference
        time_start: float = time.perf_counter()
        output = generate(token_callback=token_callback, prefill_callback=update_memory)
        time_end = time.perf_counter()
        time_total = time_end - time_start
        update_memory()

        # MTPLX does not invoke token_callback for an immediate EOS.
        if time_first_token < 0.0:
            time_first_token = time_end

        # collect metrics
        decode_tokens = max(0, output.stats.generated_tokens - 1)
        if speculative:
            forward_passes = int(output.stats.verify_calls)
            tokens_per_forward_pass = decode_tokens / forward_passes if forward_passes else 0.0
        else:
            # AR does not record verify_calls and emits one token per target forward.
            tokens_per_forward_pass = float(output.stats.generated_tokens > 0)
        decode_duration = time_end - time_first_token

        return BenchResponse(
            text=output.text,
            tokens_count=output.stats.generated_tokens,
            time_to_first_token=(time_first_token - time_start),
            prompt_tps=output.stats.prompt_tps,
            decode_tps=decode_tokens / decode_duration if decode_duration > 0.0 else 0.0,
            tokens_per_forward_pass=tokens_per_forward_pass,
            duration=time_total,
            memory_phys_footprint=mem_counters_max.phys_footprint,
            memory_resident=mem_counters_max.resident_size,
            memory_graphics_total=mem_counters_max.graphics_total,
        )


def run(model: Annotated[str, typer.Option("-m", "--model")]) -> None:
    with redirect_stdout(sys.stderr):
        engine = MTPLXEngine(model)
    run_loop(engine)


def main() -> None:
    typer.run(run)


if __name__ == "__main__":
    main()