A high-performance inference engine for AI models
import asyncio
import json
import sys
import time
from collections.abc import AsyncIterator, Callable
from contextlib import ExitStack, redirect_stdout
from functools import partial
from pathlib import Path
from typing import Annotated, Any
from unittest.mock import patch
import typer
from bench import BenchRequest, BenchResponse, BenchSampling
from common import InferenceEngine, get_model_path, get_tokenized_prompt, run_loop
from mach import MemoryCounters, get_memory_counters
from omlx.engine.base import GenerationOutput
from omlx.engine.batched import BatchedEngine
from omlx.engine_core import get_mlx_executor
from omlx.model_settings import MAX_LIGHTNING_MTP_DRAFT_TOKENS, ModelSettings
from omlx.speculative.dflash import load_dflash_drafter
from omlx.utils.model_loading import _checkpoint_has_mtp_weights, _is_mtp_compatible
class OMLXEngine(InferenceEngine):
engine: BatchedEngine
runner: asyncio.Runner
closed: bool
mtp: bool
mtp_model: Any
max_depth: int
default_depth: int
def __init__(
self,
model: str | Path,
*,
draft_model: str | Path | None = None,
):
super().__init__()
self.mtp = False
self.dflash = draft_model is not None
self.mtp_model = None
self.max_depth = 0
local_path = Path(model).expanduser()
model_path = str(local_path) if local_path.is_dir() else get_model_path(model)
if draft_model is not None:
draft_path = Path(draft_model).expanduser()
draft_path = str(draft_path) if draft_path.is_dir() else get_model_path(draft_model)
config = json.loads((Path(draft_path) / "config.json").read_text())
self.max_depth = int(config.get("block_size", 0)) - 1
if self.max_depth < 1:
raise ValueError("DFlash requires a draft checkpoint with block_size >= 2")
self.engine = BatchedEngine(
model_name=model_path,
model_settings=ModelSettings(
dflash_enabled=True,
dflash_draft_model=draft_path,
dflash_draft_quant_enabled=False,
dflash_verify_mode="dflash",
),
)
else:
config = json.loads((Path(model_path) / "config.json").read_text())
model_type = config.get("model_type")
self.mtp = (
_is_mtp_compatible(config, model_type)
# Gemma 4 MTP requires oMLX's VLM engine, not BatchedEngine.
and model_type not in ("gemma4", "gemma4_unified")
and _checkpoint_has_mtp_weights(model_path)
)
self.engine = BatchedEngine(
model_name=model_path,
model_settings=ModelSettings(mtp_enabled=self.mtp),
)
self.default_depth = self.max_depth
self.runner = asyncio.Runner()
self.closed = False
try:
self.runner.run(self.engine.start())
if self.dflash:
self.default_depth = self.max_depth = self.runner.run(self._load_drafter(draft_path))
self.mtp_model = self.engine._model
elif self.mtp:
from omlx.patches.mlx_lm_mtp.batch_generator import (
_model_has_mtp_module,
_model_mtp_decode_enabled,
_resolve_mtp_chain_depth,
)
self.mtp_model = self.engine._model
if not _model_has_mtp_module(self.mtp_model) or not _model_mtp_decode_enabled(self.mtp_model):
raise ValueError("Detected MTP weights, but oMLX did not load and enable the model's MTP heads")
chain, self.default_depth, _ = _resolve_mtp_chain_depth(self.mtp_model)
self.max_depth = MAX_LIGHTNING_MTP_DRAFT_TOKENS if chain else 1
except BaseException:
self.close()
raise
async def _load_drafter(self, path: str) -> int:
def load():
return load_dflash_drafter(
path,
self.engine.prepare_dflash_target(),
tokenizer=self.engine.tokenizer,
quant_enabled=False,
verify_mode="dflash",
)
drafter = await asyncio.get_running_loop().run_in_executor(get_mlx_executor(), load)
self.engine.set_dflash_drafter(drafter)
return drafter.depth
def _configure_depth(self, depth: int) -> None:
if depth < 0 or self.max_depth < depth:
raise ValueError(f"speculative_depth must be between 0 and {self.max_depth} for this engine")
if self.mtp or self.dflash:
model = self.mtp_model
for host in (model, getattr(model, "language_model", model), getattr(model, "_language_model", model)):
host._omlx_mtp_decode_enabled = depth > 0
host._omlx_mtp_depth = max(1, depth)
def close(self) -> None:
if self.closed:
return
self.closed = True
try:
self.runner.run(self.engine.stop())
finally:
self.runner.close()
def execute(self, request: BenchRequest) -> list[BenchResponse]:
if self.closed:
raise RuntimeError("Engine is closed")
return self.runner.run(self._execute(request))
async def _execute(self, request: BenchRequest) -> list[BenchResponse]:
num_runs = request.num_runs if request.num_runs is not None else 1
if num_runs < 1:
raise ValueError("num_runs must be 1 or greater")
if request.max_tokens is not None and request.max_tokens < 0:
raise ValueError("max_tokens must be 0 or greater")
depth = self.default_depth if request.speculative_depth is None else request.speculative_depth
self._configure_depth(depth)
prompt: list[int] = get_tokenized_prompt(request, self.engine.tokenizer)
sampling = request.sampling or BenchSampling()
generate = partial(
self.engine.stream_generate,
prompt=prompt,
max_tokens=request.max_tokens or (sys.maxsize - len(prompt) - 1),
temperature=sampling.temp if sampling.temp is not None else 0.7,
top_p=sampling.top_p if sampling.top_p is not None else 0.9,
min_p=sampling.min_p or 0.0,
top_k=sampling.top_k or 0,
)
return [await self._run(generate, len(prompt), depth) for _ in range(num_runs)]
async def _run(
self, generate: Callable[[], AsyncIterator[GenerationOutput]], prompt_size: int, depth: int
) -> BenchResponse:
time_first_token: float | None = None
time_last_token: float | None = None
completion_tokens = 0
response: GenerationOutput | None = None
mem_counters_max: MemoryCounters = get_memory_counters()
decode_forwards = speculative_forwards = 0
# Public outputs omit target-forward counts.
# Keep hooks and their state local to this run; restore even on failure.
with ExitStack() as hooks:
if self.mtp or self.dflash:
model_type = type(self.mtp_model)
original_forward = model_type.__call__
def counted(instance, inputs, *args, **kwargs):
nonlocal prompt_size, decode_forwards, speculative_forwards
if instance is self.mtp_model:
width = inputs.shape[1]
if prompt_size > 0:
prompt_size -= width
else:
decode_forwards += 1
if kwargs.get("return_hidden") and width > 1:
speculative_forwards += 1
return original_forward(instance, inputs, *args, **kwargs)
hooks.enter_context(patch.object(model_type, "__call__", counted))
time_start = time.perf_counter()
async for response in generate():
if response.completion_tokens > completion_tokens:
generated_at = response.generated_at
if generated_at is None:
generated_at = time.perf_counter()
if time_first_token is None:
time_first_token = generated_at
time_last_token = response.generated_until if response.generated_until is not None else generated_at
completion_tokens = response.completion_tokens
counters = get_memory_counters()
if counters.graphics_total > mem_counters_max.graphics_total:
mem_counters_max = counters
time_total: float = time.perf_counter() - time_start
if response is None:
raise RuntimeError("Generation did not return a response")
if response.finish_reason == "error":
raise RuntimeError("oMLX generation failed; see engine stderr")
forward_passes = None
if self.mtp or self.dflash:
forward_passes = decode_forwards
if depth and completion_tokens > 2 and speculative_forwards == 0:
raise RuntimeError("Speculative decoding requested but no verification ran")
print(
f"oMLX {'DFlash' if self.dflash else 'MTP'} depth={depth}: "
f"{speculative_forwards} speculative verifications, "
f"{decode_forwards} target decode forwards",
file=sys.stderr,
)
# An immediate stop can finish without producing any completion tokens.
time_to_first_token = time_first_token - time_start if time_first_token is not None else time_total
decode_time = (
time_last_token - time_first_token if time_first_token is not None and time_last_token is not None else 0.0
)
prompt_tokens = response.prompt_tokens - response.cached_tokens
return BenchResponse(
text=response.text,
tokens_count=completion_tokens,
time_to_first_token=time_to_first_token,
prompt_tps=prompt_tokens / time_to_first_token if time_to_first_token > 0.0 else 0.0,
decode_tps=(completion_tokens - 1) / decode_time if decode_time > 0.0 else 0.0,
tokens_per_forward_pass=(
max(0, completion_tokens - 1) / forward_passes
if forward_passes
else (1.0 if completion_tokens and forward_passes is None else 0.0)
),
duration=time_total,
memory_phys_footprint=mem_counters_max.phys_footprint,
memory_resident=mem_counters_max.resident_size,
memory_graphics_total=mem_counters_max.graphics_total,
)
def run(
model: Annotated[str, typer.Option("-m", "--model")],
draft_model: Annotated[
str | None, typer.Option("-d", "--draft-model", help="DFlash draft checkpoint path or Hugging Face repository.")
] = None,
) -> None:
engine: OMLXEngine
with redirect_stdout(sys.stderr):
engine = OMLXEngine(model, draft_model=draft_model)
try:
run_loop(engine)
finally:
with redirect_stdout(sys.stderr):
engine.close()
def main() -> None:
typer.run(run)
if __name__ == "__main__":
main()