Run 35B MoE models in ~2.5 GB of RAM. Weights stream from SSD; a trained router predicts loads a step ahead. Mac, iPhone, Android, Windows supported
Python
"""edge0 HTTP server: a single-process, one-model OpenAI-style endpoint.
The engine is exclusive to one request at a time (a single MLX process
can stream one generation well; concurrent requests would thrash the
expert cache). Requests queue on a lock: each generation streams
token-by-token over SSE, and the queue drains in FIFO order.
"""
from __future__ import annotations
import json
import threading
import time
from dataclasses import dataclass, field
from typing import Any
from edge0.engine.base import Edge0Engine
@dataclass
class ChatMessage:
role: str
content: str
tool_calls: list[dict] | None = None
tool_call_id: str | None = None
name: str | None = None
@dataclass
class ChatRequest:
model: str
messages: list[ChatMessage]
temperature: float | None = None
top_p: float | None = None
top_k: int | None = None
max_tokens: int | None = None
seed: int | None = None
stream: bool = False
enable_thinking: bool | None = None
tools: list[dict] | None = None
tool_choice: Any = None
raw: dict[str, Any] = field(default_factory=dict)
def _template_tool_calls(calls) -> list[dict] | None:
if calls is None:
return None
if not isinstance(calls, list):
raise ValueError("tool_calls must be a list of function calls")
normalized = []
for call in calls:
if not isinstance(call, dict) or not isinstance(call.get("function"), dict):
raise ValueError("tool_calls entries must contain a function object")
function = call["function"]
arguments = function.get("arguments", {})
# OpenAI messages serialize arguments; checkpoint templates iterate them.
if isinstance(arguments, str):
try:
arguments = json.loads(arguments)
except ValueError as exc:
raise ValueError(
"tool_calls function arguments must be a JSON object") from exc
if not isinstance(arguments, dict):
raise ValueError("tool_calls function arguments must be a JSON object")
normalized.append({**call, "function": {**function, "arguments": arguments}})
return normalized
def parse_chat_request(payload: dict) -> ChatRequest:
msgs = []
for m in payload.get("messages", []):
role = str(m.get("role", "user"))
content = m.get("content") or "" # tool-call-only assistant turns
# send content: null, not an omitted key
if isinstance(content, list): # multi-part content: join text parts
content = "".join(
p.get("text", "") for p in content if isinstance(p, dict))
msgs.append(ChatMessage(
role=role, content=str(content),
tool_calls=_template_tool_calls(m.get("tool_calls")),
tool_call_id=m.get("tool_call_id"),
name=m.get("name"),
))
return ChatRequest(
model=str(payload.get("model", "")),
messages=msgs,
temperature=payload.get("temperature"),
top_p=payload.get("top_p"),
top_k=payload.get("top_k"),
# OpenAI deprecated ``max_tokens`` in favour of
# ``max_completion_tokens``, which current SDKs and clients send.
max_tokens=(payload["max_completion_tokens"]
if payload.get("max_completion_tokens") is not None
else payload.get("max_tokens")),
seed=payload.get("seed"),
stream=bool(payload.get("stream", False)),
enable_thinking=payload.get("enable_thinking"),
tools=payload.get("tools"),
tool_choice=payload.get("tool_choice"),
raw=payload,
)
class ChatSession:
"""Tokenize + chat-template + generate for one request."""
def __init__(self, engine: Edge0Engine, req: ChatRequest):
self.engine = engine
self.req = req
self._tok = engine._tok
def _tools(self) -> list[dict] | None:
"""Tool definitions to forward to the chat template, or None if
the request has none or explicitly disabled calling
(``tool_choice: "none"``). Forcing a specific function or
``tool_choice: "required"`` would need constrained decoding,
which isn't implemented; both currently behave like "auto"."""
if not self.req.tools or self.req.tool_choice == "none":
return None
return self.req.tools
def prompt_ids(self) -> list[int]:
tok = self._tok
# Families with a vendored chat template
# (``engine.encode_chat`` renders a chat_template.jinja with
# enable_thinking) must go
# through it — the tokenizer's own apply_chat_template renders a
# DIFFERENT prompt (no think-mode system line, no ``<think>``
# opener), which is exactly the wrong-template bug.
encode = getattr(self.engine, "encode_chat", None)
if callable(encode):
think = self.req.enable_thinking
if think is None:
think = getattr(self.engine, "think", False)
return encode(
[m.__dict__ for m in self.req.messages], think=bool(think),
tools=self._tools())
if hasattr(tok, "apply_chat_template"):
# The qwen35 template supports ``enable_thinking``: False
# renders the canonical no-think prompt — an EMPTY think
# block closer, the model's direct-answer form. Default OFF
# (CLI demo parity).
#
# The successful render MUST be kept: substituting the
# hardcoded ChatML drops that think-block closer, the model
# then opens its own <think> and the turn derails (issue #11:
# <think> leaking into content, then empty/garbled replies).
# ``_chat_text()`` is a last resort, never the success path.
msgs = [m.__dict__ for m in self.req.messages]
think = self.req.enable_thinking
if think is None:
think = False
try:
text = tok.apply_chat_template(
msgs, tokenize=False, add_generation_prompt=True,
enable_thinking=bool(think), tools=self._tools())
except TypeError:
# tokenizer template without the kwarg: plain render
try:
text = tok.apply_chat_template(
msgs, tokenize=False, add_generation_prompt=True)
except Exception: # noqa: BLE001
text = self._chat_text()
except Exception: # noqa: BLE001 — render failed: last resort
text = self._chat_text()
else:
# Tokenizer without a chat template: plain ChatML. (This also
# used to leave ``text`` unbound -> NameError.)
text = self._chat_text()
ids = tok.encode(text)
if not ids:
ids = [tok.bos_token_id or 0]
return ids
def _chat_text(self) -> str:
parts = []
for m in self.req.messages:
if m.role == "system":
parts.append(f"<|im_start|>system\n{m.content}<|im_end|>\n")
elif m.role == "user":
parts.append(f"<|im_start|>user\n{m.content}<|im_end|>\n")
elif m.role == "assistant":
parts.append(f"<|im_start|>assistant\n{m.content}<|im_end|>\n")
parts.append("<|im_start|>assistant\n")
return "".join(parts)
def gen_config(self):
from edge0.config import GenerationConfig
base = getattr(self.engine.cfg, "gen", GenerationConfig())
kw = {}
for k in ("temperature", "top_p", "top_k", "max_new_tokens", "seed"):
v = getattr(self.req, {
"max_new_tokens": "max_tokens",
}.get(k, k), None)
if v is not None:
kw[k] = v
return GenerationConfig(**{**base.__dict__, **kw})
def run(self, on_token=None) -> tuple[list[int], dict]:
t0 = time.perf_counter()
# Per-request clean
# per-request state. A previous degenerate/truncated turn leaves
# bad pre-routing cross-token state and KV behind, which makes every
# later request collapse from its first token.
self.engine.reset()
ids = self.prompt_ids()
gen = self.gen_config()
tokens = self.engine.generate(
ids, gen_config=gen, on_token=on_token)
usage = {
"prompt_tokens": len(ids),
"completion_tokens": len(tokens),
"total_tokens": len(ids) + len(tokens),
}
# ``generate`` stops at EOS (which it does not return) or after
# ``max_new_tokens`` tokens, so reaching the cap means the reply was
# cut off: OpenAI reports that as ``"length"``.
finish_reason = ("length" if len(tokens) >= gen.max_new_tokens
else "stop")
meta = {"wall_s": round(time.perf_counter() - t0, 3)}
return tokens, {"usage": usage, "finish_reason": finish_reason, **meta}
class QueueServer:
"""Single-slot serving loop: serialize generations, stream via callback."""
def __init__(self, engine: Edge0Engine, model_name: str | None = None):
self.engine = engine
self.model_name = model_name or engine.name
self._lock = threading.Lock()
def health(self) -> dict:
st = self.engine.stats()
return {"status": "ok", "model": self.model_name,
"pos": self.engine.pos}
def chat(self, req: ChatRequest, on_token=None) -> dict:
with self._lock:
sess = ChatSession(self.engine, req)
return sess.run(on_token=on_token)
def sse_format(data: dict) -> str:
return f"data: {json.dumps(data, ensure_ascii=False)}\n\n"
def decode_tokens(engine: Edge0Engine, tokens: list[int]) -> str:
tok = engine._tok
if tok is None:
return ""
return tok.decode(tokens)