Run 35B MoE models in ~2.5 GB of RAM. Weights stream from SSD; a trained router predicts loads a step ahead. Mac, iPhone, Android, Windows supported
Python
"""HTTP app layer (Flask optional; stdlib fallback).
Preference order: if ``flask`` is importable it is used (the full
OpenAI-compatible surface with SSE streaming); otherwise a minimal
stdlib ``http.server`` handler serves the same JSON contract and SSE
streaming. Both share ``build_app_handlers`` so the route
semantics stay identical.
"""
from __future__ import annotations
import json
import queue
import threading
import time
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from urllib.parse import urlparse
from edge0.server.chat import (QueueServer, parse_chat_request, sse_format,
decode_tokens)
try: # pragma: no cover - environment dependent
from flask import Flask, Response, jsonify, request # type: ignore
_HAS_FLASK = True
except ImportError: # pragma: no cover
_HAS_FLASK = False
def _error(status: int, msg: str) -> tuple:
return {"error": {"message": msg, "type": "invalid_request"}}, status
def _split_think(text: str, think: bool):
"""With thinking enabled the model
streams the reasoning block first and closes it with ``</think>``
(token 156904); everything before is ``reasoning_content``, after
is ``content``. A generation cut off inside the reasoning block
surfaces what it has so the UI isn't a blank wait."""
if think:
if "</think>" in text:
r, c = text.split("</think>", 1)
return r.strip(), c.strip()
return text.strip(), ""
return "", text
def _extract_tool_calls(server: QueueServer, req, content: str):
"""(content, tool_calls) after stripping any ``<tool_call>`` block the
engine's chat template asked the model to emit. ``tool_calls`` is None
when the engine has no parser (unsupported family) or the request
didn't ask for tools, or when no call was found in ``content``."""
if req.tool_choice == "none" or (not req.tools
and req.tool_choice != "required"):
return content, None
parse = getattr(server.engine, "parse_tool_calls", None)
if not callable(parse):
return content, None
content, calls = parse(content)
return content, (calls or None)
def _chat_once(server: QueueServer, payload: dict):
req = parse_chat_request(payload)
tokens, meta = server.chat(req)
text = decode_tokens(server.engine, tokens)
think = bool(req.enable_thinking if req.enable_thinking is not None
else getattr(server.engine, "think", False))
reasoning, content = _split_think(text, think)
content, tool_calls = _extract_tool_calls(server, req, content)
message = {
"role": "assistant",
"content": content,
"reasoning_content": reasoning,
}
if tool_calls:
message["tool_calls"] = tool_calls
return {
"id": f"chatcmpl-{int(time.time() * 1000)}",
"object": "chat.completion",
"created": int(time.time()),
"model": server.model_name,
"choices": [{
"index": 0,
"message": message,
"finish_reason": ("tool_calls" if tool_calls
else meta["finish_reason"]),
}],
"usage": meta["usage"],
}
def _incremental_suffix(prev: str, text: str) -> str:
"""Suffix of a cumulative decode not yet emitted, holding back a
trailing incomplete character.
A byte-level BPE token is a *byte fragment*: decoding one id in
isolation turns any multi-byte UTF-8 character it belongs to into
U+FFFD. Decoding the cumulative id list instead is correct, and the
next id completes a character that was still incomplete, so a
trailing replacement char is held back until it resolves (never emit
a partial multi-byte character)."""
safe = text.rstrip("\ufffd")
if safe.startswith(prev):
return safe[len(prev):]
# Defensive resync: a prefix-stable byte-level decoder never lands
# here, but if a decoder rewrote text below the emitted prefix, emit
# only past the longest shared prefix instead of re-emitting it.
i = 0
for a, b in zip(safe, prev):
if a != b:
break
i += 1
return safe[i:]
def _chat_stream(server: QueueServer, payload: dict):
req = parse_chat_request(payload)
events = queue.Queue()
finished = object()
request_id = f"chatcmpl-{int(time.time() * 1000)}"
created = int(time.time())
# Tool-call XML (<tool_call>...</tool_call>) must not leak into content
# deltas verbatim -- the whole point of #115 is that a client asking
# for tools gets a parsed message.tool_calls, not raw template markup.
# Detecting a block incrementally, token by token, needs a lookback
# buffer for a tag that can straddle token boundaries; simpler and
# just as correct: buffer the full response and parse once, same as
# the non-streaming path. Requests with no tools (the common case)
# keep the existing immediate per-token streaming, unchanged.
buffering = bool(req.tools) and req.tool_choice != "none" and callable(
getattr(server.engine, "parse_tool_calls", None))
# Keep every id seen so far, decode the cumulative list, emit only the
# newly completed suffix (see _incremental_suffix).
seen_ids: list[int] = []
emitted = ""
def _emit(text: str):
events.put(sse_format({
"id": request_id, "object": "chat.completion.chunk",
"created": created, "model": server.model_name,
"choices": [{"index": 0,
"delta": {"content": text},
"finish_reason": None}],
}).encode("utf-8"))
def on_token(tid: int):
if buffering:
return
nonlocal emitted
seen_ids.append(tid)
text = decode_tokens(server.engine, seen_ids)
delta = _incremental_suffix(emitted, text)
emitted = text.rstrip("\ufffd")
if delta:
_emit(delta)
def produce():
try:
tokens, meta = server.chat(req, on_token=on_token)
finish_reason = meta["finish_reason"]
delta = {}
if buffering:
text = decode_tokens(server.engine, tokens)
think = bool(
req.enable_thinking if req.enable_thinking is not None
else getattr(server.engine, "think", False))
_, content = _split_think(text, think)
content, tool_calls = _extract_tool_calls(
server, req, content)
if tool_calls:
delta = {"content": content, "tool_calls": [
{"index": i, **call} for i, call in enumerate(tool_calls)
]}
finish_reason = "tool_calls"
else:
delta = {"content": content}
else:
# Flush whatever the incremental decode held back (an
# incomplete multi-byte sequence at the very end of the
# generation) so the concatenated deltas equal the
# non-streaming text exactly.
# The generation is over, so nothing will complete a
# trailing U+FFFD any more: emit it as the non-streaming
# response does instead of holding it back forever.
final = decode_tokens(server.engine, seen_ids)
tail = (final[len(emitted):] if final.startswith(emitted)
else _incremental_suffix(emitted, final))
if tail:
_emit(tail)
events.put(sse_format({
"id": request_id, "object": "chat.completion.chunk",
"created": created, "model": server.model_name,
"choices": [{"index": 0, "delta": delta,
"finish_reason": finish_reason}],
"usage": meta["usage"],
}).encode("utf-8"))
events.put(b"data: [DONE]\n\n")
except Exception as exc: # pragma: no cover - transport-dependent
events.put(sse_format({
"error": {"message": str(exc), "type": "server_error"},
}).encode("utf-8"))
finally:
events.put(finished)
threading.Thread(target=produce, daemon=True).start()
while True:
event = events.get()
if event is finished:
return
yield event
def build_app_handlers(server: QueueServer):
"""Return a handler dispatch dict shared by both transports."""
def handle_chat(payload: dict):
# Validate before sending streaming response headers.
try:
parse_chat_request(payload)
except ValueError as exc:
return _error(400, str(exc))
if payload.get("stream"):
if _HAS_FLASK:
return Response(
_chat_stream(server, payload),
mimetype="text/event-stream",
headers={
"Cache-Control": "no-cache",
"X-Accel-Buffering": "no",
},
)
return _chat_stream(server, payload)
return _chat_once(server, payload)
handlers = {
"GET /healthz": lambda: {"status": "ok", "model": server.model_name},
"GET /v1/models": lambda: {
"object": "list",
"data": [{
"id": server.model_name,
"object": "model",
"owned_by": "edge0",
}],
},
"POST /v1/chat/completions": handle_chat,
"POST /v1/completions": lambda p: _error(
400, "text completions not supported; use /v1/chat/completions"),
}
return handlers
def create_app(server: QueueServer):
"""Flask app (raises ImportError when flask is missing)."""
if not _HAS_FLASK:
raise ImportError("flask is required for create_app(); "
"use run_stdlib instead")
app = Flask("edge0")
handlers = build_app_handlers(server)
@app.get("/healthz")
def healthz():
return handlers["GET /healthz"]()
@app.get("/v1/models")
def models():
return handlers["GET /v1/models"]()
@app.post("/v1/chat/completions")
def chat():
payload = request.get_json(force=True, silent=True) or {}
out = handlers["POST /v1/chat/completions"](payload)
if isinstance(out, Response):
return out
if isinstance(out, tuple):
body, status = out
return jsonify(body), status
return jsonify(out)
@app.post("/v1/completions")
def completions():
payload = request.get_json(force=True, silent=True) or {}
out = handlers["POST /v1/completions"](payload)
if isinstance(out, tuple):
body, status = out
return jsonify(body), status
return jsonify(out)
return app
class _StdlibHandler(BaseHTTPRequestHandler):
server_q = None # type: QueueServer
protocol_version = "HTTP/1.1"
def _json(self, status: int, obj):
body = json.dumps(obj, ensure_ascii=False).encode("utf-8")
self.send_response(status)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)
def _sse(self, events):
self.send_response(200)
self.send_header("Content-Type", "text/event-stream; charset=utf-8")
self.send_header("Cache-Control", "no-cache")
self.send_header("X-Accel-Buffering", "no")
self.send_header("Transfer-Encoding", "chunked")
self.send_header("Connection", "close")
self.end_headers()
self.close_connection = True
try:
for body in events:
if not body:
continue
size = f"{len(body):X}".encode("ascii")
self.wfile.write(size + b"\r\n" + body + b"\r\n")
self.wfile.flush()
self.wfile.write(b"0\r\n\r\n")
self.wfile.flush()
except (BrokenPipeError, ConnectionResetError):
pass
def do_GET(self): # noqa: N802
path = urlparse(self.path).path
handlers = build_app_handlers(self.server_q)
if path == "/healthz":
self._json(200, handlers["GET /healthz"]())
elif path == "/v1/models":
self._json(200, handlers["GET /v1/models"]())
else:
self._json(404, {"error": {"message": f"no route {path}"}})
def do_POST(self): # noqa: N802
path = urlparse(self.path).path
handlers = build_app_handlers(self.server_q)
if path not in ("/v1/chat/completions", "/v1/completions"):
self._json(404, {"error": {"message": f"no route {path}"}})
return
length = int(self.headers.get("Content-Length", 0))
payload = json.loads(self.rfile.read(length) or b"{}")
if path == "/v1/chat/completions":
try:
parse_chat_request(payload)
except ValueError as exc:
self._json(400, {"error": {"message": str(exc),
"type": "invalid_request"}})
return
if path == "/v1/chat/completions" and payload.get("stream"):
self._sse(_chat_stream(self.server_q, payload))
return
out = handlers[("POST " + path)](payload)
if isinstance(out, tuple):
body, status = out
self._json(status, body)
else:
self._json(200, out)
def log_message(self, fmt, *args): # quiet by default
pass
def run_stdlib(server: QueueServer, host: str, port: int):
handler = type("Edge0Handler", (_StdlibHandler,),
{"server_q": server})
httpd = ThreadingHTTPServer((host, port), handler)
httpd.serve_forever()
def run_server(server: QueueServer, host: str = "127.0.0.1",
port: int = 8000, use_flask: bool | None = None):
"""Run the HTTP server in the foreground (blocking)."""
if use_flask is None:
use_flask = _HAS_FLASK
if use_flask:
app = create_app(server)
app.run(host=host, port=port, threaded=True)
else:
run_stdlib(server, host, port)