Run 35B MoE models in ~2.5 GB of RAM. Weights stream from SSD; a trained router predicts loads a step ahead. Mac, iPhone, Android, Windows supported
Python
"""Server-layer tests: request parsing, chat session, SSE framing and the
stdlib HTTP transport, all against a fake engine (no real checkpoints).
Covers the OpenAI-compatible surface the deployment servers expose:
/healthz, /v1/models, POST /v1/chat/completions (non-streaming and SSE
streaming), and the /v1/completions rejection path.
"""
from __future__ import annotations
import json
import threading
import time
from http.server import ThreadingHTTPServer
from pathlib import Path
from types import MethodType, SimpleNamespace
from urllib.error import HTTPError
from urllib import request as urlrequest
import pytest
from edge0.config import GenerationConfig
from edge0.server.app import (
_HAS_FLASK,
_chat_once,
_chat_stream,
_StdlibHandler,
build_app_handlers,
create_app,
run_stdlib,
)
from edge0.server.chat import (
ChatMessage,
ChatRequest,
ChatSession,
QueueServer,
decode_tokens,
parse_chat_request,
sse_format,
)
# ---- fake engine ----------------------------------------------------------
class FakeTok:
"""Minimal tokenizer stub (transformers-like surface)."""
def __init__(self, ids=(7, 8, 9)):
self.ids = list(ids)
self.bos_token_id = 0
self.encoded_text = None
def apply_chat_template(self, messages, tokenize=False,
add_generation_prompt=True,
enable_thinking=False, tools=None):
text = "<tpl>" + json.dumps(messages)
if tools:
text += "<tools>" + json.dumps(tools)
if add_generation_prompt:
text += "<|im_start|>assistant\n"
return text
def encode(self, text):
self.encoded_text = text
return self.ids
def decode(self, tokens):
return "".join(f"T{t}" for t in tokens)
class PlainTok:
"""Tokenizer without apply_chat_template (stdlib fallback path).
A standalone class rather than a FakeTok subclass: deleting the
inherited method off the instance does not work, so ``hasattr`` would
stay True and the fallback would never be exercised."""
def __init__(self, ids=(7, 8, 9)):
self.ids = list(ids)
self.bos_token_id = 0
self.encoded_text = None
def encode(self, text):
self.encoded_text = text
return self.ids
def decode(self, tokens):
return "".join(f"T{t}" for t in tokens)
class FakeEngine:
"""Duck-typed engine: enough of Edge0Engine for the server layer."""
name = "fake"
def __init__(self, tok=None):
self._tok = tok or FakeTok()
self.pos = 3
self.cfg = SimpleNamespace(
gen=GenerationConfig(temperature=0.7, top_p=0.95))
self.generated = []
def generate(self, ids, gen_config=None, on_token=None, **kw):
toks = [11, 12, 13]
self.generated.append((list(ids), gen_config))
if on_token is not None:
for t in toks:
on_token(t)
return toks
def stats(self):
return {"ok": 1}
def reset(self):
"""Per-request state clear (server parity with the real engine)."""
self.pos = 0
class ToolCallTok(FakeTok):
"""Decodes generated tokens to a Ling-style ``<tool_call>`` block, the
same shape the real edge0-8b template asks the model to emit.
Like a real tokenizer, decode is prefix-stable: the cumulative
decode of the id list equals the whole block."""
# FakeEngine.generate() emits ids 11, 12, 13; each id maps to
# one fixed piece so decode([11, 12, 13]) is the full block.
_PIECES = {
11: "<tool_call>calculator\n",
12: "<arg_key>expression</arg_key>\n<arg_value>2 + 2</arg_value>\n",
13: "</tool_call>\n",
}
def decode(self, tokens):
return "".join(self._PIECES.get(t, "") for t in tokens)
class ToolCallEngine(FakeEngine):
"""FakeEngine + the real Ling tool-call parser, so tests exercise the
actual edge0.server.tool_calls integration, not a re-implementation
of it."""
def __init__(self, tok=None):
super().__init__(tok=tok or ToolCallTok())
def parse_tool_calls(self, text: str):
from edge0.server.tool_calls import parse_ling_tool_calls
return parse_ling_tool_calls(text)
_TOOLS = [{
"type": "function",
"function": {
"name": "calculator",
"description": "Calculate an arithmetic expression.",
"parameters": {"type": "object",
"properties": {"expression": {"type": "string"}},
"required": ["expression"]},
},
}]
def _req(**kw) -> ChatRequest:
payload = {
"model": "fake",
"messages": [{"role": "user", "content": "hi"}],
**kw,
}
return parse_chat_request(payload)
# ---- parse_chat_request ---------------------------------------------------
def test_parse_basic_defaults():
req = _req()
assert req.model == "fake"
assert len(req.messages) == 1
assert req.messages[0].role == "user"
assert req.messages[0].content == "hi"
assert req.temperature is None
assert req.stream is False
def test_parse_multipart_content_joins_text():
req = _req(messages=[{
"role": "user",
"content": [{"type": "text", "text": "a"},
{"type": "image_url", "image_url": {"url": "x"}},
{"type": "text", "text": "b"}],
}])
assert req.messages[0].content == "ab"
def test_parse_stream_flag_and_sampling():
req = _req(temperature=0.3, top_p=0.9, top_k=32, max_tokens=16,
stream=True)
assert req.temperature == 0.3
assert req.top_p == 0.9
assert req.top_k == 32
assert req.max_tokens == 16
assert req.stream is True
def test_parse_max_completion_tokens():
# The current OpenAI name for the limit; it wins over the deprecated
# ``max_tokens`` when a client sends both.
assert _req(max_completion_tokens=24).max_tokens == 24
assert _req(max_completion_tokens=24, max_tokens=16).max_tokens == 24
assert _req(max_completion_tokens=None, max_tokens=16).max_tokens == 16
def test_parse_preserves_tools_and_tool_choice():
req = _req(tools=_TOOLS, tool_choice="auto")
assert req.tools == _TOOLS
assert req.tool_choice == "auto"
def test_parse_message_tool_calls_and_tool_role_roundtrip():
req = parse_chat_request({"messages": [
{"role": "user", "content": "compute 2+2"},
{"role": "assistant", "content": None, "tool_calls": [
{"id": "call_1", "type": "function",
"function": {"name": "calculator",
"arguments": '{"expression": "2 + 2"}'}}]},
{"role": "tool", "tool_call_id": "call_1", "content": "4"},
]})
assert req.messages[1].tool_calls[0]["function"]["name"] == "calculator"
assert req.messages[1].content == "" # null, not the string "None"
assert req.messages[2].tool_call_id == "call_1"
@pytest.mark.parametrize("arguments", ['{"expression": "2 + 2"}',
{"expression": "2 + 2"}])
def test_parse_normalizes_tool_arguments_without_mutating_payload(arguments):
payload = {"messages": [{"role": "assistant", "content": None,
"tool_calls": [{
"id": "call_1", "type": "function",
"function": {"name": "calculator",
"arguments": arguments},
}]}]}
before = json.dumps(payload)
req = parse_chat_request(payload)
call = req.messages[0].tool_calls[0]
assert call["function"]["arguments"] == {"expression": "2 + 2"}
assert call["id"] == "call_1"
assert json.dumps(payload) == before
@pytest.mark.parametrize("arguments", ["{broken", "[]", "null", '"text"',
"42", [], None])
def test_parse_rejects_invalid_tool_arguments(arguments):
with pytest.raises(ValueError, match="arguments.*JSON object"):
parse_chat_request({"messages": [{
"role": "assistant", "content": None,
"tool_calls": [{"type": "function", "function": {
"name": "calculator", "arguments": arguments}}],
}]})
@pytest.mark.parametrize("calls", ["bad", {}, [None], [{"function": "bad"}]])
def test_parse_rejects_invalid_tool_call_shapes(calls):
with pytest.raises(ValueError, match="tool_calls"):
parse_chat_request({"messages": [{
"role": "assistant", "content": None, "tool_calls": calls,
}]})
def test_parse_tool_call_without_arguments_uses_empty_object():
req = parse_chat_request({"messages": [{
"role": "assistant", "content": None,
"tool_calls": [{"function": {"name": "get_time"}}],
}]})
assert req.messages[0].tool_calls[0]["function"]["arguments"] == {}
# ---- sse / decode ---------------------------------------------------------
def test_sse_format():
line = sse_format({"id": "1", "x": "中文"})
assert line.startswith("data: ")
assert line.endswith("\n\n")
assert json.loads(line[6:-2]) == {"id": "1", "x": "中文"}
def test_decode_tokens():
eng = FakeEngine()
assert decode_tokens(eng, [1, 2]) == "T1T2"
eng._tok = None
assert decode_tokens(eng, [1]) == ""
# ---- ChatSession ----------------------------------------------------------
def test_chat_session_prompt_ids_uses_template():
eng = FakeEngine()
sess = ChatSession(eng, _req())
ids = sess.prompt_ids()
assert ids == [7, 8, 9]
# What actually gets encoded is the RENDERED template, not the
# hardcoded ChatML (issue #11: the latter loses the template's empty
# <think></think> closer and derails the turn).
assert eng._tok.encoded_text.startswith("<tpl>")
assert eng._tok.encoded_text.endswith("<|im_start|>assistant\n")
assert eng._tok.encoded_text != sess._chat_text()
# The template was called with enable_thinking=False (serve.py parity).
text = eng._tok.apply_chat_template([{"role": "user", "content": "hi"}])
assert "<|im_start|>assistant" in text
def test_chat_session_prompt_ids_fallback_text():
eng = FakeEngine(tok=PlainTok())
sess = ChatSession(eng, _req())
ids = sess.prompt_ids()
assert ids == [7, 8, 9]
# No template on the tokenizer -> hardcoded ChatML (this path used to
# raise NameError on the unbound ``text``).
assert eng._tok.encoded_text == sess._chat_text()
def test_chat_session_fallback_text_format():
eng = FakeEngine(tok=PlainTok())
req = parse_chat_request({"messages": [
{"role": "system", "content": "sys"},
{"role": "user", "content": "q"},
{"role": "assistant", "content": "a"},
]})
sess = ChatSession(eng, req)
text = sess._chat_text()
assert "<|im_start|>system\nsys<|im_end|>\n" in text
assert "<|im_start|>user\nq<|im_end|>\n" in text
assert "<|im_start|>assistant\na<|im_end|>\n" in text
assert text.endswith("<|im_start|>assistant\n")
def test_gen_config_overrides_request_fields():
eng = FakeEngine()
sess = ChatSession(eng, _req(temperature=0.1, max_tokens=5))
gen = sess.gen_config()
assert gen.temperature == 0.1
assert gen.max_new_tokens == 5
assert gen.top_p == 0.95 # engine default preserved
def test_chat_session_run_usage_and_callback():
eng = FakeEngine()
seen = []
sess = ChatSession(eng, _req())
tokens, meta = sess.run(on_token=seen.append)
assert tokens == [11, 12, 13]
assert seen == [11, 12, 13]
assert meta["usage"] == {
"prompt_tokens": 3,
"completion_tokens": 3,
"total_tokens": 6,
}
assert "wall_s" in meta
# ---- QueueServer ----------------------------------------------------------
def test_queue_server_health():
eng = FakeEngine()
srv = QueueServer(eng)
h = srv.health()
assert h["status"] == "ok"
assert h["model"] == "fake"
assert h["pos"] == 3
def test_queue_server_chat_serializes():
eng = FakeEngine()
srv = QueueServer(eng)
tokens, meta = srv.chat(_req())
assert meta["usage"]["completion_tokens"] == 3
assert len(eng.generated) == 1
gen = eng.generated[0][1]
assert gen.temperature == 0.7
# ---- handlers -------------------------------------------------------------
def test_handlers_healthz_and_models():
srv = QueueServer(FakeEngine())
handlers = build_app_handlers(srv)
assert handlers["GET /healthz"]()["status"] == "ok"
models = handlers["GET /v1/models"]()
assert models["object"] == "list"
assert models["data"][0]["id"] == "fake"
def test_handlers_completions_rejected():
srv = QueueServer(FakeEngine())
handlers = build_app_handlers(srv)
body, status = handlers["POST /v1/completions"]({})
assert status == 400
def test_chat_once_response_shape():
srv = QueueServer(FakeEngine())
out = _chat_once(srv, {"messages": [{"role": "user", "content": "hi"}]})
assert out["object"] == "chat.completion"
assert out["model"] == "fake"
ch = out["choices"][0]
assert ch["message"]["role"] == "assistant"
assert ch["message"]["content"] == "T11T12T13"
assert ch["finish_reason"] == "stop"
assert out["usage"]["completion_tokens"] == 3
class CappedEngine(FakeEngine):
"""FakeEngine whose generate() honours ``max_new_tokens`` like the real
engine: it returns at most that many tokens (the reply is cut off)."""
def generate(self, ids, gen_config=None, on_token=None, **kw):
toks = [11, 12, 13][:gen_config.max_new_tokens]
self.generated.append((list(ids), gen_config))
if on_token is not None:
for t in toks:
on_token(t)
return toks
@pytest.mark.parametrize("max_tokens, finish_reason", [
(2, "length"), # cut off at the cap
(3, "length"), # exactly at the cap: generate() never saw EOS
(8, "stop"), # EOS before the cap
])
def test_chat_once_finish_reason_length_when_capped(max_tokens, finish_reason):
srv = QueueServer(CappedEngine())
out = _chat_once(srv, {
"messages": [{"role": "user", "content": "hi"}],
"max_tokens": max_tokens,
})
assert out["choices"][0]["finish_reason"] == finish_reason
assert out["usage"]["completion_tokens"] == min(max_tokens, 3)
def test_chat_stream_finish_reason_length_when_capped():
srv = QueueServer(CappedEngine())
body = b"".join(_chat_stream(srv, {
"messages": [{"role": "user", "content": "hi"}],
"stream": True, "max_completion_tokens": 2,
}))
events = [e for e in body.decode("utf-8").split("\n\n") if e]
chunks = [json.loads(e[6:]) for e in events[:-1]] # drop [DONE]
assert chunks[-1]["choices"][0]["finish_reason"] == "length"
assert chunks[-1]["usage"]["completion_tokens"] == 2
# ---- tool calls (#115) -----------------------------------------------------
@pytest.fixture(params=["ling", "qwen"])
def real_template_engine(request):
from tokenizers import Tokenizer
from tokenizers.models import WordLevel
from transformers import PreTrainedTokenizerFast
family = request.param
template_dir = Path(__file__).parent / "fixtures" / "tool_templates" / family
template = (template_dir / "chat_template.jinja").read_text(encoding="utf-8")
tokenizer = PreTrainedTokenizerFast(
tokenizer_object=Tokenizer(WordLevel({"[UNK]": 0}, unk_token="[UNK]")),
unk_token="[UNK]", chat_template=template,
)
class TemplateTok(FakeTok):
output = (
"<tool_call>calculator<arg_key>expression</arg_key>"
"<arg_value>2 + 2</arg_value></tool_call>"
if family == "ling" else
"<tool_call>\n<function=calculator>\n<parameter=expression>\n"
"2 + 2\n</parameter>\n</function>\n</tool_call>"
)
def apply_chat_template(self, *args, **kwargs):
return tokenizer.apply_chat_template(*args, **kwargs)
def encode(self, text, **kwargs):
return super().encode(text)
def decode(self, tokens):
return self.output
engine = ToolCallEngine(tok=TemplateTok())
if family == "ling":
from edge0.engine.ling import Ling8BEngine
engine.dir = str(template_dir)
engine.think = False
engine._chat_tpl = None
engine._chat_template = MethodType(Ling8BEngine._chat_template, engine)
engine.encode_chat = MethodType(Ling8BEngine.encode_chat, engine)
else:
from edge0.engine.qwen import Qwen35Engine
engine.parse_tool_calls = MethodType(Qwen35Engine.parse_tool_calls, engine)
return engine
@pytest.mark.parametrize("stream", [False, True])
def test_tool_call_roundtrip_renders_real_template(real_template_engine, stream):
engine = real_template_engine
server = QueueServer(engine)
user = {"role": "user", "content": "Use the calculator for 2 + 2."}
payload = {"messages": [user], "tools": _TOOLS}
if stream:
frames = b"".join(_chat_stream(server, payload)).decode().split("\n\n")
chunk = json.loads(frames[-3][6:])["choices"][0]
assert chunk["finish_reason"] == "tool_calls"
assistant = {"role": "assistant", **chunk["delta"]}
# A client accumulates indexed SSE deltas into a message.
assistant["tool_calls"] = [
{k: v for k, v in call.items() if k != "index"}
for call in assistant["tool_calls"]
]
else:
choice = _chat_once(server, payload)["choices"][0]
assert choice["finish_reason"] == "tool_calls"
assistant = choice["message"]
call = assistant["tool_calls"][0]
assert isinstance(call["function"]["arguments"], str)
engine._tok.output = "The result is 4."
result = _chat_once(server, {
"messages": [user, assistant, {
"role": "tool", "tool_call_id": call["id"],
"name": "calculator", "content": "4",
}], "tools": _TOOLS,
})
prompt = engine._tok.encoded_text
assert "<tool_response>\n4\n</tool_response>" in prompt
assert "<tool_call>" in prompt
assert "2 + 2" in prompt
assert result["choices"][0]["message"]["content"] == "The result is 4."
def test_chat_once_ignores_tool_call_text_without_tools_field():
"""No ``tools`` in the request -> the raw <tool_call> XML the fake
generation returns is left as plain content, exactly today's
(buggy) behavior -- this only changes when the client opts in."""
srv = QueueServer(ToolCallEngine())
out = _chat_once(srv, {"messages": [{"role": "user", "content": "hi"}]})
ch = out["choices"][0]
assert "tool_calls" not in ch["message"]
assert "<tool_call>" in ch["message"]["content"]
assert ch["finish_reason"] == "stop"
def test_chat_once_parses_tool_calls_when_requested():
srv = QueueServer(ToolCallEngine())
out = _chat_once(srv, {
"messages": [{"role": "user", "content": "compute 2 + 2"}],
"tools": _TOOLS, "tool_choice": "auto",
})
ch = out["choices"][0]
assert ch["message"]["content"] is None
assert ch["finish_reason"] == "tool_calls"
calls = ch["message"]["tool_calls"]
assert len(calls) == 1
fn = calls[0]["function"]
assert fn["name"] == "calculator"
assert json.loads(fn["arguments"]) == {"expression": "2 + 2"}
assert calls[0]["type"] == "function"
assert calls[0]["id"].startswith("call_")
def test_chat_once_tool_choice_none_suppresses_parsing():
srv = QueueServer(ToolCallEngine())
out = _chat_once(srv, {
"messages": [{"role": "user", "content": "compute 2 + 2"}],
"tools": _TOOLS, "tool_choice": "none",
})
ch = out["choices"][0]
assert "tool_calls" not in ch["message"]
assert "<tool_call>" in ch["message"]["content"]
def test_chat_once_tools_requested_but_engine_cannot_parse():
"""An engine with no parse_tool_calls (unsupported family) must not
crash a tools-enabled request; it just can't extract structured
calls, same as today."""
srv = QueueServer(FakeEngine())
out = _chat_once(srv, {
"messages": [{"role": "user", "content": "hi"}], "tools": _TOOLS,
})
ch = out["choices"][0]
assert "tool_calls" not in ch["message"]
assert ch["finish_reason"] == "stop"
def test_chat_stream_tools_requested_buffers_and_emits_final_tool_calls():
srv = QueueServer(ToolCallEngine())
events = _chat_stream(srv, {
"messages": [{"role": "user", "content": "compute 2 + 2"}],
"stream": True, "tools": _TOOLS, "tool_choice": "auto",
})
body = b"".join(events)
frames = [e for e in body.decode("utf-8").split("\n\n") if e]
assert frames[-1] == "data: [DONE]"
chunks = [json.loads(e[6:]) for e in frames[:-1]]
# No raw <tool_call> XML in any per-token delta -- buffered, not
# streamed token-by-token, unlike the no-tools path.
for c in chunks[:-1]:
assert "<tool_call>" not in json.dumps(c)
final = chunks[-1]["choices"][0]
assert final["finish_reason"] == "tool_calls"
assert final["delta"]["content"] is None
calls = final["delta"]["tool_calls"]
assert calls[0]["index"] == 0
assert calls[0]["function"]["name"] == "calculator"
assert json.loads(calls[0]["function"]["arguments"]) == {
"expression": "2 + 2"}
def test_chat_stream_indexes_multiple_tool_calls():
class MultiCallTok(ToolCallTok):
def decode(self, tokens):
return super().decode(tokens) * 2
server = QueueServer(ToolCallEngine(tok=MultiCallTok()))
frames = b"".join(_chat_stream(server, {
"messages": [{"role": "user", "content": "hi"}], "tools": _TOOLS,
})).decode().split("\n\n")
calls = json.loads(frames[-3][6:])["choices"][0]["delta"]["tool_calls"]
assert [call["index"] for call in calls] == [0, 1]
assert calls[0]["id"] != calls[1]["id"]
def test_chat_stream_without_tools_keeps_immediate_per_token_deltas():
"""Regression guard: requests with no tools must keep the existing
immediate per-token streaming -- buffering is scoped to tool-enabled
requests only."""
srv = QueueServer(ToolCallEngine())
events = _chat_stream(srv, {
"messages": [{"role": "user", "content": "hi"}], "stream": True,
})
body = b"".join(events)
frames = [e for e in body.decode("utf-8").split("\n\n") if e]
chunks = [json.loads(e[6:]) for e in frames[:-1]]
deltas = [c["choices"][0]["delta"].get("content")
for c in chunks if c["choices"][0]["delta"]]
# unchanged from the no-tools path: one delta per token, each
# emitted as the token arrives -- not the whole buffered response
assert len(deltas) == 3
assert chunks[-1]["choices"][0]["finish_reason"] == "stop"
def test_chat_stream_events_sequence():
srv = QueueServer(FakeEngine())
events = _chat_stream(srv, {
"messages": [{"role": "user", "content": "hi"}], "stream": True,
})
body = b"".join(events)
events = [e for e in body.decode("utf-8").split("\n\n") if e]
assert events[-1] == "data: [DONE]"
chunks = [json.loads(e[6:]) for e in events[:-1]] # drop [DONE]
deltas = [c["choices"][0]["delta"]["content"]
for c in chunks if c["choices"][0]["delta"]]
assert deltas == ["T11", "T12", "T13"]
assert chunks[-1]["choices"][0]["finish_reason"] == "stop"
assert chunks[-1]["choices"][0]["delta"] == {}
class PausingFakeEngine(FakeEngine):
"""Pause after the first callback so transport buffering is observable."""
def __init__(self):
super().__init__()
self.first_token = threading.Event()
self.resume = threading.Event()
self.finished = threading.Event()
def generate(self, ids, gen_config=None, on_token=None, **kw):
toks = [11, 12, 13]
self.generated.append((list(ids), gen_config))
if on_token is not None:
on_token(toks[0])
self.first_token.set()
self.resume.wait(timeout=5)
for tid in toks[1:]:
on_token(tid)
self.finished.set()
return toks
def test_chat_stream_yields_before_generation_finishes():
eng = PausingFakeEngine()
srv = QueueServer(eng)
stream = iter(_chat_stream(srv, {
"messages": [{"role": "user", "content": "hi"}],
"stream": True,
}))
try:
first = next(stream)
assert first.startswith(b"data: ")
assert eng.first_token.is_set()
assert not eng.finished.is_set()
eng.resume.set()
body = first + b"".join(stream)
finally:
eng.resume.set()
assert b"data: [DONE]\n\n" in body
# ---- stdlib HTTP transport ------------------------------------------------
@pytest.mark.parametrize("stream", [False, True])
@pytest.mark.parametrize("transport", ["stdlib", "flask"])
def test_http_rejects_invalid_tool_arguments_before_generation(stream, transport):
engine = FakeEngine()
server = QueueServer(engine)
payload = {"stream": stream, "messages": [{
"role": "assistant", "content": None,
"tool_calls": [{"type": "function", "function": {
"name": "calculator", "arguments": "{broken"}}],
}]}
if transport == "flask":
if not _HAS_FLASK:
pytest.skip("flask is not installed")
response = create_app(server).test_client().post(
"/v1/chat/completions", json=payload)
assert response.status_code == 400
error = response.get_json()
else:
handler = type("Edge0Handler", (_StdlibHandler,), {"server_q": server})
httpd = ThreadingHTTPServer(("127.0.0.1", 0), handler)
thread = threading.Thread(target=httpd.serve_forever, daemon=True)
thread.start()
try:
req = urlrequest.Request(
f"http://127.0.0.1:{httpd.server_address[1]}/v1/chat/completions",
data=json.dumps(payload).encode(),
headers={"Content-Type": "application/json"},
)
with pytest.raises(HTTPError) as exc:
urlrequest.urlopen(req, timeout=10)
with exc.value as response:
assert response.code == 400
error = json.loads(response.read())
finally:
httpd.shutdown()
httpd.server_close()
thread.join(timeout=5)
assert "arguments must be a JSON object" in error["error"]["message"]
assert engine.generated == []
def test_stdlib_http_end_to_end():
eng = FakeEngine()
srv = QueueServer(eng, model_name="fake")
handler = type("Edge0Handler", (_StdlibHandler,), {"server_q": srv})
httpd = ThreadingHTTPServer(("127.0.0.1", 0), handler)
port = httpd.server_address[1]
t = threading.Thread(target=httpd.serve_forever, daemon=True)
t.start()
base = f"http://127.0.0.1:{port}"
try:
with urlrequest.urlopen(f"{base}/healthz", timeout=10) as r:
assert r.status == 200
assert json.loads(r.read())["status"] == "ok"
with urlrequest.urlopen(f"{base}/v1/models", timeout=10) as r:
assert json.loads(r.read())["data"][0]["id"] == "fake"
payload = json.dumps({
"model": "fake",
"messages": [{"role": "user", "content": "hi"}],
"max_tokens": 8,
}).encode("utf-8")
req = urlrequest.Request(f"{base}/v1/chat/completions",
data=payload,
headers={"Content-Type": "application/json"})
with urlrequest.urlopen(req, timeout=10) as r:
assert r.status == 200
out = json.loads(r.read())
assert out["choices"][0]["message"]["content"] == "T11T12T13"
# unknown route -> 404
with pytest.raises(Exception) as exc:
urlrequest.urlopen(f"{base}/nope", timeout=10)
assert getattr(exc.value, "code", None) == 404
finally:
httpd.shutdown()
httpd.server_close()
t.join(timeout=5)
def test_stdlib_http_streams_before_generation_finishes():
eng = PausingFakeEngine()
srv = QueueServer(eng, model_name="fake")
handler = type("Edge0Handler", (_StdlibHandler,), {"server_q": srv})
httpd = ThreadingHTTPServer(("127.0.0.1", 0), handler)
port = httpd.server_address[1]
t = threading.Thread(target=httpd.serve_forever, daemon=True)
t.start()
payload = json.dumps({
"model": "fake",
"messages": [{"role": "user", "content": "hi"}],
"stream": True,
}).encode("utf-8")
req = urlrequest.Request(
f"http://127.0.0.1:{port}/v1/chat/completions",
data=payload,
headers={"Content-Type": "application/json"},
)
try:
with urlrequest.urlopen(req, timeout=10) as response:
assert response.status == 200
assert response.headers["Transfer-Encoding"] == "chunked"
first = response.readline()
assert first.startswith(b"data: ")
assert eng.first_token.is_set()
assert not eng.finished.is_set()
eng.resume.set()
rest = response.read()
assert b"data: [DONE]\n\n" in first + rest
finally:
eng.resume.set()
httpd.shutdown()
httpd.server_close()
t.join(timeout=5)
@pytest.mark.skipif(not _HAS_FLASK, reason="flask is not installed")
def test_flask_http_streams_before_generation_finishes():
eng = PausingFakeEngine()
app = create_app(QueueServer(eng, model_name="fake"))
response = app.test_client().post(
"/v1/chat/completions",
json={
"model": "fake",
"messages": [{"role": "user", "content": "hi"}],
"stream": True,
},
buffered=False,
)
try:
assert response.status_code == 200
assert response.headers["Cache-Control"] == "no-cache"
stream = iter(response.response)
first = next(stream)
assert first.startswith(b"data: ")
assert eng.first_token.is_set()
assert not eng.finished.is_set()
eng.resume.set()
body = first + b"".join(stream)
finally:
eng.resume.set()
response.close()
assert b"data: [DONE]\n\n" in body
# ---- SSE incremental decode (multi-byte characters split across ids) ------
def _byte_level_bpe(seed_texts):
"""A real byte-level BPE (the family the shipped Qwen/Ling tokenizers
belong to): out-of-vocabulary characters fall back to per-byte tokens,
so a multi-byte character ends up split across several ids."""
from tokenizers import (Tokenizer, decoders, models, pre_tokenizers,
trainers)
tok = Tokenizer(models.BPE())
tok.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=False)
tok.decoder = decoders.ByteLevel()
trainer = trainers.BpeTrainer(
vocab_size=300, special_tokens=["<|endoftext|>"],
initial_alphabet=pre_tokenizers.ByteLevel.alphabet())
tok.train_from_iterator(seed_texts, trainer=trainer)
return tok
class _BpeTok:
"""Engine-tokenizer facade over a byte-level BPE (what the server sees)."""
def __init__(self, tok):
self._tok = tok
self.bos_token_id = 0
def apply_chat_template(self, messages, tokenize=False,
add_generation_prompt=True,
enable_thinking=False):
return "<|im_start|>user\nx<|im_end|>\n<|im_start|>assistant\n"
def encode(self, text):
return list(self._tok.encode(text).ids)
def decode(self, tokens):
return self._tok.decode(list(tokens))
class SplitEngine:
"""Engine that emits a fixed id sequence whose bytes split a character."""
name = "split"
def __init__(self, tok, ids):
self._tok = tok
self.ids = list(ids)
self.pos = 0
self.cfg = SimpleNamespace(gen=GenerationConfig())
def generate(self, ids, gen_config=None, on_token=None, **kw):
if on_token is not None:
for t in self.ids:
on_token(t)
return list(self.ids)
def stats(self):
return {}
def reset(self):
self.pos = 0
def test_chat_stream_incremental_decode_of_split_characters():
"""The streamed deltas must equal the non-streaming text.
A byte-level BPE token carries a byte fragment, not a character:
decoding each id in isolation turns any multi-byte UTF-8 character it
belongs to into U+FFFD, so the old per-id path corrupted every
non-ASCII reply while the non-streaming path (whole id list) stayed
correct. Assert on that exact trigger: an id sequence whose bytes
split a character, streamed through the handler.
"""
tok = _byte_level_bpe(["你好世界 hello world", "中文测试 沿着海滨",
"日本語テスト"])
trigger = "你好,用一句话介绍海滨城市。"
ids = list(tok.encode(trigger).ids)
# Precondition: the trigger really splits a character across ids, so
# per-id decode is corrupt by construction (this is the bug).
isolated = [tok.decode([i]) for i in ids]
assert any("\ufffd" in p for p in isolated), \
"trigger does not split a multi-byte character across ids"
assert "\ufffd" not in tok.decode(ids)
eng = SplitEngine(_BpeTok(tok), ids)
srv = QueueServer(eng, model_name="split")
events = _chat_stream(srv, {
"messages": [{"role": "user", "content": "hi"}], "stream": True,
})
body = b"".join(events)
chunks = [json.loads(e[6:]) for e in
body.decode("utf-8").split("\n\n")
if e and e != "data: [DONE]"]
deltas = [c["choices"][0]["delta"]["content"]
for c in chunks if c["choices"][0]["delta"].get("content")]
streamed = "".join(deltas)
non_streamed = decode_tokens(eng, ids)
assert streamed == non_streamed == trigger
assert "\ufffd" not in streamed
class _CutTok(FakeTok):
"""Decodes like a byte-level tokenizer whose generation stopped in the
middle of a multi-byte character (e.g. at max_tokens): the last id
leaves an incomplete sequence that decodes to U+FFFD."""
_PIECES = {11: "ok ", 12: "fine", 13: "\ufffd"}
def decode(self, tokens):
return "".join(self._PIECES.get(t, "") for t in tokens)
def test_chat_stream_flushes_trailing_replacement_char_at_end():
"""Concatenated deltas must equal the non-streaming text even when the
generation ends on an incomplete character, which no later id can
complete."""
eng = FakeEngine(tok=_CutTok())
srv = QueueServer(eng, model_name="fake")
body = b"".join(_chat_stream(srv, {
"messages": [{"role": "user", "content": "hi"}], "stream": True,
}))
chunks = [json.loads(e[6:]) for e in body.decode("utf-8").split("\n\n")
if e and e != "data: [DONE]"]
streamed = "".join(c["choices"][0]["delta"].get("content") or ""
for c in chunks)
assert streamed == decode_tokens(eng, [11, 12, 13]) == "ok fine\ufffd"