Skip to content

armgit / Edge0

Public Imported from GitHub
0

Run 35B MoE models in ~2.5 GB of RAM. Weights stream from SSD; a trained router predicts loads a step ahead. Mac, iPhone, Android, Windows supported

Python

Merge main into fix/sse-incremental-decode

mozi55e3e091048b
diff --cc python/src/edge0/server/app.py
index 35b2718,fcbda96..a0f836c
--- a/python/src/edge0/server/app.py
+++ b/python/src/edge0/server/app.py
@@@ -102,13 -95,21 +119,23 @@@ def _chat_stream(server: QueueServer, p
finished = object()
request_id = f"chatcmpl-{int(time.time() * 1000)}"
created = int(time.time())
+ # Tool-call XML (<tool_call>...</tool_call>) must not leak into content
+ # deltas verbatim -- the whole point of #115 is that a client asking
+ # for tools gets a parsed message.tool_calls, not raw template markup.
+ # Detecting a block incrementally, token by token, needs a lookback
+ # buffer for a tag that can straddle token boundaries; simpler and
+ # just as correct: buffer the full response and parse once, same as
+ # the non-streaming path. Requests with no tools (the common case)
+ # keep the existing immediate per-token streaming, unchanged.
+ buffering = bool(req.tools) and req.tool_choice != "none" and callable(
+ getattr(server.engine, "parse_tool_calls", None))
- def on_token(tid: int):
- if buffering:
- return
- text = decode_tokens(server.engine, [tid])
+ # Keep every id seen so far, decode the cumulative list, emit only the
+ # newly completed suffix (see _incremental_suffix).
+ seen_ids: list[int] = []
+ emitted = ""
+
+ def _emit(text: str):
events.put(sse_format({
"id": request_id, "object": "chat.completion.chunk",
"created": created, "model": server.model_name,
@@@ -117,26 -118,26 +144,46 @@@
"finish_reason": None}],
}).encode("utf-8"))
+ def on_token(tid: int):
++ if buffering:
++ return
+ nonlocal emitted
+ seen_ids.append(tid)
+ text = decode_tokens(server.engine, seen_ids)
+ delta = _incremental_suffix(emitted, text)
+ emitted = text.rstrip("\ufffd")
+ if delta:
+ _emit(delta)
+
def produce():
try:
- _, meta = server.chat(req, on_token=on_token)
- # Flush whatever the incremental decode held back (an
- # incomplete multi-byte sequence at the very end of the
- # generation) so the concatenated deltas equal the
- # non-streaming text exactly.
- tail = _incremental_suffix(
- emitted, decode_tokens(server.engine, seen_ids))
- if tail:
- _emit(tail)
+ tokens, meta = server.chat(req, on_token=on_token)
+ finish_reason = "stop"
+ delta = {}
+ if buffering:
+ text = decode_tokens(server.engine, tokens)
+ think = bool(
+ req.enable_thinking if req.enable_thinking is not None
+ else getattr(server.engine, "think", False))
+ _, content = _split_think(text, think)
+ content, tool_calls = _extract_tool_calls(
+ server, req, content)
+ if tool_calls:
+ delta = {"content": content, "tool_calls": [
+ {"index": i, **call} for i, call in enumerate(tool_calls)
+ ]}
+ finish_reason = "tool_calls"
+ else:
+ delta = {"content": content}
++ else:
++ # Flush whatever the incremental decode held back (an
++ # incomplete multi-byte sequence at the very end of the
++ # generation) so the concatenated deltas equal the
++ # non-streaming text exactly.
++ tail = _incremental_suffix(
++ emitted, decode_tokens(server.engine, seen_ids))
++ if tail:
++ _emit(tail)
events.put(sse_format({
"id": request_id, "object": "chat.completion.chunk",
"created": created, "model": server.model_name,
diff --cc python/tests/test_server.py
index 5e9351a,c153da2..3eba4d2
--- a/python/tests/test_server.py
+++ b/python/tests/test_server.py
@@@ -113,6 -117,42 +117,49 @@@ class FakeEngine
self.pos = 0
+ class ToolCallTok(FakeTok):
+ """Decodes generated tokens to a Ling-style ``<tool_call>`` block, the
- same shape the real edge0-8b template asks the model to emit."""
++ same shape the real edge0-8b template asks the model to emit.
++ Like a real tokenizer, decode is prefix-stable: the cumulative
++ decode of the id list equals the whole block."""
++
++ # FakeEngine.generate() emits ids 11, 12, 13; each id maps to
++ # one fixed piece so decode([11, 12, 13]) is the full block.
++ _PIECES = {
++ 11: "<tool_call>calculator\n",
++ 12: "<arg_key>expression</arg_key>\n<arg_value>2 + 2</arg_value>\n",
++ 13: "</tool_call>\n",
++ }
+
+ def decode(self, tokens):
- return ("<tool_call>calculator\n"
- "<arg_key>expression</arg_key>\n"
- "<arg_value>2 + 2</arg_value>\n"
- "</tool_call>")
++ return "".join(self._PIECES.get(t, "") for t in tokens)
+
+
+ class ToolCallEngine(FakeEngine):
+ """FakeEngine + the real Ling tool-call parser, so tests exercise the
+ actual edge0.server.tool_calls integration, not a re-implementation
+ of it."""
+
+ def __init__(self, tok=None):
+ super().__init__(tok=tok or ToolCallTok())
+
+ def parse_tool_calls(self, text: str):
+ from edge0.server.tool_calls import parse_ling_tool_calls
+ return parse_ling_tool_calls(text)
+
+
+ _TOOLS = [{
+ "type": "function",
+ "function": {
+ "name": "calculator",
+ "description": "Calculate an arithmetic expression.",
+ "parameters": {"type": "object",
+ "properties": {"expression": {"type": "string"}},
+ "required": ["expression"]},
+ },
+ }]
+
+
def _req(**kw) -> ChatRequest:
payload = {
"model": "fake",
@@@ -293,6 -397,202 +404,202 @@@ def test_chat_once_response_shape()
assert out["usage"]["completion_tokens"] == 3
+ # ---- tool calls (#115) -----------------------------------------------------
+
+
+ @pytest.fixture(params=["ling", "qwen"])
+ def real_template_engine(request):
+ from tokenizers import Tokenizer
+ from tokenizers.models import WordLevel
+ from transformers import PreTrainedTokenizerFast
+
+ family = request.param
+ template_dir = Path(__file__).parent / "fixtures" / "tool_templates" / family
+ template = (template_dir / "chat_template.jinja").read_text(encoding="utf-8")
+ tokenizer = PreTrainedTokenizerFast(
+ tokenizer_object=Tokenizer(WordLevel({"[UNK]": 0}, unk_token="[UNK]")),
+ unk_token="[UNK]", chat_template=template,
+ )
+
+ class TemplateTok(FakeTok):
+ output = (
+ "<tool_call>calculator<arg_key>expression</arg_key>"
+ "<arg_value>2 + 2</arg_value></tool_call>"
+ if family == "ling" else
+ "<tool_call>\n<function=calculator>\n<parameter=expression>\n"
+ "2 + 2\n</parameter>\n</function>\n</tool_call>"
+ )
+
+ def apply_chat_template(self, *args, **kwargs):
+ return tokenizer.apply_chat_template(*args, **kwargs)
+
+ def encode(self, text, **kwargs):
+ return super().encode(text)
+
+ def decode(self, tokens):
+ return self.output
+
+ engine = ToolCallEngine(tok=TemplateTok())
+ if family == "ling":
+ from edge0.engine.ling import Ling8BEngine
+ engine.dir = str(template_dir)
+ engine.think = False
+ engine._chat_tpl = None
+ engine._chat_template = MethodType(Ling8BEngine._chat_template, engine)
+ engine.encode_chat = MethodType(Ling8BEngine.encode_chat, engine)
+ else:
+ from edge0.engine.qwen import Qwen35Engine
+ engine.parse_tool_calls = MethodType(Qwen35Engine.parse_tool_calls, engine)
+ return engine
+
+
+ @pytest.mark.parametrize("stream", [False, True])
+ def test_tool_call_roundtrip_renders_real_template(real_template_engine, stream):
+ engine = real_template_engine
+ server = QueueServer(engine)
+ user = {"role": "user", "content": "Use the calculator for 2 + 2."}
+ payload = {"messages": [user], "tools": _TOOLS}
+ if stream:
+ frames = b"".join(_chat_stream(server, payload)).decode().split("\n\n")
+ chunk = json.loads(frames[-3][6:])["choices"][0]
+ assert chunk["finish_reason"] == "tool_calls"
+ assistant = {"role": "assistant", **chunk["delta"]}
+ # A client accumulates indexed SSE deltas into a message.
+ assistant["tool_calls"] = [
+ {k: v for k, v in call.items() if k != "index"}
+ for call in assistant["tool_calls"]
+ ]
+ else:
+ choice = _chat_once(server, payload)["choices"][0]
+ assert choice["finish_reason"] == "tool_calls"
+ assistant = choice["message"]
+ call = assistant["tool_calls"][0]
+ assert isinstance(call["function"]["arguments"], str)
+ engine._tok.output = "The result is 4."
+ result = _chat_once(server, {
+ "messages": [user, assistant, {
+ "role": "tool", "tool_call_id": call["id"],
+ "name": "calculator", "content": "4",
+ }], "tools": _TOOLS,
+ })
+ prompt = engine._tok.encoded_text
+ assert "<tool_response>\n4\n</tool_response>" in prompt
+ assert "<tool_call>" in prompt
+ assert "2 + 2" in prompt
+ assert result["choices"][0]["message"]["content"] == "The result is 4."
+
+
+ def test_chat_once_ignores_tool_call_text_without_tools_field():
+ """No ``tools`` in the request -> the raw <tool_call> XML the fake
+ generation returns is left as plain content, exactly today's
+ (buggy) behavior -- this only changes when the client opts in."""
+ srv = QueueServer(ToolCallEngine())
+ out = _chat_once(srv, {"messages": [{"role": "user", "content": "hi"}]})
+ ch = out["choices"][0]
+ assert "tool_calls" not in ch["message"]
+ assert "<tool_call>" in ch["message"]["content"]
+ assert ch["finish_reason"] == "stop"
+
+
+ def test_chat_once_parses_tool_calls_when_requested():
+ srv = QueueServer(ToolCallEngine())
+ out = _chat_once(srv, {
+ "messages": [{"role": "user", "content": "compute 2 + 2"}],
+ "tools": _TOOLS, "tool_choice": "auto",
+ })
+ ch = out["choices"][0]
+ assert ch["message"]["content"] is None
+ assert ch["finish_reason"] == "tool_calls"
+ calls = ch["message"]["tool_calls"]
+ assert len(calls) == 1
+ fn = calls[0]["function"]
+ assert fn["name"] == "calculator"
+ assert json.loads(fn["arguments"]) == {"expression": "2 + 2"}
+ assert calls[0]["type"] == "function"
+ assert calls[0]["id"].startswith("call_")
+
+
+ def test_chat_once_tool_choice_none_suppresses_parsing():
+ srv = QueueServer(ToolCallEngine())
+ out = _chat_once(srv, {
+ "messages": [{"role": "user", "content": "compute 2 + 2"}],
+ "tools": _TOOLS, "tool_choice": "none",
+ })
+ ch = out["choices"][0]
+ assert "tool_calls" not in ch["message"]
+ assert "<tool_call>" in ch["message"]["content"]
+
+
+ def test_chat_once_tools_requested_but_engine_cannot_parse():
+ """An engine with no parse_tool_calls (unsupported family) must not
+ crash a tools-enabled request; it just can't extract structured
+ calls, same as today."""
+ srv = QueueServer(FakeEngine())
+ out = _chat_once(srv, {
+ "messages": [{"role": "user", "content": "hi"}], "tools": _TOOLS,
+ })
+ ch = out["choices"][0]
+ assert "tool_calls" not in ch["message"]
+ assert ch["finish_reason"] == "stop"
+
+
+ def test_chat_stream_tools_requested_buffers_and_emits_final_tool_calls():
+ srv = QueueServer(ToolCallEngine())
+ events = _chat_stream(srv, {
+ "messages": [{"role": "user", "content": "compute 2 + 2"}],
+ "stream": True, "tools": _TOOLS, "tool_choice": "auto",
+ })
+ body = b"".join(events)
+ frames = [e for e in body.decode("utf-8").split("\n\n") if e]
+ assert frames[-1] == "data: [DONE]"
+ chunks = [json.loads(e[6:]) for e in frames[:-1]]
+ # No raw <tool_call> XML in any per-token delta -- buffered, not
+ # streamed token-by-token, unlike the no-tools path.
+ for c in chunks[:-1]:
+ assert "<tool_call>" not in json.dumps(c)
+ final = chunks[-1]["choices"][0]
+ assert final["finish_reason"] == "tool_calls"
+ assert final["delta"]["content"] is None
+ calls = final["delta"]["tool_calls"]
+ assert calls[0]["index"] == 0
+ assert calls[0]["function"]["name"] == "calculator"
+ assert json.loads(calls[0]["function"]["arguments"]) == {
+ "expression": "2 + 2"}
+
+
+ def test_chat_stream_indexes_multiple_tool_calls():
+ class MultiCallTok(ToolCallTok):
+ def decode(self, tokens):
+ return super().decode(tokens) * 2
+
+ server = QueueServer(ToolCallEngine(tok=MultiCallTok()))
+ frames = b"".join(_chat_stream(server, {
+ "messages": [{"role": "user", "content": "hi"}], "tools": _TOOLS,
+ })).decode().split("\n\n")
+ calls = json.loads(frames[-3][6:])["choices"][0]["delta"]["tool_calls"]
+ assert [call["index"] for call in calls] == [0, 1]
+ assert calls[0]["id"] != calls[1]["id"]
+
+
+ def test_chat_stream_without_tools_keeps_immediate_per_token_deltas():
+ """Regression guard: requests with no tools must keep the existing
+ immediate per-token streaming -- buffering is scoped to tool-enabled
+ requests only."""
+ srv = QueueServer(ToolCallEngine())
+ events = _chat_stream(srv, {
+ "messages": [{"role": "user", "content": "hi"}], "stream": True,
+ })
+ body = b"".join(events)
+ frames = [e for e in body.decode("utf-8").split("\n\n") if e]
+ chunks = [json.loads(e[6:]) for e in frames[:-1]]
+ deltas = [c["choices"][0]["delta"].get("content")
+ for c in chunks if c["choices"][0]["delta"]]
- # unchanged from the no-tools path: whatever decode_tokens([tid])
- # returns per call, not the whole buffered response
++ # unchanged from the no-tools path: one delta per token, each
++ # emitted as the token arrives -- not the whole buffered response
+ assert len(deltas) == 3
+ assert chunks[-1]["choices"][0]["finish_reason"] == "stop"
+
+
def test_chat_stream_events_sequence():
srv = QueueServer(FakeEngine())
events = _chat_stream(srv, {