Run 35B MoE models in ~2.5 GB of RAM. Weights stream from SSD; a trained router predicts loads a step ahead. Mac, iPhone, Android, Windows supported
Python
fix(server): report finish_reason "length" when max_tokens cuts a reply off
Anubhav Jain37075514b549
diff --git a/python/src/edge0/server/app.py b/python/src/edge0/server/app.pyindex a72a39f..9bec35c 100644--- a/python/src/edge0/server/app.py+++ b/python/src/edge0/server/app.py@@ -83,7 +83,8 @@ def _chat_once(server: QueueServer, payload: dict):"choices": [{"index": 0,"message": message,- "finish_reason": "tool_calls" if tool_calls else "stop",+ "finish_reason": ("tool_calls" if tool_calls+ else meta["finish_reason"]),}],"usage": meta["usage"],}@@ -158,7 +159,7 @@ def _chat_stream(server: QueueServer, payload: dict):def produce():try:tokens, meta = server.chat(req, on_token=on_token)- finish_reason = "stop"+ finish_reason = meta["finish_reason"]delta = {}if buffering:text = decode_tokens(server.engine, tokens)diff --git a/python/src/edge0/server/chat.py b/python/src/edge0/server/chat.pyindex 5b42cca..0de3c1e 100644--- a/python/src/edge0/server/chat.py+++ b/python/src/edge0/server/chat.py@@ -212,8 +212,13 @@ class ChatSession:"completion_tokens": len(tokens),"total_tokens": len(ids) + len(tokens),}+ # ``generate`` stops at EOS (which it does not return) or after+ # ``max_new_tokens`` tokens, so reaching the cap means the reply was+ # cut off: OpenAI reports that as ``"length"``.+ finish_reason = ("length" if len(tokens) >= gen.max_new_tokens+ else "stop")meta = {"wall_s": round(time.perf_counter() - t0, 3)}- return tokens, {"usage": usage, **meta}+ return tokens, {"usage": usage, "finish_reason": finish_reason, **meta}class QueueServer:diff --git a/python/tests/test_server.py b/python/tests/test_server.pyindex 31bbda6..a7d5bc9 100644--- a/python/tests/test_server.py+++ b/python/tests/test_server.py@@ -412,6 +412,46 @@ def test_chat_once_response_shape():assert out["usage"]["completion_tokens"] == 3+class CappedEngine(FakeEngine):+ """FakeEngine whose generate() honours ``max_new_tokens`` like the real+ engine: it returns at most that many tokens (the reply is cut off)."""++ def generate(self, ids, gen_config=None, on_token=None, **kw):+ toks = [11, 12, 13][:gen_config.max_new_tokens]+ self.generated.append((list(ids), gen_config))+ if on_token is not None:+ for t in toks:+ on_token(t)+ return toks++[email protected]("max_tokens, finish_reason", [+ (2, "length"), # cut off at the cap+ (3, "length"), # exactly at the cap: generate() never saw EOS+ (8, "stop"), # EOS before the cap+])+def test_chat_once_finish_reason_length_when_capped(max_tokens, finish_reason):+ srv = QueueServer(CappedEngine())+ out = _chat_once(srv, {+ "messages": [{"role": "user", "content": "hi"}],+ "max_tokens": max_tokens,+ })+ assert out["choices"][0]["finish_reason"] == finish_reason+ assert out["usage"]["completion_tokens"] == min(max_tokens, 3)+++def test_chat_stream_finish_reason_length_when_capped():+ srv = QueueServer(CappedEngine())+ body = b"".join(_chat_stream(srv, {+ "messages": [{"role": "user", "content": "hi"}],+ "stream": True, "max_completion_tokens": 2,+ }))+ events = [e for e in body.decode("utf-8").split("\n\n") if e]+ chunks = [json.loads(e[6:]) for e in events[:-1]] # drop [DONE]+ assert chunks[-1]["choices"][0]["finish_reason"] == "length"+ assert chunks[-1]["usage"]["completion_tokens"] == 2++# ---- tool calls (#115) -----------------------------------------------------