Skip to content

armgit / Edge0

Public Imported from GitHub
0

Run 35B MoE models in ~2.5 GB of RAM. Weights stream from SSD; a trained router predicts loads a step ahead. Mac, iPhone, Android, Windows supported

Python

fix(server): report finish_reason "length" when max_tokens cuts a reply off

Anubhav Jain37075514b549
diff --git a/python/src/edge0/server/app.py b/python/src/edge0/server/app.py
index a72a39f..9bec35c 100644
--- a/python/src/edge0/server/app.py
+++ b/python/src/edge0/server/app.py
@@ -83,7 +83,8 @@ def _chat_once(server: QueueServer, payload: dict):
"choices": [{
"index": 0,
"message": message,
- "finish_reason": "tool_calls" if tool_calls else "stop",
+ "finish_reason": ("tool_calls" if tool_calls
+ else meta["finish_reason"]),
}],
"usage": meta["usage"],
}
@@ -158,7 +159,7 @@ def _chat_stream(server: QueueServer, payload: dict):
def produce():
try:
tokens, meta = server.chat(req, on_token=on_token)
- finish_reason = "stop"
+ finish_reason = meta["finish_reason"]
delta = {}
if buffering:
text = decode_tokens(server.engine, tokens)
diff --git a/python/src/edge0/server/chat.py b/python/src/edge0/server/chat.py
index 5b42cca..0de3c1e 100644
--- a/python/src/edge0/server/chat.py
+++ b/python/src/edge0/server/chat.py
@@ -212,8 +212,13 @@ class ChatSession:
"completion_tokens": len(tokens),
"total_tokens": len(ids) + len(tokens),
}
+ # ``generate`` stops at EOS (which it does not return) or after
+ # ``max_new_tokens`` tokens, so reaching the cap means the reply was
+ # cut off: OpenAI reports that as ``"length"``.
+ finish_reason = ("length" if len(tokens) >= gen.max_new_tokens
+ else "stop")
meta = {"wall_s": round(time.perf_counter() - t0, 3)}
- return tokens, {"usage": usage, **meta}
+ return tokens, {"usage": usage, "finish_reason": finish_reason, **meta}
class QueueServer:
diff --git a/python/tests/test_server.py b/python/tests/test_server.py
index 31bbda6..a7d5bc9 100644
--- a/python/tests/test_server.py
+++ b/python/tests/test_server.py
@@ -412,6 +412,46 @@ def test_chat_once_response_shape():
assert out["usage"]["completion_tokens"] == 3
+class CappedEngine(FakeEngine):
+ """FakeEngine whose generate() honours ``max_new_tokens`` like the real
+ engine: it returns at most that many tokens (the reply is cut off)."""
+
+ def generate(self, ids, gen_config=None, on_token=None, **kw):
+ toks = [11, 12, 13][:gen_config.max_new_tokens]
+ self.generated.append((list(ids), gen_config))
+ if on_token is not None:
+ for t in toks:
+ on_token(t)
+ return toks
+
+
[email protected]("max_tokens, finish_reason", [
+ (2, "length"), # cut off at the cap
+ (3, "length"), # exactly at the cap: generate() never saw EOS
+ (8, "stop"), # EOS before the cap
+])
+def test_chat_once_finish_reason_length_when_capped(max_tokens, finish_reason):
+ srv = QueueServer(CappedEngine())
+ out = _chat_once(srv, {
+ "messages": [{"role": "user", "content": "hi"}],
+ "max_tokens": max_tokens,
+ })
+ assert out["choices"][0]["finish_reason"] == finish_reason
+ assert out["usage"]["completion_tokens"] == min(max_tokens, 3)
+
+
+def test_chat_stream_finish_reason_length_when_capped():
+ srv = QueueServer(CappedEngine())
+ body = b"".join(_chat_stream(srv, {
+ "messages": [{"role": "user", "content": "hi"}],
+ "stream": True, "max_completion_tokens": 2,
+ }))
+ events = [e for e in body.decode("utf-8").split("\n\n") if e]
+ chunks = [json.loads(e[6:]) for e in events[:-1]] # drop [DONE]
+ assert chunks[-1]["choices"][0]["finish_reason"] == "length"
+ assert chunks[-1]["usage"]["completion_tokens"] == 2
+
+
# ---- tool calls (#115) -----------------------------------------------------