Skip to content

armgit / Edge0

Public Imported from GitHub
0

Run 35B MoE models in ~2.5 GB of RAM. Weights stream from SSD; a trained router predicts loads a step ahead. Mac, iPhone, Android, Windows supported

Python

fix(server): honor max_completion_tokens in chat requests

kevin9327bc7e8c4b609b
diff --git a/python/src/edge0/server/chat.py b/python/src/edge0/server/chat.py
index 1cdb93a..5b42cca 100644
--- a/python/src/edge0/server/chat.py
+++ b/python/src/edge0/server/chat.py
@@ -87,7 +87,11 @@ def parse_chat_request(payload: dict) -> ChatRequest:
temperature=payload.get("temperature"),
top_p=payload.get("top_p"),
top_k=payload.get("top_k"),
- max_tokens=payload.get("max_tokens"),
+ # OpenAI deprecated ``max_tokens`` in favour of
+ # ``max_completion_tokens``, which current SDKs and clients send.
+ max_tokens=(payload["max_completion_tokens"]
+ if payload.get("max_completion_tokens") is not None
+ else payload.get("max_tokens")),
seed=payload.get("seed"),
stream=bool(payload.get("stream", False)),
enable_thinking=payload.get("enable_thinking"),
diff --git a/python/tests/test_server.py b/python/tests/test_server.py
index 3eba4d2..d99f82c 100644
--- a/python/tests/test_server.py
+++ b/python/tests/test_server.py
@@ -202,6 +202,14 @@ def test_parse_stream_flag_and_sampling():
assert req.stream is True
+def test_parse_max_completion_tokens():
+ # The current OpenAI name for the limit; it wins over the deprecated
+ # ``max_tokens`` when a client sends both.
+ assert _req(max_completion_tokens=24).max_tokens == 24
+ assert _req(max_completion_tokens=24, max_tokens=16).max_tokens == 24
+ assert _req(max_completion_tokens=None, max_tokens=16).max_tokens == 16
+
+
def test_parse_preserves_tools_and_tool_choice():
req = _req(tools=_TOOLS, tool_choice="auto")
assert req.tools == _TOOLS