Run 35B MoE models in ~2.5 GB of RAM. Weights stream from SSD; a trained router predicts loads a step ahead. Mac, iPhone, Android, Windows supported
Python
fix(server): honor max_completion_tokens in chat requests
kevin9327bc7e8c4b609b
diff --git a/python/src/edge0/server/chat.py b/python/src/edge0/server/chat.pyindex 1cdb93a..5b42cca 100644--- a/python/src/edge0/server/chat.py+++ b/python/src/edge0/server/chat.py@@ -87,7 +87,11 @@ def parse_chat_request(payload: dict) -> ChatRequest:temperature=payload.get("temperature"),top_p=payload.get("top_p"),top_k=payload.get("top_k"),- max_tokens=payload.get("max_tokens"),+ # OpenAI deprecated ``max_tokens`` in favour of+ # ``max_completion_tokens``, which current SDKs and clients send.+ max_tokens=(payload["max_completion_tokens"]+ if payload.get("max_completion_tokens") is not None+ else payload.get("max_tokens")),seed=payload.get("seed"),stream=bool(payload.get("stream", False)),enable_thinking=payload.get("enable_thinking"),diff --git a/python/tests/test_server.py b/python/tests/test_server.pyindex 3eba4d2..d99f82c 100644--- a/python/tests/test_server.py+++ b/python/tests/test_server.py@@ -202,6 +202,14 @@ def test_parse_stream_flag_and_sampling():assert req.stream is True+def test_parse_max_completion_tokens():+ # The current OpenAI name for the limit; it wins over the deprecated+ # ``max_tokens`` when a client sends both.+ assert _req(max_completion_tokens=24).max_tokens == 24+ assert _req(max_completion_tokens=24, max_tokens=16).max_tokens == 24+ assert _req(max_completion_tokens=None, max_tokens=16).max_tokens == 16++def test_parse_preserves_tools_and_tool_choice():req = _req(tools=_TOOLS, tool_choice="auto")assert req.tools == _TOOLS