FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
feat(server): enable reasoning_effort on /v1/chat/completions (#2)
Xiaoze Fan4f510f1ba6a7
diff --git a/python/freetoken/server/api_models.py b/python/freetoken/server/api_models.pyindex a2def1ae..b147db4e 100644--- a/python/freetoken/server/api_models.py+++ b/python/freetoken/server/api_models.py@@ -77,6 +77,7 @@ class ChatCompletionRequest(BaseModel):presence_penalty: float = 0.0frequency_penalty: float = 0.0chat_template_kwargs: dict[str, Any] = Field(default_factory=dict)+ reasoning_effort: str | None = Noneignore_eos: bool = Falsetools: list[Tool] | None = Nonetool_choice: Literal["none", "auto", "required"] | ToolChoiceObject | None = Nonediff --git a/python/freetoken/server/generation.py b/python/freetoken/server/generation.pyindex bd8741b8..5d2db719 100644--- a/python/freetoken/server/generation.py+++ b/python/freetoken/server/generation.py@@ -317,6 +317,10 @@ def _make_reasoning_parser(spec: GenSpec, state: Any) -> ReasoningParser | None:# enable_thinking is explicitly false, so the model emits only the closing# </think>. Mirror that default here, else the chain-of-thought leaks into content.force_reasoning = (spec.chat_template_kwargs or {}).get("enable_thinking") is not False+ elif parser_name == "glm":+ # GLM's template honors enable_thinking (default on) even with tools; the+ # generic fallback would force thinking and mislabel disabled output as reasoning.+ force_reasoning = (spec.chat_template_kwargs or {}).get("enable_thinking") is not Falseelif parser_name == "gemma4":# Gemma4 defaults thinking off even when tools are present: its template injects an# empty thought channel before generation. Do not let Codex tool definitions make alldiff --git a/python/freetoken/server/model_meta.py b/python/freetoken/server/model_meta.pyindex d976c924..c8b0f8f0 100644--- a/python/freetoken/server/model_meta.py+++ b/python/freetoken/server/model_meta.py@@ -67,6 +67,32 @@ def think_toggle_kwargs(reasoning_parser: str | None, enabled: bool) -> dict:return think_chat_template_kwargs(reasoning_parser, gear)+_THINKING_KWARG_KEYS = ("enable_thinking", "thinking", "thinking_mode", "reasoning_effort")+_DISABLE_EFFORTS = ("none", "off")+++def effort_toggle_kwargs(+ reasoning_parser: str | None,+ effort: str | None,+ chat_template_kwargs: dict | None,+) -> dict:+ """Fold a protocol-level reasoning-effort request into the template kwargs.+ An explicit thinking-related key wins wholesale; unrelated extras ride along.+ Effort "none"/"off" (case-insensitive) disables thinking; any other or absent+ effort enables it, forwarded for templates that grade it (gpt-oss)."""+ ctk = dict(chat_template_kwargs or {})+ if any(key in ctk for key in _THINKING_KWARG_KEYS):+ return ctk+ if isinstance(effort, str):+ effort = effort.strip().lower()+ disabled = effort in _DISABLE_EFFORTS+ mapped = dict(think_toggle_kwargs(reasoning_parser, not disabled))+ if effort and not disabled:+ mapped.setdefault("reasoning_effort", effort)+ mapped.update(ctk)+ return mapped++def moe_total_experts(config: Any) -> int:"""Total routed-expert slots the model has: experts per layer x MoE layers. Matches theengine's own basis (``Engine._resolve_auto_moe_cache_size``), so a residency rate deriveddiff --git a/python/freetoken/server/openai_api.py b/python/freetoken/server/openai_api.pyindex 5f3f81e0..4b793502 100644--- a/python/freetoken/server/openai_api.py+++ b/python/freetoken/server/openai_api.py@@ -37,8 +37,17 @@ from .generation import ()-def chat_request_to_genspec(req: ChatCompletionRequest, model_sampling: dict[str, Any]) -> GenSpec:+def chat_request_to_genspec(+ req: ChatCompletionRequest,+ model_sampling: dict[str, Any],+ reasoning_parser: str | None = None,+) -> GenSpec:"""OpenAI ChatCompletionRequest -> GenSpec (the OpenAI 'to_sampling_params')."""+ from .model_meta import effort_toggle_kwargs++ ctk = req.chat_template_kwargs+ if req.reasoning_effort:+ ctk = effort_toggle_kwargs(reasoning_parser, req.reasoning_effort, ctk)return GenSpec(messages=render_messages([m.model_dump(exclude_none=True) for m in req.messages]),sampling_params=resolve_sampling(@@ -50,7 +59,7 @@ def chat_request_to_genspec(req: ChatCompletionRequest, model_sampling: dict[strmodel_sampling=model_sampling,stop=req.stop,),- chat_template_kwargs=req.chat_template_kwargs,+ chat_template_kwargs=ctk,template_tools=_tools_for_template(req),parser_tools=(_all_tool_dicts(req.tools) if _should_parse_tools(req) else None),)@@ -133,7 +142,9 @@ async def handle_chat_completion(return create_error_response("Only n=1 is supported", param="n")try:- spec = chat_request_to_genspec(req, model_sampling)+ spec = chat_request_to_genspec(+ req, model_sampling, reasoning_parser=getattr(state.config, "reasoning_parser", None)+ )except ValueError as exc:return create_error_response(str(exc))uid = await submit_generation(spec, state)@@ -182,7 +193,9 @@ async def stream_chat_completion_chunks() -> AsyncIterator[bytes]:"""Format generate_events() into the OpenAI chat.completion.chunk SSE stream."""if spec is None:- spec = chat_request_to_genspec(req, {})+ spec = chat_request_to_genspec(+ req, {}, reasoning_parser=getattr(state.config, "reasoning_parser", None)+ )yield _sse(_chat_chunk(req,diff --git a/python/freetoken/server/responses_api.py b/python/freetoken/server/responses_api.pyindex 31a316f8..9ed1f1ac 100644--- a/python/freetoken/server/responses_api.py+++ b/python/freetoken/server/responses_api.py@@ -230,19 +230,11 @@ def convert_responses_to_genspec(else:template_tools, parser_tools = split_tool_lists(raw_tools, selected)- # Thinking toggle: an explicit chat_template_kwargs extra field wins; else- # the protocol-native `reasoning` object drives the template through the- # per-family mapping in model_meta. effort "none" disables thinking (vLLM- # semantics); other efforts enable it and are forwarded for templates that- # grade them (gpt-oss).- from .model_meta import think_toggle_kwargs+ from .model_meta import effort_toggle_kwargsctk = dict(getattr(req, "chat_template_kwargs", None) or {})- if req.reasoning and not ctk:- effort = req.reasoning.get("effort")- ctk = dict(think_toggle_kwargs(reasoning_parser, effort != "none"))- if effort and effort != "none":- ctk.setdefault("reasoning_effort", effort)+ if req.reasoning:+ ctk = effort_toggle_kwargs(reasoning_parser, req.reasoning.get("effort"), ctk)return GenSpec(messages=render_messages(messages),diff --git a/tests/server/test_openai_api.py b/tests/server/test_openai_api.pyindex 69e6173f..709203c5 100644--- a/tests/server/test_openai_api.py+++ b/tests/server/test_openai_api.py@@ -161,6 +161,65 @@ def test_chat_request_reasoning_replay_field_aliases():assert asst["thinking"] == "prior thought", field+def test_chat_reasoning_effort_enables_thinking():+ spec = chat_request_to_genspec(chat_request(reasoning_effort="high"), {})+ assert spec.chat_template_kwargs == {"enable_thinking": True, "reasoning_effort": "high"}++ # an explicit thinking-related chat_template_kwargs key wins over the mapping+ spec = chat_request_to_genspec(+ chat_request(reasoning_effort="none", chat_template_kwargs={"enable_thinking": True}), {}+ )+ assert spec.chat_template_kwargs == {"enable_thinking": True}++ # unrelated extra kwargs ride along without discarding the effort mapping+ spec = chat_request_to_genspec(+ chat_request(reasoning_effort="none", chat_template_kwargs={"custom_var": 1}), {}+ )+ assert spec.chat_template_kwargs == {"enable_thinking": False, "custom_var": 1}++ # absent effort -> kwargs pass through untouched+ assert chat_request_to_genspec(chat_request(), {}).chat_template_kwargs == {}+++def test_chat_reasoning_effort_none_disables_thinking():+ # vLLM-compatible semantics: an explicit effort "none" DISABLES thinking.+ spec = chat_request_to_genspec(chat_request(reasoning_effort="none"), {})+ assert spec.chat_template_kwargs == {"enable_thinking": False}+++def test_chat_reasoning_effort_routes_through_family_mapping():+ """The toggle goes through model_meta's per-family mapping -- for M3 that is+ thinking_mode, not the (inert) enable_thinking key."""+ on = chat_request(reasoning_effort="high")+ spec = chat_request_to_genspec(on, {}, reasoning_parser="minimax_m3")+ assert spec.chat_template_kwargs == {"thinking_mode": "enabled", "reasoning_effort": "high"}++ off = chat_request(reasoning_effort="none")+ spec = chat_request_to_genspec(off, {}, reasoning_parser="minimax_m3")+ assert spec.chat_template_kwargs == {"thinking_mode": "disabled"}++ # gpt-oss: the template grades effort and has no off gear+ spec = chat_request_to_genspec(on, {}, reasoning_parser="gpt_oss")+ assert spec.chat_template_kwargs == {"reasoning_effort": "high"}+ spec = chat_request_to_genspec(off, {}, reasoning_parser="gpt_oss")+ assert spec.chat_template_kwargs == {}+++def test_glm_reasoning_parser_honors_disabled_thinking_with_tools():+ # The parse side must match the encode side: thinking off + tools present+ # must not start the parser inside a think block.+ from freetoken.server.generation import _make_reasoning_parser++ state = FakeState([], reasoning_parser="glm")+ off = chat_request_to_genspec(chat_request(reasoning_effort="none"), {}, reasoning_parser="glm")+ parser = _make_reasoning_parser(off, state)+ assert parser is not None and parser.detector.force_reasoning is False++ on = chat_request_to_genspec(chat_request(), {}, reasoning_parser="glm")+ parser = _make_reasoning_parser(on, state)+ assert parser is not None and parser.detector.force_reasoning is True++def test_non_stream_chat_completion_returns_openai_tool_calls_and_sends_tools():output = '[TOOL_CALLS] [{"name":"get_weather","arguments":{"city":"Paris"}}]'state = FakeState(diff --git a/tests/server/test_responses_api.py b/tests/server/test_responses_api.pyindex ef9e602d..9950c665 100644--- a/tests/server/test_responses_api.py+++ b/tests/server/test_responses_api.py@@ -867,7 +867,7 @@ def test_convert_reasoning_field_enables_thinking():spec = RP.convert_responses_to_genspec(req, {})assert spec.chat_template_kwargs == {"enable_thinking": True, "reasoning_effort": "high"}- # explicit chat_template_kwargs extra field wins over the reasoning mapping+ # an explicit thinking-related chat_template_kwargs key wins over the mappingreq2 = ResponsesRequest.model_validate({"model": "m", "input": "hi", "reasoning": {"effort": "low"},"chat_template_kwargs": {"thinking_mode": "chat"}}@@ -875,9 +875,18 @@ def test_convert_reasoning_field_enables_thinking():spec2 = RP.convert_responses_to_genspec(req2, {})assert spec2.chat_template_kwargs == {"thinking_mode": "chat"}+ # unrelated extra kwargs ride along without discarding the reasoning mapping+ req3 = ResponsesRequest.model_validate(+ {"model": "m", "input": "hi", "reasoning": {"effort": "none"},+ "chat_template_kwargs": {"custom_var": 1}}+ )+ assert RP.convert_responses_to_genspec(req3, {}).chat_template_kwargs == {+ "enable_thinking": False, "custom_var": 1,+ }+# absent reasoning -> no kwargs- req3 = ResponsesRequest.model_validate({"model": "m", "input": "hi"})- assert RP.convert_responses_to_genspec(req3, {}).chat_template_kwargs == {}+ req4 = ResponsesRequest.model_validate({"model": "m", "input": "hi"})+ assert RP.convert_responses_to_genspec(req4, {}).chat_template_kwargs == {}def test_convert_reasoning_effort_none_disables_thinking():