Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

feat(server): enable reasoning_effort on /v1/chat/completions (#2)

Xiaoze Fan4f510f1ba6a7
diff --git a/python/freetoken/server/api_models.py b/python/freetoken/server/api_models.py
index a2def1ae..b147db4e 100644
--- a/python/freetoken/server/api_models.py
+++ b/python/freetoken/server/api_models.py
@@ -77,6 +77,7 @@ class ChatCompletionRequest(BaseModel):
presence_penalty: float = 0.0
frequency_penalty: float = 0.0
chat_template_kwargs: dict[str, Any] = Field(default_factory=dict)
+ reasoning_effort: str | None = None
ignore_eos: bool = False
tools: list[Tool] | None = None
tool_choice: Literal["none", "auto", "required"] | ToolChoiceObject | None = None
diff --git a/python/freetoken/server/generation.py b/python/freetoken/server/generation.py
index bd8741b8..5d2db719 100644
--- a/python/freetoken/server/generation.py
+++ b/python/freetoken/server/generation.py
@@ -317,6 +317,10 @@ def _make_reasoning_parser(spec: GenSpec, state: Any) -> ReasoningParser | None:
# enable_thinking is explicitly false, so the model emits only the closing
# </think>. Mirror that default here, else the chain-of-thought leaks into content.
force_reasoning = (spec.chat_template_kwargs or {}).get("enable_thinking") is not False
+ elif parser_name == "glm":
+ # GLM's template honors enable_thinking (default on) even with tools; the
+ # generic fallback would force thinking and mislabel disabled output as reasoning.
+ force_reasoning = (spec.chat_template_kwargs or {}).get("enable_thinking") is not False
elif parser_name == "gemma4":
# Gemma4 defaults thinking off even when tools are present: its template injects an
# empty thought channel before generation. Do not let Codex tool definitions make all
diff --git a/python/freetoken/server/model_meta.py b/python/freetoken/server/model_meta.py
index d976c924..c8b0f8f0 100644
--- a/python/freetoken/server/model_meta.py
+++ b/python/freetoken/server/model_meta.py
@@ -67,6 +67,32 @@ def think_toggle_kwargs(reasoning_parser: str | None, enabled: bool) -> dict:
return think_chat_template_kwargs(reasoning_parser, gear)
+_THINKING_KWARG_KEYS = ("enable_thinking", "thinking", "thinking_mode", "reasoning_effort")
+_DISABLE_EFFORTS = ("none", "off")
+
+
+def effort_toggle_kwargs(
+ reasoning_parser: str | None,
+ effort: str | None,
+ chat_template_kwargs: dict | None,
+) -> dict:
+ """Fold a protocol-level reasoning-effort request into the template kwargs.
+ An explicit thinking-related key wins wholesale; unrelated extras ride along.
+ Effort "none"/"off" (case-insensitive) disables thinking; any other or absent
+ effort enables it, forwarded for templates that grade it (gpt-oss)."""
+ ctk = dict(chat_template_kwargs or {})
+ if any(key in ctk for key in _THINKING_KWARG_KEYS):
+ return ctk
+ if isinstance(effort, str):
+ effort = effort.strip().lower()
+ disabled = effort in _DISABLE_EFFORTS
+ mapped = dict(think_toggle_kwargs(reasoning_parser, not disabled))
+ if effort and not disabled:
+ mapped.setdefault("reasoning_effort", effort)
+ mapped.update(ctk)
+ return mapped
+
+
def moe_total_experts(config: Any) -> int:
"""Total routed-expert slots the model has: experts per layer x MoE layers. Matches the
engine's own basis (``Engine._resolve_auto_moe_cache_size``), so a residency rate derived
diff --git a/python/freetoken/server/openai_api.py b/python/freetoken/server/openai_api.py
index 5f3f81e0..4b793502 100644
--- a/python/freetoken/server/openai_api.py
+++ b/python/freetoken/server/openai_api.py
@@ -37,8 +37,17 @@ from .generation import (
)
-def chat_request_to_genspec(req: ChatCompletionRequest, model_sampling: dict[str, Any]) -> GenSpec:
+def chat_request_to_genspec(
+ req: ChatCompletionRequest,
+ model_sampling: dict[str, Any],
+ reasoning_parser: str | None = None,
+) -> GenSpec:
"""OpenAI ChatCompletionRequest -> GenSpec (the OpenAI 'to_sampling_params')."""
+ from .model_meta import effort_toggle_kwargs
+
+ ctk = req.chat_template_kwargs
+ if req.reasoning_effort:
+ ctk = effort_toggle_kwargs(reasoning_parser, req.reasoning_effort, ctk)
return GenSpec(
messages=render_messages([m.model_dump(exclude_none=True) for m in req.messages]),
sampling_params=resolve_sampling(
@@ -50,7 +59,7 @@ def chat_request_to_genspec(req: ChatCompletionRequest, model_sampling: dict[str
model_sampling=model_sampling,
stop=req.stop,
),
- chat_template_kwargs=req.chat_template_kwargs,
+ chat_template_kwargs=ctk,
template_tools=_tools_for_template(req),
parser_tools=(_all_tool_dicts(req.tools) if _should_parse_tools(req) else None),
)
@@ -133,7 +142,9 @@ async def handle_chat_completion(
return create_error_response("Only n=1 is supported", param="n")
try:
- spec = chat_request_to_genspec(req, model_sampling)
+ spec = chat_request_to_genspec(
+ req, model_sampling, reasoning_parser=getattr(state.config, "reasoning_parser", None)
+ )
except ValueError as exc:
return create_error_response(str(exc))
uid = await submit_generation(spec, state)
@@ -182,7 +193,9 @@ async def stream_chat_completion_chunks(
) -> AsyncIterator[bytes]:
"""Format generate_events() into the OpenAI chat.completion.chunk SSE stream."""
if spec is None:
- spec = chat_request_to_genspec(req, {})
+ spec = chat_request_to_genspec(
+ req, {}, reasoning_parser=getattr(state.config, "reasoning_parser", None)
+ )
yield _sse(
_chat_chunk(
req,
diff --git a/python/freetoken/server/responses_api.py b/python/freetoken/server/responses_api.py
index 31a316f8..9ed1f1ac 100644
--- a/python/freetoken/server/responses_api.py
+++ b/python/freetoken/server/responses_api.py
@@ -230,19 +230,11 @@ def convert_responses_to_genspec(
else:
template_tools, parser_tools = split_tool_lists(raw_tools, selected)
- # Thinking toggle: an explicit chat_template_kwargs extra field wins; else
- # the protocol-native `reasoning` object drives the template through the
- # per-family mapping in model_meta. effort "none" disables thinking (vLLM
- # semantics); other efforts enable it and are forwarded for templates that
- # grade them (gpt-oss).
- from .model_meta import think_toggle_kwargs
+ from .model_meta import effort_toggle_kwargs
ctk = dict(getattr(req, "chat_template_kwargs", None) or {})
- if req.reasoning and not ctk:
- effort = req.reasoning.get("effort")
- ctk = dict(think_toggle_kwargs(reasoning_parser, effort != "none"))
- if effort and effort != "none":
- ctk.setdefault("reasoning_effort", effort)
+ if req.reasoning:
+ ctk = effort_toggle_kwargs(reasoning_parser, req.reasoning.get("effort"), ctk)
return GenSpec(
messages=render_messages(messages),
diff --git a/tests/server/test_openai_api.py b/tests/server/test_openai_api.py
index 69e6173f..709203c5 100644
--- a/tests/server/test_openai_api.py
+++ b/tests/server/test_openai_api.py
@@ -161,6 +161,65 @@ def test_chat_request_reasoning_replay_field_aliases():
assert asst["thinking"] == "prior thought", field
+def test_chat_reasoning_effort_enables_thinking():
+ spec = chat_request_to_genspec(chat_request(reasoning_effort="high"), {})
+ assert spec.chat_template_kwargs == {"enable_thinking": True, "reasoning_effort": "high"}
+
+ # an explicit thinking-related chat_template_kwargs key wins over the mapping
+ spec = chat_request_to_genspec(
+ chat_request(reasoning_effort="none", chat_template_kwargs={"enable_thinking": True}), {}
+ )
+ assert spec.chat_template_kwargs == {"enable_thinking": True}
+
+ # unrelated extra kwargs ride along without discarding the effort mapping
+ spec = chat_request_to_genspec(
+ chat_request(reasoning_effort="none", chat_template_kwargs={"custom_var": 1}), {}
+ )
+ assert spec.chat_template_kwargs == {"enable_thinking": False, "custom_var": 1}
+
+ # absent effort -> kwargs pass through untouched
+ assert chat_request_to_genspec(chat_request(), {}).chat_template_kwargs == {}
+
+
+def test_chat_reasoning_effort_none_disables_thinking():
+ # vLLM-compatible semantics: an explicit effort "none" DISABLES thinking.
+ spec = chat_request_to_genspec(chat_request(reasoning_effort="none"), {})
+ assert spec.chat_template_kwargs == {"enable_thinking": False}
+
+
+def test_chat_reasoning_effort_routes_through_family_mapping():
+ """The toggle goes through model_meta's per-family mapping -- for M3 that is
+ thinking_mode, not the (inert) enable_thinking key."""
+ on = chat_request(reasoning_effort="high")
+ spec = chat_request_to_genspec(on, {}, reasoning_parser="minimax_m3")
+ assert spec.chat_template_kwargs == {"thinking_mode": "enabled", "reasoning_effort": "high"}
+
+ off = chat_request(reasoning_effort="none")
+ spec = chat_request_to_genspec(off, {}, reasoning_parser="minimax_m3")
+ assert spec.chat_template_kwargs == {"thinking_mode": "disabled"}
+
+ # gpt-oss: the template grades effort and has no off gear
+ spec = chat_request_to_genspec(on, {}, reasoning_parser="gpt_oss")
+ assert spec.chat_template_kwargs == {"reasoning_effort": "high"}
+ spec = chat_request_to_genspec(off, {}, reasoning_parser="gpt_oss")
+ assert spec.chat_template_kwargs == {}
+
+
+def test_glm_reasoning_parser_honors_disabled_thinking_with_tools():
+ # The parse side must match the encode side: thinking off + tools present
+ # must not start the parser inside a think block.
+ from freetoken.server.generation import _make_reasoning_parser
+
+ state = FakeState([], reasoning_parser="glm")
+ off = chat_request_to_genspec(chat_request(reasoning_effort="none"), {}, reasoning_parser="glm")
+ parser = _make_reasoning_parser(off, state)
+ assert parser is not None and parser.detector.force_reasoning is False
+
+ on = chat_request_to_genspec(chat_request(), {}, reasoning_parser="glm")
+ parser = _make_reasoning_parser(on, state)
+ assert parser is not None and parser.detector.force_reasoning is True
+
+
def test_non_stream_chat_completion_returns_openai_tool_calls_and_sends_tools():
output = '[TOOL_CALLS] [{"name":"get_weather","arguments":{"city":"Paris"}}]'
state = FakeState(
diff --git a/tests/server/test_responses_api.py b/tests/server/test_responses_api.py
index ef9e602d..9950c665 100644
--- a/tests/server/test_responses_api.py
+++ b/tests/server/test_responses_api.py
@@ -867,7 +867,7 @@ def test_convert_reasoning_field_enables_thinking():
spec = RP.convert_responses_to_genspec(req, {})
assert spec.chat_template_kwargs == {"enable_thinking": True, "reasoning_effort": "high"}
- # explicit chat_template_kwargs extra field wins over the reasoning mapping
+ # an explicit thinking-related chat_template_kwargs key wins over the mapping
req2 = ResponsesRequest.model_validate(
{"model": "m", "input": "hi", "reasoning": {"effort": "low"},
"chat_template_kwargs": {"thinking_mode": "chat"}}
@@ -875,9 +875,18 @@ def test_convert_reasoning_field_enables_thinking():
spec2 = RP.convert_responses_to_genspec(req2, {})
assert spec2.chat_template_kwargs == {"thinking_mode": "chat"}
+ # unrelated extra kwargs ride along without discarding the reasoning mapping
+ req3 = ResponsesRequest.model_validate(
+ {"model": "m", "input": "hi", "reasoning": {"effort": "none"},
+ "chat_template_kwargs": {"custom_var": 1}}
+ )
+ assert RP.convert_responses_to_genspec(req3, {}).chat_template_kwargs == {
+ "enable_thinking": False, "custom_var": 1,
+ }
+
# absent reasoning -> no kwargs
- req3 = ResponsesRequest.model_validate({"model": "m", "input": "hi"})
- assert RP.convert_responses_to_genspec(req3, {}).chat_template_kwargs == {}
+ req4 = ResponsesRequest.model_validate({"model": "m", "input": "hi"})
+ assert RP.convert_responses_to_genspec(req4, {}).chat_template_kwargs == {}
def test_convert_reasoning_effort_none_disables_thinking():