FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
fix(server): honor configured output default across APIs (#411)
Earl Co953565667f31
diff --git a/python/freetoken/server/anthropic_api.py b/python/freetoken/server/anthropic_api.pyindex c941d98b..18911673 100644--- a/python/freetoken/server/anthropic_api.py+++ b/python/freetoken/server/anthropic_api.py@@ -33,6 +33,7 @@ from .anthropic_models import (AnthropicUsage,)from .generation import (+ DEFAULT_MAX_OUTPUT_TOKENS,KEEPALIVE,ContentDelta,GenDone,@@ -116,6 +117,9 @@ async def handle_anthropic_messages(spec = convert_anthropic_to_genspec(req, model_sampling,reasoning_parser=getattr(state.config, "reasoning_parser", None),+ default_max_tokens=(+ getattr(state.config, "max_output_tokens", None) or DEFAULT_MAX_OUTPUT_TOKENS+ ),)uid = await submit_generation(spec, state)except ValueError as exc:@@ -303,6 +307,7 @@ def convert_anthropic_to_genspec(req: AnthropicMessagesRequest,model_sampling: dict[str, Any],reasoning_parser: str | None = None,+ default_max_tokens: int = DEFAULT_MAX_OUTPUT_TOKENS,) -> GenSpec:messages, template_tools, parser_tools, ctk = convert_anthropic_prompt(req, reasoning_parser=reasoning_parser@@ -317,6 +322,7 @@ def convert_anthropic_to_genspec(ignore_eos=False,model_sampling=model_sampling,stop=req.stop_sequences,+ default_max_tokens=default_max_tokens,),chat_template_kwargs=ctk,template_tools=template_tools,diff --git a/python/freetoken/server/generation.py b/python/freetoken/server/generation.pyindex be05d908..28f85d87 100644--- a/python/freetoken/server/generation.py+++ b/python/freetoken/server/generation.py@@ -163,6 +163,7 @@ def resolve_sampling(ignore_eos: bool,model_sampling: dict[str, Any],stop: str | list[str] | None = None,+ default_max_tokens: int = DEFAULT_MAX_OUTPUT_TOKENS,) -> SamplingParams:"""Map a protocol's sampling fields onto the engine's neutral SamplingParams,filling unspecified fields from the checkpoint's recommended defaults."""@@ -179,7 +180,7 @@ def resolve_sampling(raise ValueError(f"max_tokens must be at least 1, got {max_tokens}")return SamplingParams(ignore_eos=ignore_eos,- max_tokens=DEFAULT_MAX_OUTPUT_TOKENS if max_tokens is None else max_tokens,+ max_tokens=default_max_tokens if max_tokens is None else max_tokens,temperature=pick(temperature, "temperature", 0.0),top_k=pick(top_k, "top_k", -1),top_p=pick(top_p, "top_p", 1.0),diff --git a/python/freetoken/server/openai_api.py b/python/freetoken/server/openai_api.pyindex b4becd26..a76898ab 100644--- a/python/freetoken/server/openai_api.py+++ b/python/freetoken/server/openai_api.py@@ -23,6 +23,7 @@ from .api_models import (from .function_call_parser import ToolCallItemfrom .request_logger import log_requestfrom .generation import (+ DEFAULT_MAX_OUTPUT_TOKENS,ContentDelta,GenDone,GenerationError,@@ -58,6 +59,7 @@ def _thinking_type(req: Any) -> str | None:def chat_request_to_genspec(req: ChatCompletionRequest,model_sampling: dict[str, Any],+ default_max_tokens: int = DEFAULT_MAX_OUTPUT_TOKENS,) -> GenSpec:"""OpenAI ChatCompletionRequest -> GenSpec (the OpenAI 'to_sampling_params')."""from .model_meta import effort_toggle_kwargs@@ -76,6 +78,7 @@ def chat_request_to_genspec(ignore_eos=req.ignore_eos,model_sampling=model_sampling,stop=req.stop,+ default_max_tokens=default_max_tokens,),chat_template_kwargs=ctk,template_tools=_tools_for_template(req),@@ -179,7 +182,10 @@ async def handle_chat_completion()try:- spec = chat_request_to_genspec(req, model_sampling)+ default_max_tokens = (+ getattr(state.config, "max_output_tokens", None) or DEFAULT_MAX_OUTPUT_TOKENS+ )+ spec = chat_request_to_genspec(req, model_sampling, default_max_tokens=default_max_tokens)except ValueError as exc:return create_error_response(str(exc))@@ -386,7 +392,10 @@ async def handle_completion(if unsupported is not None:return create_error_response(unsupported)try: # surfaces an out-of-range max_tokens as a 400 rather than a 500 from the worker- _resolve_sampling(req, model_sampling)+ default_max_tokens = (+ getattr(state.config, "max_output_tokens", None) or DEFAULT_MAX_OUTPUT_TOKENS+ )+ _resolve_sampling(req, model_sampling, default_max_tokens=default_max_tokens)except ValueError as exc:return create_error_response(str(exc), param="max_tokens")@@ -397,7 +406,9 @@ async def handle_completion(return create_error_response("Streaming completions only support a single text prompt")uid = state.new_user()await state.send_one(- TokenizeMsg(uid=uid, text=prompts[0], sampling_params=_resolve_sampling(req, model_sampling))+ TokenizeMsg(uid=uid, text=prompts[0], sampling_params=_resolve_sampling(+ req, model_sampling, default_max_tokens=default_max_tokens+ )))chunks = stream_completion_chunks(uid, req, state)if request is not None:@@ -410,7 +421,15 @@ async def handle_completion(cached_tokens = 0for index, prompt in enumerate(prompts):uid = state.new_user()- await state.send_one(TokenizeMsg(uid=uid, text=prompt, sampling_params=_resolve_sampling(req, model_sampling)))+ await state.send_one(+ TokenizeMsg(+ uid=uid,+ text=prompt,+ sampling_params=_resolve_sampling(+ req, model_sampling, default_max_tokens=default_max_tokens+ ),+ )+ )text = ""finish_reason = "stop"async for ack in state.wait_for_ack(uid):@@ -517,6 +536,7 @@ def create_error_response(def _resolve_sampling(req: ChatCompletionRequest | CompletionRequest,model_sampling: dict[str, Any],+ default_max_tokens: int = DEFAULT_MAX_OUTPUT_TOKENS,) -> SamplingParams:return resolve_sampling(temperature=req.temperature,@@ -526,6 +546,7 @@ def _resolve_sampling(ignore_eos=req.ignore_eos,model_sampling=model_sampling,stop=req.stop,+ default_max_tokens=default_max_tokens,)diff --git a/tests/server/test_openai_api.py b/tests/server/test_openai_api.pyindex facd469b..58d15d7c 100644--- a/tests/server/test_openai_api.py+++ b/tests/server/test_openai_api.py@@ -391,31 +391,41 @@ def test_completion_forwards_length_finish_reason():assert response["choices"][0]["finish_reason"] == "length"-def test_omitted_max_tokens_defaults_to_hardcoded_32k():+def test_omitted_max_tokens_honors_server_default():from freetoken.server.generation import DEFAULT_MAX_OUTPUT_TOKENSchat_state = FakeState([UserReply(uid=42, incremental_output="hi", finished=True)])+ chat_state.config.max_output_tokens = 4096run(handle_chat_completion(ChatCompletionRequest(model="m", messages=[{"role": "user", "content": "hi"}]),request=None, state=chat_state, model_sampling={},))- assert chat_state.sent.sampling_params.max_tokens == DEFAULT_MAX_OUTPUT_TOKENS+ assert chat_state.sent.sampling_params.max_tokens == 4096cmpl_state = FakeState([UserReply(uid=42, incremental_output="hi", finished=True)])+ cmpl_state.config.max_output_tokens = 4096run(handle_completion(CompletionRequest(model="m", prompt="hi"),request=None, state=cmpl_state, model_sampling={},))- assert cmpl_state.sent.sampling_params.max_tokens == DEFAULT_MAX_OUTPUT_TOKENS+ assert cmpl_state.sent.sampling_params.max_tokens == 4096# explicit value winsexp_state = FakeState([UserReply(uid=42, incremental_output="hi", finished=True)])+ exp_state.config.max_output_tokens = 4096run(handle_completion(CompletionRequest(model="m", prompt="hi", max_tokens=50),request=None, state=exp_state, model_sampling={},))assert exp_state.sent.sampling_params.max_tokens == 50+ fallback_state = FakeState([UserReply(uid=42, incremental_output="hi", finished=True)])+ run(handle_chat_completion(+ ChatCompletionRequest(model="m", messages=[{"role": "user", "content": "hi"}]),+ request=None, state=fallback_state, model_sampling={},+ ))+ assert fallback_state.sent.sampling_params.max_tokens == DEFAULT_MAX_OUTPUT_TOKENS+def test_models_route_returns_served_model_name():state = FakeState([])