Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: serve /v1/messages and count_tokens from MLX and other in-process models (#13795)
Long Yixing9a9a4aff0ccc
diff --git a/studio/backend/core/inference/chat_template_helpers.py b/studio/backend/core/inference/chat_template_helpers.pyindex 4b5611632a8..e4979e471fd 100644--- a/studio/backend/core/inference/chat_template_helpers.py+++ b/studio/backend/core/inference/chat_template_helpers.py@@ -1974,6 +1974,11 @@ def forced_tool_catalog(tool_choice, tools):]+def forced_turn_tools(tool_choice, tools):+ """The catalog a local tool-loop turn renders: only the forced tool while one is pinned."""+ return forced_tool_catalog(tool_choice, tools) or tools++def _tokenizer_objects(tokenizer) -> tuple:"""Return a processor/tokenizer and its distinct nested tokenizer."""if tokenizer is None:diff --git a/studio/backend/core/inference/orchestrator.py b/studio/backend/core/inference/orchestrator.pyindex ccf9ec705c9..3aa0641291d 100644--- a/studio/backend/core/inference/orchestrator.py+++ b/studio/backend/core/inference/orchestrator.py@@ -3272,6 +3272,8 @@ class InferenceOrchestrator:context_policy: Optional[str] = None,compaction_headroom_ratio: Optional[float] = None,compact_context: bool = False,+ tool_choice = None,+ disable_parallel_tool_use: bool = False,**_unused,):"""Run the safetensors agentic tool loop in the parent process, calling the worker for each@@ -3476,6 +3478,8 @@ class InferenceOrchestrator:# then no sink to protect anything in.caller_image_indexes = tuple(caller_image_indexes) if loop_images else (),context_fitter = _fit_iteration,+ tool_choice = tool_choice,+ disable_parallel_tool_use = disable_parallel_tool_use,)def generate_with_adapter_control(diff --git a/studio/backend/core/inference/safetensors_agentic.py b/studio/backend/core/inference/safetensors_agentic.pyindex a68a58109b0..ecba0478c77 100644--- a/studio/backend/core/inference/safetensors_agentic.py+++ b/studio/backend/core/inference/safetensors_agentic.py@@ -78,6 +78,8 @@ from core.inference.tool_loop_controller import (from core.inference.chat_template_helpers import (append_assistant_turn,trailing_assistant_resume_kind,+ forced_tool_name,+ forced_turn_tools,)from core.inference.passthrough_healing import nudge_enabledfrom state.tool_approvals import (@@ -646,6 +648,8 @@ def run_safetensors_tool_loop(images_sink: Optional[list] = None,caller_image_indexes: "tuple[int, ...]" = (),context_fitter: Optional[Callable[[list, list, list], dict]] = None,+ tool_choice = None,+ disable_parallel_tool_use: bool = False,) -> Generator[dict, None, None]:"""Drive an agentic tool loop on top of a cumulative-text generator.@@ -754,6 +758,8 @@ def run_safetensors_tool_loop(# Autoinject ran a KB search outside the controller, so it counts as an# executed tool for the plan-without-action gate.rag_autoinjected = bool(_auto)+ # As on llama-server: a forced tool is the only one offered until a call runs or is denied.+ forced_resolved = rag_autoinjected and forced_tool_name(tool_choice) == "search_knowledge_base"unrestricted_tools = not tools# Gate telling a genuine NAME[ARGS] rehearsal from inactive-name prose; built from the@@ -842,6 +848,8 @@ def run_safetensors_tool_loop(if not active_tools and not unrestricted_tools:final_attempt_done = Trueactive_tools = []+ elif not forced_resolved:+ active_tools = forced_turn_tools(tool_choice, active_tools)tool_protocol_active = not final_attempt_done and (unrestricted_tools or bool(active_tools))tool_xml_signals = TOOL_XML_SIGNALS if tool_protocol_active else ()@@ -1494,6 +1502,9 @@ def run_safetensors_tool_loop(_MAX_TOOL_CALLS_PER_TURN,)tool_calls = deduped+ # Truncated before the assistant turn is built, so extra calls never run.+ if disable_parallel_tool_use and len(tool_calls) > 1:+ tool_calls, over_cap = tool_calls[:1], []assistant_msg: dict = {"role": "assistant", "content": content_text}assistant_appended = False@@ -1817,6 +1828,8 @@ def run_safetensors_tool_loop(# Loading MCP tools is not a tool round, or a budget of one could never call what it found.if decision.tool_name != "mcp_tool_search":_turn_executed_real_tool = True+ # Any executed call, discovery included, releases a forced choice (as on llama-server).+ forced_resolved = Trueyield completion.tool_end_event()conversation.append(completion.tool_message())# Parsed only when there is a sink for them.@@ -1874,6 +1887,7 @@ def run_safetensors_tool_loop(caller_images = trim_image_turns(conversation, images_sink, keep = caller_images)yield {"type": "status", "text": ""}+ forced_resolved = forced_resolved or tool_deniedif tool_controller.force_final_answer:final_attempt_done = Truediff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex 996875dd9c6..e502dd228ba 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -5748,13 +5748,19 @@ class _TrackedCancel:run_id = None,model = None,kind = "chat",+ track_run = True,):self.event = eventself.keys = tuple(_account_cancel_key(k) for k in keys if k)# kind reaches the swap prompt: embeddings and raw completions have no conversation, so# naming them chats would offer to stop something the user never started from a thread.- self._active = active_generations.ActiveGeneration(- event, thread_id = thread_id, run_id = run_id, model = model, kind = kind+ # track_run=False only adds cancel keys to a run another tracker already records.+ self._active = (+ active_generations.ActiveGeneration(+ event, thread_id = thread_id, run_id = run_id, model = model, kind = kind+ )+ if track_run+ else None)self._slot = None@@ -5781,11 +5787,12 @@ class _TrackedCancel:for k in self.keys:if k and _PENDING_CANCELS.pop(k, None) is not None:should_cancel = True- self._active.__enter__()- self._slot = routed_slot.get()- if self._slot is not None:- with model_slots.lock:- self._slot.generations.add(self.event)+ if self._active is not None:+ self._active.__enter__()+ self._slot = routed_slot.get()+ if self._slot is not None:+ with model_slots.lock:+ self._slot.generations.add(self.event)if should_cancel:self.event.set()return self.event@@ -5799,7 +5806,8 @@ class _TrackedCancel:bucket.discard(self.event)if not bucket:_CANCEL_REGISTRY.pop(k, None)- self._active.__exit__(*exc)+ if self._active is not None:+ self._active.__exit__(*exc)if self._slot is not None:with model_slots.lock:self._slot.generations.discard(self.event)@@ -6733,9 +6741,29 @@ async def _select_request_tools(from state.tool_policy import get_tool_policyif get_tool_policy() is not False:tools = tools + [DEEP_RESEARCH_TOOL]+ delegated = _delegated_tool_names.get()+ if delegated is not None:+ tools = [tool for tool in tools if tool["function"]["name"] in delegated]return tools+# Set by a compat route that delegates to chat with its own tool selection: chat's additions+# (an archive's search_conversation, MCP, skills) must not widen what that route selected.+_delegated_tool_names: contextvars.ContextVar[Optional[frozenset]] = contextvars.ContextVar(+ "unsloth_delegated_tool_names",+ default = None,+)+++@contextmanager+def _delegated_tool_selection(names):+ token = _delegated_tool_names.set(None if names is None else frozenset(names))+ try:+ yield+ finally:+ _delegated_tool_names.reset(token)++def _search_images_enabled() -> bool:from core.inference.search_images import search_images_enabledreturn search_images_enabled()@@ -8029,6 +8057,8 @@ def _monitor_anthropic_response(if cancel_event is not None:cancel_event.set()api_monitor.finish(monitor_id, "cancelled")+ # Cancelled at a yield, the wrapped stream is suspended too; its cleanup must not wait for GC.+ await _close_openai_admitted_stream_iterator(body_iterator, cancelled = True)raiseexcept Exception as exc:api_monitor.fail(monitor_id, _friendly_error(exc))@@ -34610,6 +34640,8 @@ async def produce_openai_chat_completions(auto_heal_tool_calls = _sf_auto_heal_tool_calls,deduplicate_tool_calls = payload.deduplicate_tool_calls is not False,nudge_tool_calls = payload.nudge_tool_calls,+ tool_choice = payload.tool_choice,+ disable_parallel_tool_use = payload.parallel_tool_calls is False,max_tool_iterations = _sf_tool_budget,tool_call_timeout = payload.tool_call_timeoutif payload.tool_call_timeout is not None@@ -41062,16 +41094,28 @@ class _InProcessChatStream:self.cancel_event = cancel_event@classmethod- async def open(cls, chat_req: ChatCompletionRequest, request, current_subject):+ async def open(+ cls,+ chat_req: ChatCompletionRequest,+ request,+ current_subject,+ cancel_event: Optional[threading.Event] = None,+ tool_names = None,+ ):+ """``tool_names``: the caller's server-tool selection, run with chat's tool-loop frames."""chat_req.stream = Truechat_req.stream_options = {"include_usage": True}- cancel_event = threading.Event()- with _delegated_request_state(- request,- skip_api_monitor = True,- generation_cancel_event = cancel_event,- # The Responses translator drops control frames, so an approval prompt would be lost.- ui_stream_events = False,+ cancel_event = cancel_event or threading.Event()+ with (+ _delegated_request_state(+ request,+ skip_api_monitor = True,+ generation_cancel_event = cancel_event,+ # Only a server-tool caller relays chat's control frames; elsewhere an approval+ # prompt would be lost, so the client's opt-in is hidden.+ ui_stream_events = tool_names is not None,+ ),+ _delegated_tool_selection(tool_names),):response = await openai_chat_completions(chat_req, request, current_subject)if getattr(response, "body_iterator", None) is None:@@ -42601,6 +42645,12 @@ async def _mlx_count_chat_tokens(system_prompt, request, image = _renders_media, controls = _date_controls)+ from core.inference.chat_template_helpers import (+ forced_tool_name,+ forced_turn_tools,+ mapped_chat_template,+ renderable_tool_catalog,+ )from state.tool_policy import get_tool_policy as _get_tool_policy_mlx_tools_on = bool(_effective_enable_tools(payload))@@ -42815,6 +42865,21 @@ async def _mlx_count_chat_tokens(else _msgfor _msg in messages]+ # The first turn renders only a forced tool, narrowed as the loop narrows the catalog it+ # authorizes: a forced tool the sanitizer drops narrows nothing. The nudge names them all.+ _choice = getattr(payload, "tool_choice", None)+ if forced_tool_name(_choice):+ # Off the loop: the first call may fetch the native template.+ _safe_tools = await asyncio.to_thread(+ lambda: renderable_tool_catalog(+ _tools_to_use,+ entry.get("tokenizer"),+ entry,+ active_model_name = active,+ template = mapped_chat_template(entry, active),+ )+ )+ _tools_to_use = forced_turn_tools(_choice, _safe_tools)# Nothing to price: the template would render its generation marker alone and the# total would describe a conversation nobody has started (#8882). The GGUF branch@@ -43196,6 +43261,125 @@ async def chat_count_tokens()+def _anthropic_chat_fields(+ payload: AnthropicMessagesRequest,+ *,+ reads_images: bool,+ server_tools = (),+) -> dict:+ """The chat request fields a /v1/messages request becomes on an in-process model.++ Tool intent is stated for chat: a client catalog or tool history already keeps the+ launcher's tools default out of it, and a request with neither sends tool_choice "none",+ so chat never opens a tool loop the Messages route did not select. ``server_tools`` is+ the Messages route's own selection, run by chat's loop with no confirmation to wait on.+ """+ messages = _named_anthropic_tool_results(+ anthropic_messages_to_openai(+ [m.model_dump() for m in payload.messages],+ payload.system,+ # Kept for the template to decide, as for a chat caller replaying reasoning_content.+ preserve_thinking = payload.preserve_thinking is not False,+ tool_result_images = reads_images,+ )+ )+ studio_tools = _anthropic_requested_studio_tools(payload.tools)+ tool_choice = anthropic_tool_choice_to_openai(payload.tool_choice) or "auto"+ tools = _anthropic_client_tools_for_turn(+ [+ tool+ for tool in anthropic_tools_to_openai(payload.tools or [])+ if tool.get("function", {}).get("name") not in studio_tools+ ],+ tool_choice,+ messages,+ )+ fields = {+ "messages": messages,+ "tools": tools or None,+ "tool_choice": tool_choice if tools else "none",+ "thread_id": payload.thread_id,+ **_anthropic_reasoning_args(payload),+ }+ if "model" in payload.model_fields_set:+ fields["model"] = payload.model+ if payload.auto_heal_tool_calls is not None:+ fields["auto_heal_tool_calls"] = payload.auto_heal_tool_calls+ if server_tools:+ from core.inference.chat_template_helpers import catalog_tool_names++ # What the llama-server loop is handed on this route. Messages refused a confirm+ # gate before the switch, so the loop is told there is none.+ fields.update(+ tools = None,+ tool_choice = _anthropic_server_tool_choice(tool_choice),+ enable_tools = True,+ enabled_tools = sorted(catalog_tool_names(server_tools)),+ mcp_enabled = False,+ confirm_tool_calls = False,+ permission_mode = payload.permission_mode,+ sandbox_level = payload.sandbox_level,+ bypass_permissions = bool(payload.bypass_permissions),+ rag_scope = getattr(payload, "rag_scope", None),+ max_tool_calls_per_message = 25,+ tool_call_timeout = 300,+ auto_heal_tool_calls = True,+ )+ return fields+++def _anthropic_server_tool_choice(openai_tool_choice):+ """``tool_choice`` for the server-tool loop: a forced Anthropic alias names its Unsloth tool."""+ if isinstance(openai_tool_choice, dict):+ forced_function = openai_tool_choice.get("function")+ forced_name = forced_function.get("name") if isinstance(forced_function, dict) else None+ canonical_name = _STUDIO_ANTHROPIC_TOOL_ALIASES.get(forced_name)+ if canonical_name:+ return {"type": "function", "function": {"name": canonical_name}}+ return openai_tool_choice+++def _anthropic_server_tool_refusal(payload, server_tool_choice, tools) -> Optional[str]:+ """Why the server-tool loop cannot take this request, found once the target is resident."""+ from core.inference.chat_template_helpers import catalog_tool_names, forced_tool_name++ # Bypass Permissions suppresses confirm, so both flags together is fine.+ if bool(getattr(payload, "confirm_tool_calls", False)) and not bool(+ getattr(payload, "bypass_permissions", False)+ ):+ return "confirm_tool_calls is not supported for Anthropic Messages server tools."+ forced_name = forced_tool_name(server_tool_choice)+ if forced_name and forced_name not in catalog_tool_names(tools):+ return f"Invalid 'tool_choice': function '{forced_name}' is not enabled."+ return None+++async def _anthropic_in_process_count(+ payload, request, current_subject, *, server_tools: list+) -> Optional[JSONResponse]:+ """Count on a resident MLX model through the chat count route, or None if none is serving."""+ backend = await asyncio.to_thread(get_inference_backend)+ active = getattr(backend, "active_model_name", None)+ entry = (getattr(backend, "models", {}) or {}).get(active) or {}+ if not active or not entry.get("is_mlx"):+ return None+ fields = _anthropic_chat_fields(+ payload, reads_images = bool(entry.get("is_vision")), server_tools = server_tools+ )+ return JSONResponse(+ content = {"input_tokens": await _anthropic_chat_count(fields, request, current_subject)}+ )+++async def _anthropic_chat_count(fields: dict, request, current_subject) -> int:+ """Chat's count of the translated request, priced with the tool selection it would run."""+ with _delegated_tool_selection(fields["enabled_tools"] if fields.get("enable_tools") else None):+ counted = await chat_count_tokens(+ ChatCountTokensRequest(**fields), current_subject, request+ )+ return int(json.loads(counted.body)["input_tokens"])++def _anthropic_connection_error_event(error: Any) -> str:message = error.get("message") if isinstance(error, dict) else Nonetry:@@ -43567,8 +43751,8 @@ async def anthropic_count_tokens(Translates the Anthropic request to OpenAI form (the same translation the/messages handler uses), counts prompt tokens with the loaded GGUF model's- tokenizer, and returns ``{"input_tokens": int}`` only. Unlike /messages,- max_tokens is NOT required here.+ tokenizer, or a resident MLX model's through the chat count route, and returns+ ``{"input_tokens": int}`` only. Unlike /messages, max_tokens is NOT required here."""if _connection_model_ref(payload.model) is not None:raise HTTPException(@@ -43580,6 +43764,40 @@ async def anthropic_count_tokens(_validate_anthropic_client_tools(payload.tools)# Counting compiles no output grammar, matching Anthropic's count_tokens behavior._count_response_format = _anthropic_response_format(payload, require_llama_compatibility = False)+ _count_studio_tools = _anthropic_requested_studio_tools(payload.tools)+ _count_has_client_tool = any(+ (t if isinstance(t, dict) else t.model_dump()).get("input_schema") is not None+ or anthropic_schema_client_tool_kind(t) is not None+ for t in payload.tools or []+ )+ _count_openai_tool_choice = anthropic_tool_choice_to_openai(payload.tool_choice) or "auto"+ _count_schema_disables_server_tools = (+ _count_openai_tool_choice == "none" and _count_response_format is not None+ )+ _count_selects_server_tools = (+ _anthropic_selects_server_tools(payload, _count_studio_tools, _count_has_client_tool)+ and not _count_schema_disables_server_tools+ )+ from core.inference.tools import ALL_TOOLS as _ANTHROPIC_COUNT_TOOLS+ from utils.hardware import DeviceType, get_device++ _count_in_process_server_tools = (+ _select_anthropic_server_tools(+ _ANTHROPIC_COUNT_TOOLS, _count_studio_tools, payload.enabled_tools+ )+ if _count_selects_server_tools+ else []+ )+ # Only an MLX model counts in process, and only what the chat count route prices, so any+ # other non-GGUF target is refused before it loads.+ _in_process_refusal = None+ # A replayed MCP envelope is an image to a vision target, which the chat count refuses.+ _count_replay = _anthropic_chat_fields(payload, reads_images = True)["messages"]+ if _anthropic_request_has_image(payload) or (+ _messages_mention_mcp_images(_count_replay)+ and await asyncio.to_thread(_messages_have_promotable_mcp_images, _count_replay)+ ):+ _in_process_refusal = "Cannot count tokens for messages containing images."# Count with the requested model's tokenizer, like the sibling /messages.# Carry the vision guard too: an image count naming a text-only GGUF must not# evict a loaded vision model for a swap that can't serve the request.@@ -43591,11 +43809,22 @@ async def anthropic_count_tokens(# count_tokens only tokenizes (no generation), so it must not adopt the resident# model; the middleware likewise excludes count_tokens from its claim.claim_resident = False,- gguf_only = True,+ gguf_only = (+ _in_process_refusal is not None or await asyncio.to_thread(get_device) != DeviceType.MLX+ ),+ gguf_only_reason = _in_process_refusal,)llama_backend = get_llama_cpp_backend()if not llama_backend.is_loaded:+ _in_process_count = await _anthropic_in_process_count(+ payload,+ request,+ current_subject,+ server_tools = _count_in_process_server_tools,+ )+ if _in_process_count is not None:+ return _in_process_count_status, _detail = await _no_model_loaded_error("No GGUF model loaded. Load a GGUF model first.",_switch_model_for_payload(payload),@@ -43651,25 +43880,13 @@ async def anthropic_count_tokens(# Only the client-tool passthrough is forwarded verbatim, so reproduce /messages' own# routing rather than "any tools": a Studio server-tool alias, or a template without# passthrough support, falls through to plain generation there and does carry the date.- _count_studio_tools = _anthropic_requested_studio_tools(payload.tools)- _count_has_client_tool = any(- (t if isinstance(t, dict) else t.model_dump()).get("input_schema") is not None- or anthropic_schema_client_tool_kind(t) is not None- for t in payload.tools or []- )- _count_openai_tool_choice = anthropic_tool_choice_to_openai(payload.tool_choice) or "auto"- _count_schema_disables_server_tools = (- _count_openai_tool_choice == "none" and _count_response_format is not None- )_count_server_tools = (- _anthropic_selects_server_tools(payload, _count_studio_tools, _count_has_client_tool)+ _count_selects_server_toolsand llama_backend.supports_toolsand not _anthropic_request_has_image(payload, tool_results = llama_backend.is_vision)- and not _count_schema_disables_server_tools)_count_selected_server_tools = []if _count_server_tools:- from core.inference.tools import ALL_TOOLS as _ANTHROPIC_COUNT_TOOLS_count_selected_server_tools = _tools_for_search_images(_select_anthropic_server_tools(_ANTHROPIC_COUNT_TOOLS,@@ -43799,6 +44016,398 @@ def _set_or_prepend_system_message(return [system, *others]+def _anthropic_chat_sampling(payload) -> dict:+ """Client-set sampling fields; omitted ones stay omitted so chat applies the model's defaults."""+ return {+ name: getattr(payload, name)+ for name in (+ "temperature",+ "top_p",+ "top_k",+ "min_p",+ "repetition_penalty",+ "presence_penalty",+ "seed",+ "nudge_tool_calls",+ )+ if getattr(payload, name) is not None+ }+++def _anthropic_chat_sampling_refusal(payload, messages) -> Optional[str]:+ """Why chat would reject this request's sampling fields; Messages bounds some less tightly."""+ from pydantic import ValidationError++ try:+ ChatCompletionRequest(+ messages = messages,+ max_tokens = payload.max_tokens,+ **_anthropic_chat_sampling(payload),+ )+ except ValidationError as exc:+ error = exc.errors()[0]+ field = ".".join(str(part) for part in error.get("loc", ())) or "request"+ return f"'{field}': {error.get('msg')} on a non-GGUF model."+ return None+++def _in_process_model_resident() -> bool:+ orchestrator = _peek_inference_backend()+ entry = (getattr(orchestrator, "models", None) or {}).get(+ getattr(orchestrator, "active_model_name", None)+ )+ # Chat answers a TTS model with a finished clip, not a text reply.+ return isinstance(entry, dict) and not (+ entry.get("is_audio") and entry.get("audio_type") != "whisper"+ )+++async def _anthropic_in_process_input_tokens(fields: dict, request, current_subject) -> int:+ """``message_start`` input tokens: the count route's total, or 0 where it cannot price one."""+ try:+ return await _anthropic_chat_count(fields, request, current_subject)+ except HTTPException:+ return 0+++class _ChatToolLoopEvents:+ """Chat's tool-loop stream as the events the llama-server tool loop yields.++ Text is cumulative until each tool_end, with reasoning folded in as one leading ``<think>``+ span recorded in ``provenance`` the way that loop records it. Reasoning from a re-prompted+ turn that follows visible text has no such place, so it is dropped (thinking is optional).+ """++ def __init__(self, provenance: dict) -> None:+ self._prov = provenance+ self._text = ""+ self._wrap = None+ self._finish_reason = None+ self._usage = {}++ def _content(self) -> dict:+ return {"type": "content", "text": self._text, _WRAPPED_SO_FAR: self._prov["wrapped"]}++ def feed(self, chunk: dict) -> list:+ etype = chunk.get("type")+ if etype in ("tool_start", "tool_end", "tool_output", "tool_args"):+ events = []+ if etype == "tool_start" and self._wrap is not None:+ self._text, self._wrap = self._text + "</think>", None+ events.append(self._content())+ if etype == "tool_end":+ self._text, self._wrap = "", None+ return events + [chunk]+ if etype is not None or not chunk.get("choices"):+ self._usage = chunk.get("usage") or self._usage+ return []+ events = []+ for choice in chunk["choices"]:+ delta = choice.get("delta") or {}+ reasoning = delta.get("reasoning_content")+ if reasoning and (self._wrap is not None or not self._text):+ if self._wrap is None:+ self._prov["wrapped"] += 1+ self._wrap = {"len": 0}+ self._prov.setdefault("wraps", []).append(self._wrap)+ self._text += "<think>"+ self._text += reasoning+ self._wrap["len"] += len(reasoning)+ events.append(self._content())+ if delta.get("content"):+ if self._wrap is not None:+ self._text, self._wrap = self._text + "</think>", None+ self._text += delta["content"]+ events.append(self._content())+ self._finish_reason = choice.get("finish_reason") or self._finish_reason+ return events++ def finish(self) -> list:+ return [{"type": "metadata", "finish_reason": self._finish_reason, "usage": self._usage}]+++def _anthropic_in_process_sse(+ source: "_InProcessChatStream",+ *,+ request,+ monitor_id,+ message_id: str,+ model_name: str,+ input_tokens: int,+ disable_parallel_tool_use: bool,+ server_tools = (),+):+ """Chat's stream as Anthropic SSE, ending the way the matching llama-server stream ends."""+ from core.inference.llama_keepwarm import mark_response_failed++ if server_tools:+ provenance: dict = {"wrapped": 0}+ emitter = AnthropicStreamEmitter(parse_think = True, think_provenance = provenance)+ rules = _AnthropicToolEventRules(+ emitter, server_tools, disable_parallel_tool_use, provenance+ )+ tool_events = _ChatToolLoopEvents(provenance)++ def feed(chunk):+ # A stopped server-tool stream ends without message_stop, as on llama-server.+ if source.cancelled:+ return [], True+ # Raised, so the error event is the one a failed llama-server tool loop gets.+ error_message = _monitor_openai_error_message(chunk)+ if error_message:+ raise RuntimeError(error_message)+ return [line for event in tool_events.feed(chunk) for line in rules.feed(event)], False++ def finish():+ if source.cancelled:+ return []+ return [line for event in tool_events.finish() for line in rules.feed(event)] + (+ rules.finish()+ )+ else:+ emitter = AnthropicPassthroughEmitter(reasoning_as_thinking = True)++ def feed(chunk):+ return _anthropic_passthrough_chunk_lines(+ emitter, chunk, request, disable_parallel_tool_use+ )++ finish = emitter.finish+ # The chat handler's tracker holds cancel_id and session_id; the message id is Messages' own.+ tracker = _TrackedCancel(source.cancel_event, message_id, track_run = False)+ tracker.__enter__()++ async def _stream():+ cancelled = False+ chunks = source.chunks()+ try:+ for line in emitter.start(message_id, model_name, input_tokens):+ yield line+ while (chunk := await source.next_chunk(chunks)) is not None:+ if chunk is _LLAMA_STREAM_KEEPALIVE:+ yield _OPENAI_PASSTHROUGH_SSE_KEEPALIVE+ continue+ lines, failed = feed(chunk)+ for line in lines:+ yield line+ if failed:+ return+ except asyncio.CancelledError:+ cancelled = True+ source.cancel_event.set()+ raise+ except Exception as e:+ if not source.cancelled:+ logger.error("anthropic_messages in-process stream error: %s", e)+ mark_response_failed(getattr(request, "scope", None))+ event = _anthropic_stream_error_event(e, force = True)+ if event is not None:+ yield event+ return+ finally:+ if source.cancelled:+ mark_response_failed(getattr(request, "scope", None))+ try:+ await chunks.aclose()+ await source.close(cancelled = cancelled)+ finally:+ tracker.__exit__(None, None, None)++ for line in finish():+ yield line++ async def _unstarted_cleanup() -> None:+ api_monitor.finish(monitor_id, "cancelled")+ try:+ await source.close(cancelled = True)+ finally:+ tracker.__exit__(None, None, None)++ return _sse_streaming_response(_stream(), unstarted_cleanup = _unstarted_cleanup)+++async def _anthropic_in_process_tool_message(+ chat_req,+ request,+ current_subject,+ *,+ cancel_event,+ message_id,+ model_name,+ server_tools,+ disable_parallel_tool_use,+):+ """A non-streaming server-tool reply: chat's tool-loop stream reduced as llama-server's is.++ Chat's own JSON reply drops the tool calls it ran, so the stream is driven here instead.+ """+ provenance: dict = {"wrapped": 0}+ tool_events = _ChatToolLoopEvents(provenance)+ events: list = []+ source = await _InProcessChatStream.open(+ chat_req,+ request,+ current_subject,+ cancel_event = cancel_event,+ tool_names = chat_req.enabled_tools,+ )+ cancelled = False+ chunks = source.chunks()+ try:+ while (chunk := await source.next_chunk(chunks)) is not None:+ if chunk is _LLAMA_STREAM_KEEPALIVE:+ continue+ error_message = _monitor_openai_error_message(chunk)+ if error_message:+ raise _anthropic_map_generation_error(RuntimeError(error_message))+ events.extend(tool_events.feed(chunk))+ except asyncio.CancelledError:+ cancelled = True+ raise+ finally:+ await chunks.aclose()+ await source.close(cancelled = cancelled)+ response = _anthropic_tool_response_from_events(+ events + tool_events.finish(),+ message_id,+ model_name,+ disable_parallel_tool_use = disable_parallel_tool_use,+ openai_tools = server_tools,+ parse_think = True,+ think_provenance = provenance,+ )+ # A stopped run answers with what it had, as the llama-server tool path does.+ _mark_cancelled_json_response_failed(request, cancel_event)+ return response+++async def _anthropic_in_process_messages(+ payload: AnthropicMessagesRequest,+ request,+ current_subject,+ server_tools = (),+):+ """Serve /v1/messages from an in-process model (MLX, transformers) through chat."""+ backend = await asyncio.to_thread(get_inference_backend)+ entry = (getattr(backend, "models", {}) or {}).get(backend.active_model_name) or {}+ image_location = not entry.get("is_vision") and _anthropic_top_level_image_location(payload)+ if image_location:+ # Named as on llama-server, so a client can drop the image and keep the session going.+ raise HTTPException(+ status_code = 400,+ detail = f"{image_location}: Image provided but current model does not support vision.",+ )+ fields = _anthropic_chat_fields(+ payload, reads_images = bool(entry.get("is_vision")), server_tools = server_tools+ )+ refusal = server_tools and _anthropic_server_tool_refusal(+ payload, fields["tool_choice"], server_tools+ )+ if refusal:+ raise HTTPException(+ status_code = 400,+ detail = anthropic_error_body(refusal, status = 400, err_type = "invalid_request_error"),+ )+ # Validated against llama.cpp's compiler before the switch; dropped when a tool may run, as there.+ response_format = _anthropic_response_format(payload, require_llama_compatibility = False)+ if (server_tools or fields["tools"]) and fields["tool_choice"] != "none":+ response_format = None+ model_name = _orchestrator_public_model_id(backend) or payload.model+ message_id = f"msg_{uuid.uuid4().hex[:24]}"+ disable_parallel = bool(+ isinstance(payload.tool_choice, dict)+ and payload.tool_choice.get("disable_parallel_tool_use")+ )+ chat_req = ChatCompletionRequest(+ **fields,+ **_anthropic_chat_sampling(payload),+ max_tokens = payload.max_tokens,+ stop = payload.stop_sequences or None,+ parallel_tool_calls = False if disable_parallel else None,+ cancel_id = payload.cancel_id,+ session_id = payload.session_id,+ response_format = response_format,+ )++ monitor_id = None+ context_length = _monitor_context_length()+ if not getattr(getattr(request, "state", None), "skip_api_monitor", False):+ monitor_id = api_monitor.start(+ endpoint = getattr(getattr(request, "url", None), "path", "/v1/messages"),+ method = getattr(request, "method", "POST"),+ via_api_key = _request_used_api_key(request),+ model = model_name,+ prompt = _monitor_prompt_from_messages(fields["messages"]),+ context_length = context_length,+ subject = current_subject,+ )+ cancel_event = threading.Event()+ try:+ if payload.stream:+ input_tokens = await _anthropic_in_process_input_tokens(+ fields, request, current_subject+ )+ source = await _InProcessChatStream.open(+ chat_req,+ request,+ current_subject,+ cancel_event = cancel_event,+ tool_names = chat_req.enabled_tools if server_tools else None,+ )+ response = _anthropic_in_process_sse(+ source,+ request = request,+ monitor_id = monitor_id,+ message_id = message_id,+ model_name = model_name,+ input_tokens = input_tokens,+ disable_parallel_tool_use = disable_parallel,+ server_tools = server_tools,+ )+ elif server_tools:+ response = await _anthropic_in_process_tool_message(+ chat_req,+ request,+ current_subject,+ cancel_event = cancel_event,+ message_id = message_id,+ model_name = model_name,+ server_tools = server_tools,+ disable_parallel_tool_use = disable_parallel,+ )+ else:+ perf: dict = {"monitor_id": monitor_id, "context_length": context_length}+ sink_token = _monitor_perf_sink.set(perf if monitor_id else None)+ try:+ with _delegated_request_state(+ request, skip_api_monitor = True, generation_cancel_event = cancel_event+ ):+ result = await openai_chat_completions(chat_req, request, current_subject)+ finally:+ _monitor_perf_sink.reset(sink_token)+ # A stopped run returns what it had; the llama-server path never answers one.+ if cancel_event.is_set():+ raise asyncio.CancelledError()+ response = _anthropic_message_from_openai_completion(+ json.loads(result.body),+ message_id,+ model_name,+ openai_tools = chat_req.tools,+ allowed_tools = None,+ disable_parallel_tool_use = disable_parallel,+ response_format = response_format,+ parse_think = True,+ )+ except asyncio.CancelledError:+ cancel_event.set()+ api_monitor.finish(monitor_id, "cancelled")+ raise+ except Exception as exc:+ api_monitor.fail(monitor_id, _friendly_error(exc))+ raise+ return _monitor_anthropic_response(response, monitor_id, context_length, cancel_event)++@router.post("/messages")@account_access.gpu_busy_routeasync def anthropic_messages(@@ -43828,7 +44437,11 @@ async def anthropic_messages(# an automatic load can run (auto-switch or a standalone idle TTL), fall through# so validation runs before the reload hook gets a chance to restore the model.# Plain detail, not _no_model_loaded_error: that helper leaves this case unchanged.- if not llama_backend.is_loaded and not _automatic_model_load_may_run():+ if (+ not llama_backend.is_loaded+ and not _in_process_model_resident()+ and not _automatic_model_load_may_run()+ ):raise HTTPException(status_code = 503,detail = _no_model_loaded_detail("No GGUF model loaded. Load a GGUF model first."),@@ -43948,6 +44561,23 @@ async def anthropic_messages(),)+ # What an in-process model cannot serve here is refused before a non-GGUF target loads.+ _chat_req_pre = ChatCompletionRequest(+ messages = _anthropic_chat_fields(payload, reads_images = True)["messages"]+ )+ _chat_parts = _extract_content_parts(_chat_req_pre.messages, keep_tool_images = True)+ _in_process_refusal = None+ if not _chat_parts[1]:+ _in_process_refusal = "At least one non-system message is required."+ else:+ _in_process_refusal = _anthropic_chat_sampling_refusal(payload, _chat_req_pre.messages)+ # Chat's image checks for a non-GGUF target; b64s stays what a GGUF target validates.+ _anthropic_image_preflight = None+ if _anthropic_has_image:+ _anthropic_image_preflight = {+ **_chat_image_preflight(_chat_req_pre, _chat_parts[2]),+ "b64s": _anthropic_image_b64s,+ }# require_vision rejects a swap to a text-only target before it runs, so an# image request can't evict the resident vision model only to hit the vision# guard (_normalize_anthropic_openai_images) below after the load.@@ -43961,18 +44591,26 @@ async def anthropic_messages(# the middleware claims on a 2xx, so a rejected request never strands a preview-owned# model.claim_resident = False,- gguf_only = True,- image_preflight = (- {- "b64": _anthropic_image_b64s[0],- "b64s": _anthropic_image_b64s,- "multiple": False,- }- if _anthropic_image_b64s- else None- ),+ gguf_only = _in_process_refusal is not None,+ gguf_only_reason = _in_process_refusal,+ image_preflight = _anthropic_image_preflight,)llama_backend = get_llama_cpp_backend()+ if not llama_backend.is_loaded and _in_process_model_resident():+ if _in_process_refusal is not None:+ raise HTTPException(+ status_code = 400,+ detail = anthropic_error_body(+ _in_process_refusal, status = 400, err_type = "invalid_request_error"+ ),+ )+ # An image anywhere keeps the llama-server route off its server-tool loop too.+ return await _anthropic_in_process_messages(+ payload,+ request,+ current_subject,+ server_tools = [] if _anthropic_has_image else selected_server_tools,+ )if not llama_backend.is_loaded:_status, _detail = await _no_model_loaded_error("No GGUF model loaded. Load a GGUF model first.",@@ -44501,22 +45139,6 @@ async def anthropic_messages(# As the tool loop does, so the opening usage count prices what it sends.openai_messages = await asyncio.to_thread(restore_replayed_tool_results, openai_messages)- # Bypass Permissions suppresses confirm, so both flags together is fine.- if bool(getattr(payload, "confirm_tool_calls", False)) and not bool(- getattr(payload, "bypass_permissions", False)- ):- api_monitor.fail(- monitor_id,- "confirm_tool_calls is not supported for Anthropic Messages server tools.",- )- raise HTTPException(- status_code = 400,- detail = anthropic_error_body(- "confirm_tool_calls is not supported for Anthropic Messages server tools.",- status = 400,- err_type = "invalid_request_error",- ),- )from core.inference.tools import apply_full_access_tool_descriptions# ask/auto (and an omitted mode selecting a gate-needing terminal/python@@ -44537,21 +45159,9 @@ async def anthropic_messages(getattr(payload, "sandbox_level", None),)- server_tool_choice = openai_tool_choice- if isinstance(server_tool_choice, dict):- forced_function = server_tool_choice.get("function")- forced_name = forced_function.get("name") if isinstance(forced_function, dict) else None- canonical_name = _STUDIO_ANTHROPIC_TOOL_ALIASES.get(forced_name)- if canonical_name:- server_tool_choice = {- "type": "function",- "function": {"name": canonical_name},- }- from core.inference.chat_template_helpers import catalog_tool_names, forced_tool_name-- forced_name = forced_tool_name(server_tool_choice)- if forced_name and forced_name not in catalog_tool_names(openai_tools):- message = f"Invalid 'tool_choice': function '{forced_name}' is not enabled."+ server_tool_choice = _anthropic_server_tool_choice(openai_tool_choice)+ message = _anthropic_server_tool_refusal(payload, server_tool_choice, openai_tools)+ if message:api_monitor.fail(monitor_id, message)raise HTTPException(status_code = 400,@@ -44718,6 +45328,81 @@ async def anthropic_messages()+class _AnthropicToolEventRules:+ """What the server-tool stream does with each tool-loop event, whatever produced it."""++ def __init__(self, emitter, openai_tools, disable_parallel_tool_use, think_provenance) -> None:+ self._emitter = emitter+ # Gate the display strip on the declared tools: an inactive NAME[ARGS]{...} in a final+ # answer is prose and must survive in the delivered text.+ self._display_names = _display_tool_name_gate(openai_tools)+ self._disable_parallel_tool_use = disable_parallel_tool_use+ self._finish_reason = None+ # Response ends on a pending tool_use block rather than final text; a server tool+ # that keeps generating flips this back to False.+ self._ends_on_tool_use = False+ self._tool_blocks_emitted = 0+ self._drop_until_tool_end = False+ self._span_guard = _ReasoningSpanGuard(think_provenance)+ # Last drop-branch keepalive, seeded to stream start so a chatty tool busy past the+ # stall window still gets one though its events are dropped.+ self._last_drop_keepalive = time.monotonic()++ def feed(self, event: dict) -> list:+ etype = event.get("type")+ if etype == "heartbeat":+ # Tool-wrapper heartbeat -> SSE keepalive, checked BEFORE the drop skip:+ # a dropped tool still runs and suppresses the stall keepalive.+ return [_OPENAI_PASSTHROUGH_SSE_KEEPALIVE]+ if etype in ("tool_output", "tool_args"):+ # No Messages equivalent, but they suppress the stall keepalive: send a rate-limited+ # one instead of going silent past the ~100s proxy cap.+ _now = time.monotonic()+ if _now - self._last_drop_keepalive >= _LOCAL_TOOL_STREAM_STALL_KEEPALIVE_S:+ self._last_drop_keepalive = _now+ return [_OPENAI_PASSTHROUGH_SSE_KEEPALIVE]+ return []+ if self._drop_until_tool_end:+ # Skip through the dropped call's tool_end.+ if etype == "tool_end":+ self._drop_until_tool_end = False+ return []+ if etype == "metadata":+ _fr = event.get("finish_reason")+ if _fr is not None:+ self._finish_reason = _fr+ # Strip leaked tool XML first; the protected helper keeps <think> rehearsal and balanced+ # [TOOL_CALLS] prose, which a raw sub corrupts.+ if etype == "content":+ event = dict(event)+ event["text"] = self._span_guard.strip(+ event["text"],+ auto_heal_tool_calls = True,+ enabled_tool_names = self._display_names,+ )+ # Parallel cap by state, not id: ids may be empty.+ if etype == "tool_start":+ if self._disable_parallel_tool_use and self._tool_blocks_emitted >= 1:+ self._drop_until_tool_end = True+ return []+ self._ends_on_tool_use = True+ elif etype == "tool_end":+ self._tool_blocks_emitted += 1+ self._span_guard.tool_end()+ # Unsloth ran the tool server-side, so the response no longer ends on a pending+ # client action; otherwise stop_reason "tool_use" tells the client to run it again.+ self._ends_on_tool_use = False+ elif etype == "content" and event.get("text"):+ self._ends_on_tool_use = False+ return self._emitter.feed(event)++ def finish(self) -> list:+ stop_reason = openai_finish_to_anthropic_stop(+ self._finish_reason, had_tool_calls = self._ends_on_tool_use+ )+ return self._emitter.finish(stop_reason = stop_reason, stop_sequence = None)++async def _anthropic_tool_stream(request,cancel_event,@@ -44735,10 +45420,6 @@ async def _anthropic_tool_stream("""Streaming response for the tool-calling path."""_sentinel = object()- # Gate the display strip on the declared tools: an inactive NAME[ARGS]{...} in a final- # answer is prose and must survive in the delivered text.- _display_names = _display_tool_name_gate(openai_tools)-# Prompt-token count for message_start.usage.input_tokens. count_chat_tokens# makes blocking HTTP calls to llama-server, so run it off the event loop.# Pass the tools so tool-schema tokens are counted (the generator renders@@ -44769,17 +45450,9 @@ async def _anthropic_tool_stream()for line in emitter.start(message_id, model_name, input_tokens = input_tokens):yield line-- captured_finish_reason = None- # Response ends on a pending tool_use block rather than final text; a server tool- # that keeps generating flips this back to False.- ends_on_tool_use = False- tool_blocks_emitted = 0- drop_until_tool_end = False- _span_guard = _ReasoningSpanGuard(think_provenance)- # Last drop-branch keepalive, seeded to stream start so a chatty tool busy past the- # stall window still gets one though its events are dropped.- _last_drop_keepalive = time.monotonic()+ rules = _AnthropicToolEventRules(+ emitter, openai_tools, disable_parallel_tool_use, think_provenance+ )# See the GGUF tool stream: this loop drives the same tool generator, and its# respawn refit records the refusal from inside the worker thread, so the slot@@ -44816,57 +45489,7 @@ async def _anthropic_tool_stream(_next_task = Noneif event is _sentinel:break- etype = event.get("type")- if etype == "heartbeat":- # Tool-wrapper heartbeat -> SSE keepalive, checked BEFORE the drop skip:- # a dropped tool still runs and suppresses the stall keepalive.- yield _OPENAI_PASSTHROUGH_SSE_KEEPALIVE- continue- if etype in ("tool_output", "tool_args"):- # No Anthropic Messages equivalent (the full call/result follow in tool_use /- # tool_result), so drop them. They suppress the stall keepalive, so emit a- # rate-limited one instead of going silent past the ~100s proxy cap.- _now = time.monotonic()- if _now - _last_drop_keepalive >= _LOCAL_TOOL_STREAM_STALL_KEEPALIVE_S:- _last_drop_keepalive = _now- yield _OPENAI_PASSTHROUGH_SSE_KEEPALIVE- continue- if drop_until_tool_end:- # disable_parallel_tool_use: skip every event until (and- # including) this dropped tool call's tool_end.- if etype == "tool_end":- drop_until_tool_end = False- continue- if etype == "metadata":- _fr = event.get("finish_reason")- if _fr is not None:- captured_finish_reason = _fr- # Strip leaked tool-call XML first, so a purely-tool-XML content event doesn't- # count as text. The protected helper keeps <think> rehearsal and balanced- # [TOOL_CALLS] trailing prose, which a raw sub corrupts.- if etype == "content":- event = dict(event)- event["text"] = _span_guard.strip(- event["text"],- auto_heal_tool_calls = True,- enabled_tool_names = _display_names,- )- # disable_parallel_tool_use: keep only the first tool_use block, dropping- # later tool_start/tool_end pairs (by state, not id: ids may be empty).- if etype == "tool_start":- if disable_parallel_tool_use and tool_blocks_emitted >= 1:- drop_until_tool_end = True- continue- ends_on_tool_use = True- elif etype == "tool_end":- tool_blocks_emitted += 1- _span_guard.tool_end()- # Unsloth ran the tool server-side, so the response no longer ends on a pending- # client action; otherwise stop_reason "tool_use" tells the client to run it again.- ends_on_tool_use = False- elif etype == "content" and event.get("text"):- ends_on_tool_use = False- for line in emitter.feed(event):+ for line in rules.feed(event):yield lineexcept Exception as e:logger.error("anthropic_messages stream error: %s", e)@@ -44887,10 +45510,7 @@ async def _anthropic_tool_stream(except (RuntimeError, ValueError):pass- stop_reason = openai_finish_to_anthropic_stop(- captured_finish_reason, had_tool_calls = ends_on_tool_use- )- for line in emitter.finish(stop_reason = stop_reason, stop_sequence = None):+ for line in rules.finish():yield linefinally:_tracker.__exit__(None, None, None)@@ -45729,6 +46349,29 @@ async def _passthrough_retry_url(llama_backend, exc):return f"{llama_backend.base_url}/v1/chat/completions"+def _anthropic_passthrough_chunk_lines(+ emitter, chunk, request, disable_parallel_tool_use: bool+) -> tuple[list[str], bool]:+ """One OpenAI chat chunk -> Anthropic SSE lines; True when an in-band error ended the stream."""+ if isinstance(chunk, dict):+ error_message = _monitor_openai_error_message(chunk)+ if error_message:+ # A 200 SSE error payload has no choices, so feed_chunk would end cleanly and the+ # middleware would claim a preview-owned model: flag failed and send an error event.+ from core.inference.llama_keepwarm import mark_response_failed+ mark_response_failed(getattr(request, "scope", None))+ # The formatter recovers n_prompt_tokens/n_ctx from the payload when the message lacks them.+ return [+ _anthropic_upstream_stream_error_event(+ error_message,+ counts_source = _json_dumps_safe(chunk.get("error", chunk)),+ )+ ], True+ if disable_parallel_tool_use:+ _drop_parallel_tool_call_deltas(chunk)+ return emitter.feed_chunk(chunk), False++async def _anthropic_passthrough_stream(request,cancel_event,@@ -45996,29 +46639,13 @@ async def _anthropic_passthrough_stream(chunk = json.loads(data_str)except json.JSONDecodeError:continue- if isinstance(chunk, dict):- error_message = _monitor_openai_error_message(chunk)- if error_message:- # Upstream HTTP-200 SSE error payload (data: {"error": ...}):- # emitter.feed_chunk ignores chunks without choices, so the stream- # would finish cleanly and the middleware would claim a preview-owned- # model. Flag failed and surface an Anthropic error event instead of a- # silent message_stop (mirrors the OpenAI passthrough / Responses).- from core.inference.llama_keepwarm import mark_response_failed-- mark_response_failed(getattr(request, "scope", None))- # The message alone can be count-less while the chunk still- # carries n_prompt_tokens/n_ctx; keep the clean message and let- # the formatter recover the totals from the payload.- yield _anthropic_upstream_stream_error_event(- error_message,- counts_source = _json_dumps_safe(chunk.get("error", chunk)),- )- return- if disable_parallel_tool_use:- _drop_parallel_tool_call_deltas(chunk)- for line in emitter.feed_chunk(chunk):+ lines, failed = _anthropic_passthrough_chunk_lines(+ emitter, chunk, request, disable_parallel_tool_use+ )+ for line in lines:yield line+ if failed:+ returnexcept Exception as e:if not cancel_event.is_set():logger.error("anthropic_messages passthrough stream error: %s", e)@@ -46079,6 +46706,104 @@ def _sum_passthrough_attempt_stats(first: dict, retry: dict, selected: dict) ->return result+def _anthropic_message_from_openai_completion(+ data: dict,+ message_id,+ model_name,+ *,+ openai_tools,+ allowed_tools,+ disable_parallel_tool_use: bool,+ response_format,+ parse_think: bool,+):+ """A non-streaming OpenAI chat completion -> the Anthropic Messages JSON response."""+ choice = (data.get("choices") or [{}])[0]+ message = choice.get("message") or {}+ finish_reason = choice.get("finish_reason")++ healing_active = bool(allowed_tools)+ healed_events = (+ heal_openai_message_events(message, allowed_tools, openai_tools) if healing_active else None+ )++ content_blocks = []+ tool_calls = []+ # Anthropic orders thinking ahead of the answer; reading only `content` drops the trace.+ reasoning = message.get("reasoning_content") or ""+ if reasoning.strip():+ if parse_think:+ content_blocks.append(AnthropicResponseThinkingBlock(thinking = reasoning))+ else:+ # Thinking effectively off: the parser shunted a literal+ # example into reasoning_content; reconstruct it as text.+ content_blocks.append(AnthropicResponseTextBlock(text = f"<think>{reasoning}</think>"))+ if healed_events:+ emitted_tool_uses = 0+ for kind, value in healed_events:+ if kind == "text":+ text = str(value).strip()+ if text:+ content_blocks.append(AnthropicResponseTextBlock(text = text))+ continue+ if disable_parallel_tool_use and emitted_tool_uses >= 1:+ continue+ fn = value.get("function") or {}+ try:+ args = json.loads(fn.get("arguments", "{}"))+ except json.JSONDecodeError:+ args = {}+ tool_calls.append(value)+ emitted_tool_uses += 1+ content_blocks.append(+ AnthropicResponseToolUseBlock(+ id = anthropic_tool_use_id(value.get("id")),+ name = fn.get("name", ""),+ input = args,+ )+ )+ else:+ text = message.get("content") or ""+ if text:+ # Keep unpromoted bytes when healing is active; legacy stripping is only for opted-out+ # or no-client-tool requests. The protected helper preserves <think> rehearsal and+ # balanced [TOOL_CALLS] prose, gated on the declared tools so an inactive+ # NAME[ARGS]{...} example is kept.+ if not healing_active and response_format is None:+ text = _strip_tool_xml_for_display(+ text,+ auto_heal_tool_calls = True,+ enabled_tool_names = _display_tool_name_gate(openai_tools),+ )+ text = text.strip()+ if text:+ content_blocks.append(AnthropicResponseTextBlock(text = text))++ tool_calls = message.get("tool_calls") or []+ if disable_parallel_tool_use and len(tool_calls) > 1:+ tool_calls = tool_calls[:1]+ for tc in tool_calls:+ fn = tc.get("function") or {}+ try:+ args = json.loads(fn.get("arguments", "{}"))+ except json.JSONDecodeError:+ args = {}+ content_blocks.append(+ AnthropicResponseToolUseBlock(+ id = anthropic_tool_use_id(tc.get("id")),+ name = fn.get("name", ""),+ input = args,+ )+ )++ stop_reason = openai_finish_to_anthropic_stop(finish_reason, had_tool_calls = bool(tool_calls))++ usage = data.get("usage") or {}+ return _anthropic_message_json_response(+ message_id, model_name, content_blocks, stop_reason, usage+ )++async def _anthropic_passthrough_non_streaming(llama_backend,openai_messages,@@ -46235,98 +46960,15 @@ async def _anthropic_passthrough_non_streaming(except (httpx.RequestError, ValueError) as exc:logger.warning("tool-call nudge retry failed; keeping original: %s", exc)- choice = (data.get("choices") or [{}])[0]- message = choice.get("message") or {}- finish_reason = choice.get("finish_reason")-- healing_active = bool(_allowed_tools)- healed_events = (- heal_openai_message_events(message, _allowed_tools, openai_tools)- if healing_active- else None- )-- content_blocks = []- tool_calls = []- # Reasoning first: llama-server splits <think> into reasoning_content on any- # turn whose format it can parse, and Anthropic orders thinking ahead of the- # answer. Reading only `content` drops the trace and the model looks like it- # never thought.- reasoning = message.get("reasoning_content") or ""- if reasoning.strip():- if parse_think:- content_blocks.append(AnthropicResponseThinkingBlock(thinking = reasoning))- else:- # Thinking effectively off: the parser shunted a literal- # example into reasoning_content; reconstruct it as text.- content_blocks.append(- AnthropicResponseTextBlock(text = f"<think>{reasoning}</think>")- )- if healed_events:- emitted_tool_uses = 0- for kind, value in healed_events:- if kind == "text":- text = str(value).strip()- if text:- content_blocks.append(AnthropicResponseTextBlock(text = text))- continue- if disable_parallel_tool_use and emitted_tool_uses >= 1:- continue- fn = value.get("function") or {}- try:- args = json.loads(fn.get("arguments", "{}"))- except json.JSONDecodeError:- args = {}- tool_calls.append(value)- emitted_tool_uses += 1- content_blocks.append(- AnthropicResponseToolUseBlock(- id = anthropic_tool_use_id(value.get("id")),- name = fn.get("name", ""),- input = args,- )- )- else:- text = message.get("content") or ""- if text:- # Keep unpromoted bytes when healing is active; legacy stripping is only for opted-out- # or no-client-tool requests. The protected helper preserves <think> rehearsal and- # balanced [TOOL_CALLS] prose, gated on the declared tools so an inactive- # NAME[ARGS]{...} example is kept.- if not healing_active and response_format is None:- text = _strip_tool_xml_for_display(- text,- auto_heal_tool_calls = True,- enabled_tool_names = _display_tool_name_gate(openai_tools),- )- text = text.strip()- if text:- content_blocks.append(AnthropicResponseTextBlock(text = text))-- tool_calls = message.get("tool_calls") or []- if disable_parallel_tool_use and len(tool_calls) > 1:- tool_calls = tool_calls[:1]- for tc in tool_calls:- fn = tc.get("function") or {}- try:- args = json.loads(fn.get("arguments", "{}"))- except json.JSONDecodeError:- args = {}- content_blocks.append(- AnthropicResponseToolUseBlock(- id = anthropic_tool_use_id(tc.get("id")),- name = fn.get("name", ""),- input = args,- )- )-- stop_reason = openai_finish_to_anthropic_stop(- finish_reason, had_tool_calls = bool(tool_calls)- )-- usage = data.get("usage") or {}- return _anthropic_message_json_response(- message_id, model_name, content_blocks, stop_reason, usage+ return _anthropic_message_from_openai_completion(+ data,+ message_id,+ model_name,+ openai_tools = openai_tools,+ allowed_tools = _allowed_tools,+ disable_parallel_tool_use = disable_parallel_tool_use,+ response_format = response_format,+ parse_think = parse_think,)finally:await _stop_local_disconnect_cancel_watcher(_cancel_watcher)diff --git a/studio/backend/tests/test_anthropic_messages.py b/studio/backend/tests/test_anthropic_messages.pyindex d5f47cf9f08..b274fe39cd3 100644--- a/studio/backend/tests/test_anthropic_messages.py+++ b/studio/backend/tests/test_anthropic_messages.py@@ -3,6 +3,7 @@"""Tests for Anthropic Messages API schemas and translation layer (no server/GPU)."""+import contextlibimport sysimport osimport json@@ -6757,3 +6758,369 @@ def test_the_anthropic_count_refusal_is_name_aware():assert _messages_have_promotable_mcp_images([{"role": "tool", "tool_call_id": "x", "content": "out\n" + mcp_images.SENTINEL + envelope}])+++_COUNT_TOOL = {"name": "lookup", "description": "d", "input_schema": {"type": "object"}}+_COUNT_OPENAI_TOOL = {+ "type": "function",+ "function": {"name": "lookup", "description": "d", "parameters": {"type": "object"}},+}+_COUNT_USER = {"role": "user", "content": "hi"}+_COUNT_OTHER = {"name": "other", "input_schema": {"type": "object"}}+_COUNT_OPENAI_OTHER = {+ "type": "function",+ "function": {"name": "other", "description": "", "parameters": {"type": "object"}},+}+_COUNT_FORCED = {"type": "function", "function": {"name": "lookup"}}+_IP_WEB_SEARCH = {"type": "web_search_20250305", "name": "web_search"}+++class TestInProcessCountTokens:+ """/v1/messages/count_tokens on a resident MLX model prices what chat's count prices."""++ def _count(+ self,+ monkeypatch,+ route,+ payload,+ *,+ is_mlx = True,+ ):+ from starlette.requests import Request+ import routes.inference as inf++ calls = []++ class _Resident:+ active_model_name = "mlx/m"+ load_generation = 0+ models = {"mlx/m": {"is_mlx": is_mlx, "is_vision": False}}++ def count_chat_tokens(self, messages, system, **kwargs):+ calls.append((messages, system, kwargs))+ return 7, "mlx/m"++ async def _switch(*_args, **_kwargs):+ return None++ monkeypatch.setattr(inf, "get_inference_backend", lambda: _Resident())+ monkeypatch.setattr(inf, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False))+ monkeypatch.setattr(inf, "_maybe_auto_switch_model", _switch)+ monkeypatch.setattr(+ inf, "_detect_safetensors_features", lambda *a, **k: {"supports_tools": True}+ )+ scope = {"type": "http", "method": "POST", "path": "/", "headers": [], "query_string": b""}+ response = asyncio.run(route(payload = payload, request = Request(scope), current_subject = "t"))+ return json.loads(response.body), calls++ @pytest.mark.parametrize("policy", [None, True])+ @pytest.mark.parametrize(+ "anthropic, chat, tools",+ [+ (+ {"system": "be brief", "messages": [{"role": "user", "content": "hi"}]},+ {+ "messages": [+ {"role": "system", "content": "be brief"},+ {"role": "user", "content": "hi"},+ ],+ "tool_choice": "none",+ },+ [],+ ),+ (+ {+ "messages": [_COUNT_USER],+ "tools": [_COUNT_TOOL, _COUNT_OTHER],+ "tool_choice": {"type": "tool", "name": "lookup"},+ },+ {+ "messages": [_COUNT_USER],+ "tools": [_COUNT_OPENAI_TOOL, _COUNT_OPENAI_OTHER],+ "tool_choice": _COUNT_FORCED,+ },+ ["lookup"],+ ),+ (+ {"messages": [_COUNT_USER], "tools": [_IP_WEB_SEARCH]},+ {"messages": [_COUNT_USER], "enable_tools": True, "enabled_tools": ["web_search"]},+ ["web_search"],+ ),+ ],+ ids = ["plain", "forced_client_tool", "server_tool"],+ )+ def test_counts_what_chat_counts_for_the_translated_request(+ self, monkeypatch, policy, anthropic, chat, tools+ ):+ import routes.inference as inf+ from models.inference import ChatCountTokensRequest++ set_tool_policy(policy)+ try:+ body, calls = self._count(+ monkeypatch, inf.anthropic_count_tokens, AnthropicMessagesRequest(**anthropic)+ )+ _, chat_calls = self._count(+ monkeypatch, inf.chat_count_tokens, ChatCountTokensRequest(**chat)+ )+ finally:+ reset_tool_policy()+ assert body == {"input_tokens": 7}+ assert calls == chat_calls+ # A hard --enable-tools policy prices no catalog the Messages route did not select.+ assert [t["function"]["name"] for t in calls[0][2]["tools"] or []] == tools++ def test_an_archived_thread_prices_only_the_selected_server_tools(self, monkeypatch):+ import routes.inference as inf++ # Chat's own selection adds search_conversation for an archived thread.+ monkeypatch.setattr(inf, "_thread_has_conversation_archive", lambda _thread: True)+ payload = AnthropicMessagesRequest(+ messages = [_COUNT_USER], tools = [_IP_WEB_SEARCH], thread_id = "t"+ )+ _, calls = self._count(monkeypatch, inf.anthropic_count_tokens, payload)+ assert [t["function"]["name"] for t in calls[0][2]["tools"]] == ["web_search"]++ @pytest.mark.parametrize(+ "fields, is_mlx, status, detail",+ [+ ({"enable_tools": True}, False, 503, "No GGUF model loaded"),+ ],+ ids = ["not_mlx"],+ )+ def test_what_it_cannot_price_is_refused(self, monkeypatch, fields, is_mlx, status, detail):+ import routes.inference as inf++ payload = AnthropicMessagesRequest(messages = [{"role": "user", "content": "hi"}], **fields)+ with pytest.raises(HTTPException) as exc:+ self._count(monkeypatch, inf.anthropic_count_tokens, payload, is_mlx = is_mlx)+ assert exc.value.status_code == status+ assert detail in json.dumps(exc.value.detail)+++_IP_CHUNKS = [{"delta": {"content": "hi"}}, {"delta": {}, "finish_reason": "stop"}]+_IP_SSE = (+ "".join(f"data: {json.dumps({'choices': [c]})}\n\n" for c in _IP_CHUNKS) + "data: [DONE]\n\n"+)+_IP_JSON = {"choices": [{"message": {"content": "hi"}, "finish_reason": "stop"}]}+_IP_SOURCE = {"type": "base64", "media_type": "image/png", "data": "QUJD"}+_IP_IMAGE = {"type": "image", "source": _IP_SOURCE}+_IP_FORMAT = {"output_config": {"format": {"type": "json_schema", "schema": {}}}}+++class TestInProcessMessages:+ """/v1/messages on an in-process model: chat's output through the passthrough translators."""++ def _serve(+ self,+ monkeypatch,+ stream,+ *timing,+ sse = _IP_SSE,+ **fields,+ ):+ stop_at, taken = timing or (None, -1)+ from starlette.requests import Request+ from starlette.responses import JSONResponse+ import routes.inference as inf++ seen = self.seen = {"closed": [], "unstarted": 0, "monitor": []}++ async def fake_chat(chat_req, request, subject):+ seen.update(req = chat_req, skip = request.state.skip_api_monitor)+ seen["ui"] = inf._ui_stream_events_enabled(request)+ seen["cap"] = inf._delegated_tool_names.get()+ if not chat_req.stream:+ if stop_at is not None:+ request.state.generation_cancel_event.set()+ return JSONResponse(_IP_JSON)++ async def body():+ try:+ for i, line in enumerate(sse.split("\n\n")[:-1]):+ if i == stop_at: # a Stop by the Messages id+ seen["stopped"] = inf._cancel_by_keys([seen["message_id"]])+ seen["runs"] = inf.active_generations.snapshot()+ if raise_on_stop: # as the managed vLLM / SGLang tool path does+ raise asyncio.CancelledError()+ yield line + "\n\n"+ except BaseException as exc:+ seen["closed"].append(type(exc).__name__ + "!" * seen.get("late", 0))+ raise++ unstarted = lambda: seen.update(unstarted = 1) or asyncio.sleep(0) # noqa: E731+ return inf._SameTaskStreamingResponse(body(), unstarted_cleanup = unstarted)++ raise_on_stop = fields.pop("raise_on_stop", False)+ entry = {"is_mlx": True, "is_vision": fields.pop("vision", False)}+ if fields.pop("speech", False):+ entry.update(is_audio = True, audio_type = "csm")+ backend = SimpleNamespace(active_model_name = "m", models = {"m": entry})+ backend.count_chat_tokens = lambda *a, **k: (5, "m")+ switch = lambda *a, **k: seen.update(switch = k) or asyncio.sleep(0) # noqa: E731+ for name, value in [+ ("openai_chat_completions", fake_chat),+ ("get_inference_backend", lambda: backend),+ ("get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)),+ ("_maybe_auto_switch_model", switch),+ ("_detect_safetensors_features", lambda *a, **k: {}),+ ]:+ monkeypatch.setattr(inf, name, value)+ monkeypatch.setattr(inf.api_monitor, "finish", lambda _id, *a: seen["monitor"].append(a))+ scope = {"type": "http", "method": "POST", "path": "/v1/messages", "headers": []}+ request = seen["request"] = Request(scope)+ fields.setdefault("messages", [_COUNT_USER])+ payload = AnthropicMessagesRequest(model = "m", max_tokens = 16, stream = bool(stream), **fields)++ async def run():+ response = await inf.anthropic_messages(payload, request, "t")+ if not stream or taken is None:+ return response.body.decode() if not stream else await response._unstarted_cleanup()+ out, it = [], response.body_iterator+ async for part in it:+ out.append(part)+ if "message_start" in part:+ seen["message_id"] = json.loads(part.split("data: ", 1)[1])["message"]["id"]+ if len(out) == taken:+ with pytest.raises(asyncio.CancelledError):+ await it.athrow(asyncio.CancelledError())+ seen["late"] = 1 # closing only at loop shutdown is a leak+ return "".join(out)++ return asyncio.run(run()), seen++ @pytest.mark.parametrize(+ "stream, stop_at, taken, closed",+ [+ (1, 1, -1, ["GeneratorExit"]),+ (1, None, None, []),+ (1, None, 2, ["CancelledError"]),+ (0, 0, -1, []),+ ],+ ids = ["stop_by_message_id", "never_started", "outer_cancel", "stopped_json"],+ )+ def test_stop_ends_as_on_llama_server(self, monkeypatch, stream, stop_at, taken, closed):+ import routes.inference as inf++ with pytest.raises(asyncio.CancelledError) if not stream else contextlib.nullcontext():+ out, seen = self._serve(monkeypatch, stream, stop_at, taken, session_id = "s1")+ if not stream:+ return # the llama-server JSON path never answers a stopped run either+ assert seen["skip"] is True and seen["req"].session_id == "s1"+ assert seen["closed"] == closed and seen["unstarted"] == (taken is None)+ assert ("cancelled",) in seen["monitor"]+ assert not [k for k in inf._CANCEL_REGISTRY if "msg_" in str(k)]+ if stop_at is not None:+ assert seen["stopped"] == 1 and out.endswith('{"type": "message_stop"}\n\n')+ assert seen["runs"] == [] # chat's own tracker records the run+ assert seen["request"].scope.get("_unsloth_keepwarm_response_failed")++ @pytest.mark.parametrize(+ "fields, reason, multiple",+ [+ ({}, None, None),+ (_IP_FORMAT, None, None),+ ({"enable_tools": True, "permission_mode": "off"}, None, None),+ (+ {"messages": [{"role": "user", "content": [_IP_IMAGE, _IP_IMAGE]}], "vision": True},+ None,+ True,+ ),+ ({"messages": []}, "non-system", None),+ ],+ ids = ["plain", "json_format", "server_tools", "two_images", "no_turn"],+ )+ def test_refused_before_a_load(self, monkeypatch, fields, reason, multiple):+ with pytest.raises(HTTPException) if reason else contextlib.nullcontext():+ self._serve(monkeypatch, 0, **fields)+ sw = self.seen["switch"]+ assert sw["gguf_only"] is (reason is not None)+ assert reason in sw["gguf_only_reason"] if reason else sw["gguf_only_reason"] is None+ assert (sw["image_preflight"] or {}).get("multiple") is multiple++ def test_a_source_raising_on_stop_still_ends_the_stream_cleanly(self, monkeypatch):+ out, seen = self._serve(monkeypatch, 1, 1, -1, raise_on_stop = True)+ assert seen["stopped"] == 1 and out.endswith('{"type": "message_stop"}\n\n')+ assert not [+ k for k in __import__("routes.inference").inference._CANCEL_REGISTRY if "msg_" in str(k)+ ]++ @pytest.mark.parametrize("stream", [0, 1])+ def test_a_sampling_value_chat_rejects_is_a_400_before_a_load(self, monkeypatch, stream):+ # Messages bounds top_k less tightly than chat; a GGUF target still takes it.+ with pytest.raises(HTTPException) as err:+ self._serve(monkeypatch, stream, top_k = 200)+ assert err.value.status_code == 400 and "top_k" in json.dumps(err.value.detail)+ assert self.seen["switch"]["gguf_only"] is True and "req" not in self.seen++ @pytest.mark.parametrize("stream", [0, 1])+ def test_a_resident_speech_model_is_not_served_through_chat(self, monkeypatch, stream):+ # Chat answers a TTS model with a finished clip, never a text reply.+ with pytest.raises(HTTPException) as err:+ self._serve(monkeypatch, stream, speech = True)+ assert "req" not in self.seen+ assert "GGUF" in json.dumps(err.value.detail)++ @pytest.mark.parametrize("vision", [True, False])+ def test_an_image_on_a_text_only_model_names_its_block(self, monkeypatch, vision):+ turns = [+ {"role": "user", "content": "Hi."},+ {"role": "assistant", "content": "Hello."},+ {"role": "user", "content": [{"type": "text", "text": "What?"}, _IP_IMAGE]},+ ]+ with contextlib.nullcontext() if vision else pytest.raises(HTTPException) as err:+ self._serve(monkeypatch, 0, messages = turns, vision = vision)+ assert ("req" in self.seen) is vision+ if not vision:+ assert err.value.detail.startswith("messages.2.content.1.image: ")++ @pytest.mark.parametrize("choice, kept", [(None, True), ({"type": "auto"}, False)])+ def test_a_json_output_format_reaches_chat_unless_a_tool_may_run(+ self, monkeypatch, choice, kept+ ):+ tool = {"name": "lookup", "input_schema": {"type": "object"}}+ tools = {"tools": [tool], "tool_choice": choice} if choice else {}+ self._serve(monkeypatch, 0, **_IP_FORMAT, **tools)+ assert (self.seen["req"].response_format is not None) is kept++ @pytest.mark.parametrize("stream", [1, 0])+ def test_server_tools_run_in_chats_loop_and_answer_as_on_llama_server(+ self, monkeypatch, stream+ ):+ frames = [+ {"choices": [{"delta": {"reasoning_content": "plan"}}]},+ {"choices": [{"delta": {"content": "Searching."}}]},+ {+ "type": "tool_start",+ "tool_name": "web_search",+ "tool_call_id": "c1",+ "arguments": {},+ },+ {"type": "tool_status", "content": "Searching"},+ {"type": "tool_end", "tool_name": "web_search", "tool_call_id": "c1", "result": "r"},+ {"choices": [{"delta": {"reasoning_content": "ok"}}]},+ {"choices": [{"delta": {"content": "Done."}, "finish_reason": "stop"}]},+ {"choices": [], "usage": {"prompt_tokens": 9, "completion_tokens": 4}},+ ]+ sse = "".join(f"data: {json.dumps(f)}\n\n" for f in frames) + "data: [DONE]\n\n"+ choice = {"type": "tool", "name": "web_search_20250305", "disable_parallel_tool_use": True}+ out, seen = self._serve(+ monkeypatch, stream, sse = sse, tools = [_IP_WEB_SEARCH], tool_choice = choice+ )+ req = seen["req"]+ assert seen["ui"] is True and seen["cap"] == {"web_search"}+ assert req.enabled_tools == ["web_search"]+ assert req.tool_choice == {"type": "function", "function": {"name": "web_search"}}+ assert req.confirm_tool_calls is False and req.parallel_tool_calls is False+ if stream:+ starts = [+ json.loads(e.split("data: ")[1]) for e in out.split("\n\n") if "block_start" in e+ ]+ kinds = [s["content_block"]["type"] for s in starts]+ assert out.endswith('{"type": "message_stop"}\n\n')+ else:+ body = json.loads(out)+ kinds = [b["type"] for b in body["content"]]+ assert body["usage"]["input_tokens"] == 9 and body["stop_reason"] == "end_turn"+ tool_blocks = ["server_tool_use", "web_search_tool_result"]+ assert kinds == ["thinking", "text", *tool_blocks, "thinking", "text"]diff --git a/studio/backend/tests/test_openai_auto_switch.py b/studio/backend/tests/test_openai_auto_switch.pyindex 3c1f6e306c1..7244ca532dc 100644--- a/studio/backend/tests/test_openai_auto_switch.py+++ b/studio/backend/tests/test_openai_auto_switch.py@@ -4841,36 +4841,62 @@ def test_count_tokens_rejects_malformed_tool_before_switch(monkeypatch):assert rec.calls == []-def test_count_tokens_forwards_vision_guard_to_switch(monkeypatch):+ "image, server_tools, device, gguf_only, reason",+ [+ (True, False, "mlx", True, "Cannot count tokens for messages containing images."),+ ("mcp", False, "mlx", True, "Cannot count tokens for messages containing images."),+ (False, True, "mlx", False, None),+ (False, False, "mlx", False, None),+ (False, False, "cuda", True, None),+ ],+ ids = ["image", "mcp_replay", "server_tools", "mlx_text", "transformers"],+)+def test_count_tokens_forwards_vision_guard_to_switch(+ monkeypatch, image, server_tools, device, gguf_only, reason+):# Codex P2: an image /v1/messages/count_tokens naming a text-only GGUF must# carry the same require_vision guard as /messages, so it can't evict a loaded- # vision model for a swap that can't serve the request.+ # vision model for a swap that can't serve the request. A non-GGUF target is+ # loaded only when the resident-MLX count can price the request.+ from utils.hardware import DeviceType+captured = {}- async def _capture(- model,- request,- subject,- *,- require_vision = False,- claim_resident = True,- require_audio_input = False,- gguf_only = False,- ):- captured["require_vision"] = require_vision- captured["claim_resident"] = claim_resident- captured["gguf_only"] = gguf_only+ async def _capture(model, request, subject, **kwargs):+ captured.update(kwargs)raise _Reached()- monkeypatch.setattr(inference_route, "_anthropic_request_has_image", lambda p, **_: True)+ from core.inference import mcp_images+ from models.inference import AnthropicMessage++ monkeypatch.setattr(+ inference_route, "_anthropic_request_has_image", lambda p, **_: image is True+ )monkeypatch.setattr(inference_route, "_maybe_auto_switch_model", _capture)+ monkeypatch.setattr("utils.hardware.get_device", lambda: DeviceType(device))payload = _anthropic_payload_with_tools(None) # no tools -> tool validation passes+ payload.enable_tools = server_tools+ if image == "mcp":+ envelope = json.dumps([{"data": "QUJD", "mimeType": "image/png"}])+ call = {"type": "tool_use", "id": "toolu_0", "name": "mcp__shot__capture", "input": {}}+ result = {+ "type": "tool_result",+ "tool_use_id": "toolu_0",+ "content": "out\n" + mcp_images.SENTINEL + envelope,+ }+ payload.messages += [+ AnthropicMessage(role = "assistant", content = [call]),+ AnthropicMessage(role = "user", content = [result]),+ ]with pytest.raises(_Reached):asyncio.run(inference_route.anthropic_count_tokens(payload, object(), "tester"))- assert captured["require_vision"] is True+ assert captured["require_vision"] is (image is True)assert captured["claim_resident"] is False- # llama.cpp serves this endpoint alone, so a non-GGUF swap must not be attempted.- assert captured["gguf_only"] is True+ assert captured["gguf_only"] is gguf_only+ assert (+ reason in captured["gguf_only_reason"] if reason else captured["gguf_only_reason"] is None+ )# ── /chat/count_tokens: what the recount prices ───────────────────diff --git a/studio/backend/tests/test_safetensors_tool_loop.py b/studio/backend/tests/test_safetensors_tool_loop.pyindex fe96399cbb9..293647bd5be 100644--- a/studio/backend/tests/test_safetensors_tool_loop.py+++ b/studio/backend/tests/test_safetensors_tool_loop.py@@ -1124,6 +1124,9 @@ def _collect_events(generator, max_events = 200):return events+_MAKE_LOOP_TOOLS = ["web_search", "python", "terminal"]++def _make_loop(*,turns,@@ -1136,7 +1139,8 @@ def _make_loop("""turn_iter = iter(turns)- def _gen(_messages):+ def _gen(_messages, **kw):+ exec_fn.offered.append([t["function"]["name"] for t in kw.get("active_tools") or []])try:chunks = next(turn_iter)except StopIteration:@@ -1147,14 +1151,11 @@ def _make_loop(yield accexec_fn = FakeExecuteTool(exec_results or [])+ exec_fn.offered = []return run_safetensors_tool_loop(single_turn = _gen,messages = [{"role": "user", "content": "hi"}],- tools = [- {"type": "function", "function": {"name": "web_search"}},- {"type": "function", "function": {"name": "python"}},- {"type": "function", "function": {"name": "terminal"}},- ],+ tools = [{"type": "function", "function": {"name": n}} for n in _MAKE_LOOP_TOOLS],execute_tool = exec_fn,**kwargs,), exec_fn@@ -6010,3 +6011,26 @@ def test_a_browser_turn_is_nudged_after_retrieval_was_injected(monkeypatch):))assert [name for name, _ in exec_fn.calls] == ["browser_snapshot", "browser_click"]+++def test_a_forced_tool_is_offered_alone_until_it_runs_and_the_cap_runs_one_call():+ loop, exec_fn = _make_loop(+ turns = [["<function=web_search></function><function=python></function>"], ["Done."]],+ tool_choice = {"type": "function", "function": {"name": "web_search"}},+ disable_parallel_tool_use = True,+ )+ _collect_events(loop)+ assert exec_fn.offered == [["web_search"], _MAKE_LOOP_TOOLS]+ assert exec_fn.calls == [("web_search", {})]+++def test_a_forced_tool_search_is_released_once_discovery_runs(monkeypatch):+ # As on llama-server: any executed call resolves the forced choice, discovery included.+ monkeypatch.setitem(globals(), "_MAKE_LOOP_TOOLS", ["web_search", "mcp_tool_search"])+ loop, exec_fn = _make_loop(+ turns = [["<function=mcp_tool_search><parameter=query>x</parameter></function>"], ["Done."]],+ tool_choice = {"type": "function", "function": {"name": "mcp_tool_search"}},+ )+ _collect_events(loop)+ assert [name for name, _ in exec_fn.calls] == ["mcp_tool_search"]+ assert exec_fn.offered == [["mcp_tool_search"], ["web_search", "mcp_tool_search"]]