Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: resume a reply that stopped mid-thought on MLX (#12504)
Long Yixing95531ba1e149
diff --git a/studio/backend/core/inference/chat_template_helpers.py b/studio/backend/core/inference/chat_template_helpers.pyindex 377f1073811..16bb5d270e3 100644--- a/studio/backend/core/inference/chat_template_helpers.py+++ b/studio/backend/core/inference/chat_template_helpers.py@@ -2584,6 +2584,7 @@ def detect_think_prefill(special_tokens = None,*,preserves_think_close: bool = False,+ resumes_thought: bool = False,) -> str:"""Return the trailing open ``<think>`` prefill of a rendered prompt.@@ -2606,6 +2607,9 @@ def detect_think_prefill(``NativeToolTokenDecoder`` does so the parser can see a call rehearsed inside the block, and asa path streaming the detokenizer's own text does. The special-token list then says nothing, andskipping the opener is the same bug mirrored: a stray ``</think>``.++ ``resumes_thought``: the prompt ends inside a resumed thought (the client's text), so only the+ bare opener is returned."""if not prompt:return ""@@ -2613,11 +2617,11 @@ def detect_think_prefill(if open_idx == -1:return ""tail = prompt[open_idx:]- if _THINK_CLOSE in tail or tail.strip() != _THINK_OPEN:+ if _THINK_CLOSE in tail or (tail.strip() != _THINK_OPEN and not resumes_thought):return ""if not preserves_think_close and special_tokens and _THINK_CLOSE in set(special_tokens):return ""- return tail+ return _THINK_OPEN if resumes_thought else taildef _normalize_tool_call_arguments(messages: list) -> list:@@ -3124,6 +3128,34 @@ def strip_open_reasoning_prefill(prefix: str) -> str:return prefix[:open_at]+class ThoughtUnresumableError(ValueError):+ """A trailing thought this model cannot reopen; the message is client-safe."""++ public = True+ openai_param = "continue_final_message"++ def __init__(self):+ super().__init__("This model cannot resume a response that stopped mid-thought. Use Retry.")+++def template_resumes_thought(tokenizer, tools = None) -> bool:+ """Whether a thought can be reopened as ``<think>`` text (not native reasoning channels)."""+ if detect_reasoning_channel_markers(tokenizer, tools = tools) is not None:+ return False+ return any(+ _THINK_OPEN in template for template in _selected_chat_template_strings(tokenizer, tools)+ )+++def splice_resumed_thought(prefix: str, thought: str) -> str:+ """Reopen *thought* on a generation prompt, cutting the template's own reasoning prefill (open+ or empty closed block) as llama-server does."""+ open_at = prefix.rfind(_THINK_OPEN)+ if open_at != -1 and prefix[open_at + len(_THINK_OPEN) :].strip() in ("", _THINK_CLOSE):+ prefix = prefix[:open_at]+ return f"{prefix}{_THINK_OPEN}{thought}"++def render_prompt_with_boundary(processor,messages: list,@@ -3232,6 +3264,11 @@ def apply_chat_template_for_generation(# renders as an ordinary new turn._continue_text = trailing_assistant_text(messages) if continue_final_message else None_continuing = bool(_continue_text)+ _resumes_thought = (+ continue_final_message and trailing_assistant_resume_kind(messages) == "reasoning_content"+ )+ if _resumes_thought and not template_resumes_thought(tokenizer, tools):+ raise ThoughtUnresumableError()_boundary_kwargs = ({"add_generation_prompt": False, "continue_final_message": True}if _continuing@@ -3275,7 +3312,22 @@ def apply_chat_template_for_generation(return f"{strip_open_reasoning_prefill(prefix)}{partial}"raise TypeError("no attempt rendered the continuation prefix")+ def _render_thought_continuation(msgs: list) -> str:+ # Templates render a final thought closed, so there is no boundary to cut at.+ for kwargs in attempts:+ swept = _swept_for(kwargs, msgs)+ try:+ prefix = tokenizer.apply_chat_template(+ swept[:-1], tokenize = False, add_generation_prompt = True, **kwargs+ )+ except TypeError:+ continue+ return splice_resumed_thought(prefix, swept[-1]["reasoning_content"])+ raise TypeError("no attempt rendered the thought continuation prefix")+def _render_with_fallback(msgs: list) -> str:+ if _resumes_thought:+ return _render_thought_continuation(msgs)try:return _render(msgs)except TypeError:diff --git a/studio/backend/core/inference/mlx_inference.py b/studio/backend/core/inference/mlx_inference.pyindex a78b10ee40d..cbfa407f110 100644--- a/studio/backend/core/inference/mlx_inference.py+++ b/studio/backend/core/inference/mlx_inference.py@@ -44,6 +44,8 @@ from core.inference.chat_template_helpers import (normalize_reasoning_snapshots,prompt_opens_reasoning_channel,strip_open_reasoning_prefill,+ ThoughtUnresumableError,+ trailing_assistant_resume_kind,trailing_assistant_text,vlm_prompt_issue as _vlm_prompt_issue,)@@ -1090,6 +1092,23 @@ def _ascii_registry_key(value):return value.lower()+def _resumes_thought(messages, continue_final_message) -> bool:+ return bool(continue_final_message) and (+ trailing_assistant_resume_kind(messages) == "reasoning_content"+ )+++def _think_prefix(prompt, special_tokens, messages, continue_final_message, **kwargs) -> str:+ """``detect_think_prefill``; refuses a resumed thought whose ``</think>`` decoding would strip."""+ from core.inference.chat_template_helpers import detect_think_prefill++ resumes = _resumes_thought(messages, continue_final_message)+ prefix = detect_think_prefill(prompt, special_tokens, resumes_thought = resumes, **kwargs)+ if resumes and not prefix:+ raise ThoughtUnresumableError()+ return prefix++def _render_registered_vlm_prompt(processor,model,@@ -1131,6 +1150,9 @@ def _render_registered_vlm_prompt(config = dict(config) if isinstance(config, dict) else dict(config.__dict__)config["model_type"] = canonical+ # mlx-vlm's formatters carry no reasoning opener to reopen a thought on.+ if _resumes_thought(messages, continue_final_message):+ return None# Recovery path: sweeps the caller's original list rather than reusing a copy (#7066).swept = neutralize_control_markup_in_messages(messages, None, markup_for_tokenizer(processor))partial = trailing_assistant_text(swept) if continue_final_message else None@@ -5172,8 +5194,6 @@ class MLXInferenceBackend:from mlx_lm import stream_generatefrom mlx_lm.sample_utils import make_sampler- from core.inference.chat_template_helpers import detect_think_prefill-render_result = self._render_text_prompt(messages,tools = tools,@@ -5196,9 +5216,11 @@ class MLXInferenceBackend:) and decoder_preserves_token(self._tokenizer, "</think>", reasoning_control_tokens(reasoning_channel_markers))- think_prefix = detect_think_prefill(+ think_prefix = _think_prefix(prompt,getattr(self._tokenizer, "all_special_tokens", None),+ messages,+ continue_final_message,preserves_think_close = think_close_survives,)constraint = _build_grammar_constraint(@@ -5479,8 +5501,6 @@ class MLXInferenceBackend:) -> "_TextRowPlan":from mlx_lm.sample_utils import make_sampler- from core.inference.chat_template_helpers import detect_think_prefill-render_result = self._render_text_prompt(messages,tools = tools,@@ -5493,9 +5513,11 @@ class MLXInferenceBackend:reasoning_channel_markers = render_result.reasoning_channel_markers_resumed_partial = bool(continue_final_message and trailing_assistant_text(messages))- think_prefix = detect_think_prefill(+ think_prefix = _think_prefix(prompt,getattr(self._tokenizer, "all_special_tokens", None),+ messages,+ continue_final_message,preserves_think_close = (bool(tools) or tool_protocol_active or reasoning_channel_markers is not None)@@ -5755,10 +5777,11 @@ class MLXInferenceBackend:continue_final_message = continue_final_message,)- from core.inference.chat_template_helpers import detect_think_prefill-- think_prefix = detect_think_prefill(- prompt, getattr(chat_target, "all_special_tokens", None)+ think_prefix = _think_prefix(+ prompt,+ getattr(chat_target, "all_special_tokens", None),+ messages,+ continue_final_message,)normalizer = (make_reasoning_normalizer(@@ -5872,9 +5895,11 @@ class MLXInferenceBackend:from core.inference.chat_template_helpers import detect_think_prefill# Detected once: the decoder keeps the delimiters the normalizer below consumes.- prefill = detect_think_prefill(+ prefill = _think_prefix(prompt,getattr(chat_target, "all_special_tokens", None),+ messages,+ continue_final_message,# The same activation the decoder below uses: in unrestricted mode ``tools`` is# empty while the protocol is live, so ``bool(tools)`` said the closer would be# stripped, the opener was suppressed, and the stream ran on to an orphandiff --git a/studio/backend/core/inference/safetensors_agentic.py b/studio/backend/core/inference/safetensors_agentic.pyindex ffc2b6b3aa5..b5b894ba629 100644--- a/studio/backend/core/inference/safetensors_agentic.py+++ b/studio/backend/core/inference/safetensors_agentic.py@@ -73,7 +73,7 @@ from core.inference.tool_loop_controller import ()from core.inference.chat_template_helpers import (append_assistant_turn,- trailing_assistant_text,+ trailing_assistant_resume_kind,)from core.inference.passthrough_healing import nudge_enabledfrom state.tool_approvals import (@@ -357,6 +357,26 @@ def _status_for_tool(tool_name: str, arguments: dict) -> str:return status_for_tool(tool_name, arguments)+def _append_raw_turn(conversation: list, assistant_msg: dict, *, continue_final_message: bool):+ """``append_assistant_turn`` for raw-text turns: a resumed thought is replayed whole after the+ re-emitted ``<think>``; text without that opener drops it."""+ if not (+ continue_final_message+ and trailing_assistant_resume_kind(conversation) == "reasoning_content"+ and isinstance(assistant_msg.get("content"), str)+ ):+ append_assistant_turn(+ conversation, assistant_msg, continue_final_message = continue_final_message+ )+ return+ thought = conversation[-1]["reasoning_content"]+ merged = {**conversation[-1], **assistant_msg}+ merged.pop("reasoning_content", None)+ if merged["content"].startswith("<think>"):+ merged["content"] = f"<think>{thought}{merged['content'][len('<think>'):]}"+ conversation[-1] = merged++def _reprompt_intent_text(text: str,*,@@ -677,7 +697,7 @@ def run_safetensors_tool_loop(# plus its result, moving the boundary so the model opens a fresh answer._skip_autoinject = (confirm_tool_calls and not bypass_permissions and permission_mode not in ("auto", "off")- ) or bool(continue_final_message and trailing_assistant_text(conversation))+ ) or bool(continue_final_message and trailing_assistant_resume_kind(conversation) is not None)_auto = None if _skip_autoinject else build_rag_autoinject(conversation, rag_scope)if _auto:for _ev in _auto["events"]:@@ -1241,7 +1261,7 @@ def run_safetensors_tool_loop()# Merges into a resumed partial: the nudge that follows is a user# turn, so a second assistant turn breaks alternation.- append_assistant_turn(+ _append_raw_turn(conversation,{"role": "assistant", "content": intent_text},continue_final_message = continue_final_message,@@ -1414,7 +1434,7 @@ def run_safetensors_tool_loop(if not decision.should_execute:if content_text and not assistant_appended:- append_assistant_turn(+ _append_raw_turn(conversation,assistant_msg,continue_final_message = continue_final_message,@@ -1444,7 +1464,7 @@ def run_safetensors_tool_loop(assistant_msg["tool_calls"] = [decision.as_assistant_tool_call()]# Merges into a resumed partial, so a continued turn that calls a tool# stays one assistant message.- append_assistant_turn(+ _append_raw_turn(conversation,assistant_msg,continue_final_message = continue_final_message,diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex b5886c9725c..fc0aa5666bc 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -445,7 +445,7 @@ def _continue_final_message(payload, *, thought: bool = False) -> bool:Nothing resumable (no assistant turn, or one holding tool calls) degrades to anordinary new turn rather than erroring. ``thought`` also resumes a turn holding only- ``reasoning_content``, which only llama-server can continue inside its reasoning block.+ ``reasoning_content``, which only llama-server and MLX reopen inside the reasoning block."""if not getattr(payload, "continue_final_message", None):return False@@ -27038,7 +27038,7 @@ async def produce_openai_chat_completions("call: guided decoding admits only the document, so no tool would be ""called. Use tool_choice 'auto' or 'none'.",)- if _continue_final_message(payload):+ if _continue_final_message(payload, thought = True):_raise_unsupported_openai_parameter("response_format","response_format cannot resume a partial assistant message: guided "@@ -29745,8 +29745,9 @@ async def produce_openai_chat_completions(# A continued turn renders no generation prompt, so nothing is prefilled and the# resumed text is the visible answer. Only the first turn continues; later tool-loop- # turns render a fresh generation prompt and prefill as usual.- _sf_continue = _continue_final_message(payload)+ # turns render a fresh generation prompt and prefill as usual. MLX also resumes a thought.+ _sf_continue = _continue_final_message(payload, thought = bool(_sf_model_info.get("is_mlx")))+ _sf_resumes_thought = _sf_continue and not _continue_final_message(payload)_sf_continued_turn = [_sf_continue]_sf_single_block = _response_format_constrains_decoding(payload)@@ -29755,7 +29756,7 @@ async def produce_openai_chat_completions(prefilled = _sf_reasoning_prefilledif _sf_continued_turn[0]:_sf_continued_turn[0] = False- prefilled = False+ prefilled = _sf_resumes_thoughtreturn _ResponsesReasoningExtractor(parse_think_markers = _sf_parse_think,reasoning_prefilled = prefilled,@@ -29895,9 +29896,7 @@ async def produce_openai_chat_completions(# Strip stale tool-call XML from prior assistant turns. The continuation target# keeps its exact whitespace (see the GGUF path).- _sf_continue_target = (- chat_messages[-1] if _continue_final_message(payload) and chat_messages else None- )+ _sf_continue_target = chat_messages[-1] if _sf_continue and chat_messages else None_sf_chat_messages = []for _msg in chat_messages:if _msg.get("role") == "assistant" and isinstance(_msg.get("content"), str):@@ -29984,7 +29983,7 @@ async def produce_openai_chat_completions(enable_thinking = payload.enable_thinking,reasoning_effort = payload.reasoning_effort,preserve_thinking = payload.preserve_thinking,- continue_final_message = _continue_final_message(payload),+ continue_final_message = _sf_continue,auto_heal_tool_calls = _sf_auto_heal_tool_calls,nudge_tool_calls = payload.nudge_tool_calls,max_tool_iterations = _sf_tool_budget,@@ -30258,7 +30257,7 @@ async def produce_openai_chat_completions(# boundary and pin the mode of the turn that text came from (the same events# reset the streaming path's extractor).continued = _sf_continue- prefilled = _sf_reasoning_prefilled and not continued+ prefilled = _sf_resumes_thought if continued else _sf_reasoning_prefilledgen = sf_generate_with_tools()for event in gen:if cancel_event.is_set():@@ -30279,7 +30278,7 @@ async def produce_openai_chat_completions(auto_heal_tool_calls = _sf_auto_heal_tool_calls,enabled_tool_names = _sf_display_tool_names,)- prefilled = _sf_reasoning_prefilled and not continued+ prefilled = _sf_resumes_thought if continued else _sf_reasoning_prefilledelif _event_type == "tool_start" or (_event_type == "status" and not event.get("text")):@@ -30406,7 +30405,7 @@ async def produce_openai_chat_completions(gen_kwargs["reasoning_effort"] = payload.reasoning_effortif payload.preserve_thinking is not None:gen_kwargs["preserve_thinking"] = payload.preserve_thinking- if _continue_final_message(payload):+ if _sf_continue:gen_kwargs["continue_final_message"] = True# Client-tool passthrough (safetensors + MLX): render client tools, generate one turn,@@ -31024,7 +31023,9 @@ async def produce_openai_chat_completions(_reasoning_text, _visible_text = _extract_responses_reasoning(full_text,parse_think_markers = _sf_parse_think,- reasoning_prefilled = _sf_reasoning_prefilled and not _sf_continue,+ reasoning_prefilled = (+ _sf_resumes_thought if _sf_continue else _sf_reasoning_prefilled+ ),single_block = _sf_single_block,)# Client-tool passthrough: promote text-form calls; opt-in singlediff --git a/studio/backend/tests/test_chat_template_continuation.py b/studio/backend/tests/test_chat_template_continuation.pyindex 54dca4253dc..d1dfacb9e6d 100644--- a/studio/backend/tests/test_chat_template_continuation.py+++ b/studio/backend/tests/test_chat_template_continuation.py@@ -11,6 +11,7 @@ assistant turn can be resumed, so anything else renders normally instead of raisfrom __future__ import annotations+import copyimport importlibimport sysimport types@@ -25,8 +26,10 @@ if str(_BACKEND) not in sys.path:from core.inference.chat_template_helpers import ( # noqa: E402append_assistant_turn,apply_chat_template_for_generation,+ detect_think_prefill,last_user_text,render_prompt_with_boundary,+ ThoughtUnresumableError,trailing_assistant_text,)@@ -835,6 +838,9 @@ def _sf_completion(monkeypatch,events,stats = None,+ messages = None,+ is_mlx = False,+ seen = None,**body,):"""POST a non-streaming safetensors tool-loop chat and return the assistant message."""@@ -851,9 +857,11 @@ def _sf_completion(class _Safetensors:active_model_name = "qwen3"- models = {"qwen3": {"chat_template_info": {"template": _THINK_TEMPLATE}}}+ models = {"qwen3": {"chat_template_info": {"template": _THINK_TEMPLATE}, "is_mlx": is_mlx}}def generate_chat_completion_with_tools(self, **kwargs):+ if seen is not None:+ seen.update(kwargs)# The worker fills this on gen_done; the route reads it for usage and# for finish_reason "length".if stats is not None:@@ -879,7 +887,8 @@ def _sf_completion(resp = TestClient(app).post("/v1/chat/completions",json = {- "messages": [+ "messages": messages+ or [{"role": "user", "content": "weather?"},{"role": "assistant", "content": "Let me check the "},],@@ -912,6 +921,150 @@ def test_a_resumed_turn_without_a_tool_keeps_the_partial_visible(monkeypatch):assert not message["reasoning_content"]+_CUT_THOUGHT = [+ {"role": "user", "content": "weather?"},+ {"role": "assistant", "content": "", "reasoning_content": "\nThe user wants the "},+]+# The MLX backend re-emits a bare opener ahead of a resumed thought's tail.+_RESUMED_THOUGHT_EVENTS = [+ {"type": "content", "text": "<think>forecast.</think>\n\nIt is 21C."},+ {"type": "status", "text": ""},+]+++def test_mlx_resumes_a_thought_and_reports_its_tail_as_reasoning(monkeypatch):+ seen = {}+ message = _sf_completion(+ monkeypatch,+ _RESUMED_THOUGHT_EVENTS,+ messages = _CUT_THOUGHT,+ is_mlx = True,+ seen = seen,+ # The turn is reasoning whatever the toggle now says.+ enable_thinking = False,+ )["message"]+ assert seen["continue_final_message"] is True+ assert message["reasoning_content"] == "forecast."+ assert message["content"] == "\n\nIt is 21C."+++def test_transformers_answers_after_a_thought_instead_of_resuming_it(monkeypatch):+ seen = {}+ _sf_completion(monkeypatch, _RESUMED_THOUGHT_EVENTS, messages = _CUT_THOUGHT, seen = seen)+ assert seen["continue_final_message"] is False+++class _ThinkTokenizer(_ChatMLTokenizer):+ chat_template = "{{ '<think>' }}"++ def __init__(self, prefill):+ self.prefill = prefill++ def apply_chat_template(+ self,+ messages,+ *,+ add_generation_prompt = True,+ **kw,+ ):+ out = super().apply_chat_template(+ messages, add_generation_prompt = add_generation_prompt, **kw+ )+ return out + self.prefill if add_generation_prompt else out+++ "prefill", ["", "<think>\n", "<think>\n\n</think>\n\n"], ids = ["none", "open", "closed"]+)+def test_a_thought_resumes_inside_its_reasoning_block(prefill):+ thought = "\nTry 2 then"+ prompt = apply_chat_template_for_generation(+ _ThinkTokenizer(prefill),+ [+ {"role": "user", "content": "q"},+ {"role": "assistant", "content": "", "reasoning_content": thought},+ ],+ continue_final_message = True,+ )+ assert prompt == "<|im_start|>user\nq<|im_end|>\n<|im_start|>assistant\n<think>\nTry 2 then"+ assert detect_think_prefill(prompt, resumes_thought = True) == "<think>"+ assert detect_think_prefill(prompt) == ""+++def test_a_think_in_history_is_not_taken_for_the_prefill():+ history = [+ {"role": "user", "content": "q"},+ {"role": "assistant", "content": "<think>a</think>b"},+ {"role": "user", "content": "r"},+ ]+ prompt = apply_chat_template_for_generation(+ _ThinkTokenizer(""),+ history + [{"role": "assistant", "content": "", "reasoning_content": "c"}],+ continue_final_message = True,+ )+ assert prompt.endswith("<|im_start|>user\nr<|im_end|>\n<|im_start|>assistant\n<think>c")+ assert "<think>a</think>b" in prompt+++class _GemmaTokenizer(_ChatMLTokenizer):+ chat_template = "{{ '<think>' }}<|channel>thought\n"+++ "tokenizer", [_ChatMLTokenizer, _GemmaTokenizer], ids = ["no-think", "native"]+)+def test_a_template_without_a_think_opener_refuses_to_resume_a_thought(tokenizer):+ with pytest.raises(ThoughtUnresumableError) as refused:+ apply_chat_template_for_generation(+ tokenizer(),+ [+ {"role": "user", "content": "q"},+ {"role": "assistant", "content": "", "reasoning_content": "x"},+ ],+ continue_final_message = True,+ )+ # The worker forwards only a public refusal, and the route answers its parameter with a 400.+ assert refused.value.public and refused.value.openai_param == "continue_final_message"+++def test_mlx_refuses_a_thought_its_decoder_would_never_close():+ from core.inference.mlx_inference import _think_prefix++ prompt = "<|im_start|>assistant\n<think>\nThe user wants the "+ closer = ["</think>"]+ assert (+ _think_prefix(prompt, closer, _CUT_THOUGHT, True, preserves_think_close = True) == "<think>"+ )+ with pytest.raises(ThoughtUnresumableError):+ _think_prefix(prompt, closer, _CUT_THOUGHT, True)+ assert _think_prefix(prompt, closer, _CUT_THOUGHT, False) == ""+++def test_the_tool_loop_replays_a_resumed_thought_whole():+ from core.inference.safetensors_agentic import _append_raw_turn++ conversation = copy.deepcopy(_CUT_THOUGHT)+ _append_raw_turn(+ conversation,+ {"role": "assistant", "content": "<think>forecast.</think>Checking.", "tool_calls": [{}]},+ continue_final_message = True,+ )+ assert conversation[1:] == [+ {+ "role": "assistant",+ "content": "<think>\nThe user wants the forecast.</think>Checking.",+ "tool_calls": [{}],+ }+ ]+ conversation = copy.deepcopy(_CUT_THOUGHT)+ _append_raw_turn(+ conversation,+ {"role": "assistant", "content": "I will search."},+ continue_final_message = True,+ )+ assert conversation[1:] == [{"role": "assistant", "content": "I will search."}]++@pytest.mark.parametrize("gguf", [False, True], ids = ["safetensors", "gguf"])def test_a_tts_model_refuses_a_continuation(monkeypatch, gguf):"""Both TTS branches re-speak the newest user text before any continuation handling,diff --git a/studio/backend/tests/test_safetensors_reasoning_stream.py b/studio/backend/tests/test_safetensors_reasoning_stream.pyindex 58b63241697..1941a5e0514 100644--- a/studio/backend/tests/test_safetensors_reasoning_stream.py+++ b/studio/backend/tests/test_safetensors_reasoning_stream.py@@ -14,6 +14,7 @@ loop against synthetic events, and covers the split through the route itself.from __future__ import annotations+import jsonimport sysfrom pathlib import Path@@ -633,9 +634,11 @@ def _sf_route_message(is_mlx = False,features = None,model_info = None,+ seen = None,+ status = 200,**body,):- """POST a non-streaming safetensors chat completion and return the assistant message."""+ """POST a safetensors chat completion and return the assistant message, joined if streamed."""from fastapi import FastAPIfrom fastapi.testclient import TestClient@@ -659,6 +662,8 @@ def _sf_route_message(}def generate_chat_response(self, **kwargs):+ if seen is not None:+ seen.update(kwargs)yield from snapshotsdef reset_generation_state(self, *_args):@@ -687,8 +692,59 @@ def _sf_route_message(**body,},)- assert resp.status_code == 200, resp.text- return resp.json()["choices"][0]["message"]+ assert resp.status_code == status, resp.text+ if status != 200:+ return resp.json()["error"]+ if not body.get("stream"):+ return resp.json()["choices"][0]["message"]+ message = {"content": "", "reasoning_content": ""}+ for line in resp.text.splitlines():+ if line.startswith("data: {"):+ for choice in json.loads(line[6:]).get("choices") or []:+ for key in message:+ message[key] += (choice.get("delta") or {}).get(key) or ""+ return message++[email protected]("stream", [False, True], ids = ["json", "sse"])+def test_route_resumes_an_mlx_thought_as_reasoning(monkeypatch, stream):+ seen = {}+ message = _sf_route_message(+ monkeypatch,+ _TEMPLATE_DEFAULT_OFF_TPL,+ # The MLX backend re-emits a bare opener ahead of the resumed tail.+ ["<think>tail", "<think>tail.</think>\n\nanswer"],+ is_mlx = True,+ seen = seen,+ messages = [+ {"role": "user", "content": "hi"},+ {"role": "assistant", "content": "", "reasoning_content": "The head, then the "},+ ],+ continue_final_message = True,+ # The turn is reasoning whatever the toggle now says.+ enable_thinking = False,+ stream = stream,+ )+ assert seen["continue_final_message"] is True+ assert message["reasoning_content"] == "tail."+ assert message["content"] == "\n\nanswer"+++def test_route_refuses_response_format_on_an_mlx_thought_resume(monkeypatch):+ error = _sf_route_message(+ monkeypatch,+ _TEMPLATE_DEFAULT_OFF_TPL,+ [],+ is_mlx = True,+ status = 400,+ messages = [+ {"role": "user", "content": "hi"},+ {"role": "assistant", "content": "", "reasoning_content": "The head"},+ ],+ continue_final_message = True,+ response_format = {"type": "json_object"},+ )+ assert error["param"] == "response_format"def test_route_returns_the_answer_as_content_when_the_template_closes_its_block(monkeypatch):diff --git a/studio/frontend/src/components/assistant-ui/thread.tsx b/studio/frontend/src/components/assistant-ui/thread.tsxindex 8067dd8fa9c..cba3c72a73f 100644--- a/studio/frontend/src/components/assistant-ui/thread.tsx+++ b/studio/frontend/src/components/assistant-ui/thread.tsx@@ -154,7 +154,7 @@ import {attachLibraryChatFiles,useLibraryChatHandoffStore,} from "@/features/library/chat-handoff-store";-import { isServedByLlamaCpp } from "@/features/model-picker";+import { resumesThought } from "@/features/model-picker";import { cancelResearchRun } from "@/features/chat/api/research-api";import {ingestResearchUpdate,@@ -7429,10 +7429,10 @@ function useContinuation() {const researchActive = useThreadResearchActive();const status = useAuiState(({ message }) => message.status);const metadata = useAuiState(({ message }) => message.metadata);- // Only llama-server can resume a thought.const thoughtResumable = useChatRuntimeStore((s) =>- isServedByLlamaCpp({+ resumesThought({loadedIsGguf: s.loadedIsGguf,+ loadedIsMlx: s.loadedIsMlx,activeGgufVariant: s.activeGgufVariant,activeNativePathToken: s.activeNativePathToken,checkpoint: s.params.checkpoint,diff --git a/studio/frontend/src/features/chat/api/chat-adapter.ts b/studio/frontend/src/features/chat/api/chat-adapter.tsindex c57b436dd62..aecec6f0fe9 100644--- a/studio/frontend/src/features/chat/api/chat-adapter.ts+++ b/studio/frontend/src/features/chat/api/chat-adapter.ts@@ -38,6 +38,7 @@ import {import { isHiddenModelId } from "@/features/hub/lib/hidden-models";import {isServedByLlamaCpp,+ resumesThought,isServedByMlx,loadedContextFields,resolveInitialConfig,@@ -2405,6 +2406,7 @@ type QueuedResolvedModelRuntime = {preserveThinking: boolean;loadedContextLength: number | null;loadedIsGguf: boolean | null;+ loadedIsMlx: boolean | null;loadedIsMultimodal: boolean;modelCapabilities: QueuedModelCapabilities | null;};@@ -2557,6 +2559,7 @@ function queuedResolvedModelFromStore(preserveThinking: state.preserveThinking,loadedContextLength: state.loadedContextLength,loadedIsGguf: state.loadedIsGguf,+ loadedIsMlx: state.loadedIsMlx,loadedIsMultimodal: state.loadedIsMultimodal,modelCapabilities: activeModel? {@@ -4195,6 +4198,7 @@ async function resolveQueuedEmptyLocalModel(abortSignal: AbortSignal): Promise<{preserveThinking: resolvePreserveThinkingOnLoad(status),loadedContextLength: loadedContextFields(status).loadedContextLength,loadedIsGguf: loadedContextFields(status).loadedIsGguf,+ loadedIsMlx: loadedContextFields(status).loadedIsMlx,loadedIsMultimodal: isMultimodalResponse(status),modelCapabilities: {isVision: status.is_vision ?? false,@@ -4427,6 +4431,7 @@ export function createOpenAIStreamAdapter(preserveThinking: queuedEmptyModelRuntime.preserveThinking,loadedContextLength: queuedEmptyModelRuntime.loadedContextLength,loadedIsGguf: queuedEmptyModelRuntime.loadedIsGguf,+ loadedIsMlx: queuedEmptyModelRuntime.loadedIsMlx,models: mergeQueuedModelCapabilities(base.models,queuedEmptyModelRuntime.checkpoint,@@ -4827,6 +4832,10 @@ export function createOpenAIStreamAdapter(queuedEmptyModelRuntime !== null? queuedEmptyModelRuntime.loadedIsGguf: liveRuntime.loadedIsGguf,+ loadedIsMlx:+ queuedEmptyModelRuntime !== null+ ? queuedEmptyModelRuntime.loadedIsMlx+ : liveRuntime.loadedIsMlx,loadedIsMultimodal:queuedEmptyModelRuntime?.loadedIsMultimodal ??liveRuntime.loadedIsMultimodal,@@ -5058,11 +5067,12 @@ export function createOpenAIStreamAdapter();}- // Carry reasoning only to llama-server, the one backend that can resume it.+ // Carry reasoning only to a backend that can resume it.const resumedThought =continuation &&- isServedByLlamaCpp({+ resumesThought({loadedIsGguf: runtime.loadedIsGguf,+ loadedIsMlx: runtime.loadedIsMlx,activeGgufVariant: runtime.activeGgufVariant,activeNativePathToken: runtime.activeNativePathToken,checkpoint: params.checkpoint,@@ -5072,7 +5082,7 @@ export function createOpenAIStreamAdapter(if (continuation && !continuation.partial && !resumedThought) {toast.error("This response cannot be resumed", {description:- "It stopped mid-thought, and only GGUF models can resume a thought. Use Retry instead.",+ "It stopped mid-thought, and only GGUF and MLX models can resume a thought. Use Retry instead.",});throw new Error("A response that stopped mid-thought cannot be resumed here.");}diff --git a/studio/frontend/src/features/chat/utils/continuation.ts b/studio/frontend/src/features/chat/utils/continuation.tsindex c5d3a2da7df..b471560c043 100644--- a/studio/frontend/src/features/chat/utils/continuation.ts+++ b/studio/frontend/src/features/chat/utils/continuation.ts@@ -359,7 +359,7 @@ export const CONTINUATION_RUN_CONFIG_KEY = "unslothContinuation";export type ContinuationRequest = {/** The partial answer exactly as rendered; empty when stopped mid-thought. */partial: string;- /** Carried only to llama-server. */+ /** Carried only to a backend that resumes a thought. */reasoning?: string;/** Seconds, so the resumed turn keeps its timer. */reasoningDuration?: number;diff --git a/studio/frontend/src/features/chat/utils/queued-chat-run-settings.ts b/studio/frontend/src/features/chat/utils/queued-chat-run-settings.tsindex 5df89a03c98..f686ba9a209 100644--- a/studio/frontend/src/features/chat/utils/queued-chat-run-settings.ts+++ b/studio/frontend/src/features/chat/utils/queued-chat-run-settings.ts@@ -40,6 +40,7 @@ const QUEUED_SETTING_KEYS = [// queued against. Without this an Ollama or native-path GGUF, which reports no quant and// no .gguf suffix, reads as non-GGUF and loses its compaction policy."loadedIsGguf",+ "loadedIsMlx","autoHealToolCalls","nudgeToolCalls","maxToolCallsPerMessage",diff --git a/studio/frontend/src/features/model-picker/index.ts b/studio/frontend/src/features/model-picker/index.tsindex 69f5f5248fd..5556630866d 100644--- a/studio/frontend/src/features/model-picker/index.ts+++ b/studio/frontend/src/features/model-picker/index.ts@@ -73,6 +73,7 @@ export {adoptCachedRepoConfig,adoptLegacyConfigKey,isServedByLlamaCpp,+ resumesThought,contextPinPatch,listPerModelConfigs,isServedByMlx,diff --git a/studio/frontend/src/features/model-picker/model-config/per-model-config.ts b/studio/frontend/src/features/model-picker/model-config/per-model-config.tsindex dc1497f0b7f..993249b7cf0 100644--- a/studio/frontend/src/features/model-picker/model-config/per-model-config.ts+++ b/studio/frontend/src/features/model-picker/model-config/per-model-config.ts@@ -190,6 +190,20 @@ export function isServedByLlamaCpp(x: {);}+/** Whether the backend can resume a reply stopped mid-thought: llama-server or reported MLX. */+export function resumesThought(x: {+ loadedIsGguf?: boolean | null;+ loadedIsMlx?: boolean | null;+ activeGgufVariant?: string | null;+ activeNativePathToken?: string | null;+ checkpoint?: string | null;+}): boolean {+ return (+ isServedByLlamaCpp(x) ||+ (!isExternalModelId(x.checkpoint) && x.loadedIsMlx === true)+ );+}+/** The store's record of the context window a load left behind.** A window counts when the backend that reported it sized one. MLX always does, so its `context_length` standsdiff --git a/studio/frontend/tests/mlx-context-helpers.test.ts b/studio/frontend/tests/mlx-context-helpers.test.tsindex a4bdd7a59b5..5473190e15e 100644--- a/studio/frontend/tests/mlx-context-helpers.test.ts+++ b/studio/frontend/tests/mlx-context-helpers.test.ts@@ -22,6 +22,7 @@ const {isServedByMlx,loadedContextFields,residentIsServedByMlx,+ resumesThought,} = await import("../src/features/model-picker/model-config/per-model-config.ts");@@ -136,6 +137,18 @@ test("a load that reported a non-GGUF backend outranks a stale variant", () => {assert.equal(isServedByLlamaCpp({ checkpoint: "external::openai/gpt-4" }), false);});+test("a thought resumes on llama-server and on a load MLX reports serving", () => {+ assert.equal(resumesThought({ loadedIsGguf: true }), true);+ assert.equal(resumesThought({ loadedIsGguf: false, loadedIsMlx: true }), true);+ // Transformers, an unloaded pick, and an external provider cannot.+ assert.equal(resumesThought({ loadedIsGguf: false, loadedIsMlx: false }), false);+ assert.equal(resumesThought({ loadedIsMlx: null }), false);+ assert.equal(+ resumesThought({ loadedIsMlx: true, checkpoint: "external::openai/gpt-4" }),+ false,+ );+});+test("MLX is a Mac non-GGUF load, and the reasons that rule it out", () => {assert.equal(isServedByMlx(false, "mac", null), true);assert.equal(isServedByMlx(true, "mac", null), false);@@ -376,3 +389,31 @@ test("a background load cannot leave the visible model reading another model's wassert.match(list, new RegExp(`"${key}"`), `${key} is not preserved`);}});++test("a queued run keeps its own model's MLX thought-resume verdict", async () => {+ const { snapshotQueuedChatRunSettings } = await import(+ "../src/features/chat/utils/queued-chat-run-settings.ts"+ );+ const resident = {+ params: { checkpoint: "mlx-community/Qwen3-0.6B-4bit" },+ activeGgufVariant: null,+ activeNativePathToken: null,+ loadedIsGguf: false,+ loadedIsMlx: true,+ };+ const queued = snapshotQueuedChatRunSettings(+ resident as unknown as Parameters<typeof snapshotQueuedChatRunSettings>[0],+ );+ const live = { ...resident, ...loadedContextFields(null) };+ const runtime = { ...live, ...queued };+ assert.equal(+ resumesThought({+ loadedIsGguf: runtime.loadedIsGguf,+ loadedIsMlx: runtime.loadedIsMlx,+ activeGgufVariant: runtime.activeGgufVariant,+ activeNativePathToken: runtime.activeNativePathToken,+ checkpoint: runtime.params.checkpoint,+ }),+ true,+ );+});