Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: resume a reply that stopped mid-thought on MLX (#12504)

Long Yixing95531ba1e149
diff --git a/studio/backend/core/inference/chat_template_helpers.py b/studio/backend/core/inference/chat_template_helpers.py
index 377f1073811..16bb5d270e3 100644
--- a/studio/backend/core/inference/chat_template_helpers.py
+++ b/studio/backend/core/inference/chat_template_helpers.py
@@ -2584,6 +2584,7 @@ def detect_think_prefill(
special_tokens = None,
*,
preserves_think_close: bool = False,
+ resumes_thought: bool = False,
) -> str:
"""Return the trailing open ``<think>`` prefill of a rendered prompt.
@@ -2606,6 +2607,9 @@ def detect_think_prefill(
``NativeToolTokenDecoder`` does so the parser can see a call rehearsed inside the block, and as
a path streaming the detokenizer's own text does. The special-token list then says nothing, and
skipping the opener is the same bug mirrored: a stray ``</think>``.
+
+ ``resumes_thought``: the prompt ends inside a resumed thought (the client's text), so only the
+ bare opener is returned.
"""
if not prompt:
return ""
@@ -2613,11 +2617,11 @@ def detect_think_prefill(
if open_idx == -1:
return ""
tail = prompt[open_idx:]
- if _THINK_CLOSE in tail or tail.strip() != _THINK_OPEN:
+ if _THINK_CLOSE in tail or (tail.strip() != _THINK_OPEN and not resumes_thought):
return ""
if not preserves_think_close and special_tokens and _THINK_CLOSE in set(special_tokens):
return ""
- return tail
+ return _THINK_OPEN if resumes_thought else tail
def _normalize_tool_call_arguments(messages: list) -> list:
@@ -3124,6 +3128,34 @@ def strip_open_reasoning_prefill(prefix: str) -> str:
return prefix[:open_at]
+class ThoughtUnresumableError(ValueError):
+ """A trailing thought this model cannot reopen; the message is client-safe."""
+
+ public = True
+ openai_param = "continue_final_message"
+
+ def __init__(self):
+ super().__init__("This model cannot resume a response that stopped mid-thought. Use Retry.")
+
+
+def template_resumes_thought(tokenizer, tools = None) -> bool:
+ """Whether a thought can be reopened as ``<think>`` text (not native reasoning channels)."""
+ if detect_reasoning_channel_markers(tokenizer, tools = tools) is not None:
+ return False
+ return any(
+ _THINK_OPEN in template for template in _selected_chat_template_strings(tokenizer, tools)
+ )
+
+
+def splice_resumed_thought(prefix: str, thought: str) -> str:
+ """Reopen *thought* on a generation prompt, cutting the template's own reasoning prefill (open
+ or empty closed block) as llama-server does."""
+ open_at = prefix.rfind(_THINK_OPEN)
+ if open_at != -1 and prefix[open_at + len(_THINK_OPEN) :].strip() in ("", _THINK_CLOSE):
+ prefix = prefix[:open_at]
+ return f"{prefix}{_THINK_OPEN}{thought}"
+
+
def render_prompt_with_boundary(
processor,
messages: list,
@@ -3232,6 +3264,11 @@ def apply_chat_template_for_generation(
# renders as an ordinary new turn.
_continue_text = trailing_assistant_text(messages) if continue_final_message else None
_continuing = bool(_continue_text)
+ _resumes_thought = (
+ continue_final_message and trailing_assistant_resume_kind(messages) == "reasoning_content"
+ )
+ if _resumes_thought and not template_resumes_thought(tokenizer, tools):
+ raise ThoughtUnresumableError()
_boundary_kwargs = (
{"add_generation_prompt": False, "continue_final_message": True}
if _continuing
@@ -3275,7 +3312,22 @@ def apply_chat_template_for_generation(
return f"{strip_open_reasoning_prefill(prefix)}{partial}"
raise TypeError("no attempt rendered the continuation prefix")
+ def _render_thought_continuation(msgs: list) -> str:
+ # Templates render a final thought closed, so there is no boundary to cut at.
+ for kwargs in attempts:
+ swept = _swept_for(kwargs, msgs)
+ try:
+ prefix = tokenizer.apply_chat_template(
+ swept[:-1], tokenize = False, add_generation_prompt = True, **kwargs
+ )
+ except TypeError:
+ continue
+ return splice_resumed_thought(prefix, swept[-1]["reasoning_content"])
+ raise TypeError("no attempt rendered the thought continuation prefix")
+
def _render_with_fallback(msgs: list) -> str:
+ if _resumes_thought:
+ return _render_thought_continuation(msgs)
try:
return _render(msgs)
except TypeError:
diff --git a/studio/backend/core/inference/mlx_inference.py b/studio/backend/core/inference/mlx_inference.py
index a78b10ee40d..cbfa407f110 100644
--- a/studio/backend/core/inference/mlx_inference.py
+++ b/studio/backend/core/inference/mlx_inference.py
@@ -44,6 +44,8 @@ from core.inference.chat_template_helpers import (
normalize_reasoning_snapshots,
prompt_opens_reasoning_channel,
strip_open_reasoning_prefill,
+ ThoughtUnresumableError,
+ trailing_assistant_resume_kind,
trailing_assistant_text,
vlm_prompt_issue as _vlm_prompt_issue,
)
@@ -1090,6 +1092,23 @@ def _ascii_registry_key(value):
return value.lower()
+def _resumes_thought(messages, continue_final_message) -> bool:
+ return bool(continue_final_message) and (
+ trailing_assistant_resume_kind(messages) == "reasoning_content"
+ )
+
+
+def _think_prefix(prompt, special_tokens, messages, continue_final_message, **kwargs) -> str:
+ """``detect_think_prefill``; refuses a resumed thought whose ``</think>`` decoding would strip."""
+ from core.inference.chat_template_helpers import detect_think_prefill
+
+ resumes = _resumes_thought(messages, continue_final_message)
+ prefix = detect_think_prefill(prompt, special_tokens, resumes_thought = resumes, **kwargs)
+ if resumes and not prefix:
+ raise ThoughtUnresumableError()
+ return prefix
+
+
def _render_registered_vlm_prompt(
processor,
model,
@@ -1131,6 +1150,9 @@ def _render_registered_vlm_prompt(
config = dict(config) if isinstance(config, dict) else dict(config.__dict__)
config["model_type"] = canonical
+ # mlx-vlm's formatters carry no reasoning opener to reopen a thought on.
+ if _resumes_thought(messages, continue_final_message):
+ return None
# Recovery path: sweeps the caller's original list rather than reusing a copy (#7066).
swept = neutralize_control_markup_in_messages(messages, None, markup_for_tokenizer(processor))
partial = trailing_assistant_text(swept) if continue_final_message else None
@@ -5172,8 +5194,6 @@ class MLXInferenceBackend:
from mlx_lm import stream_generate
from mlx_lm.sample_utils import make_sampler
- from core.inference.chat_template_helpers import detect_think_prefill
-
render_result = self._render_text_prompt(
messages,
tools = tools,
@@ -5196,9 +5216,11 @@ class MLXInferenceBackend:
) and decoder_preserves_token(
self._tokenizer, "</think>", reasoning_control_tokens(reasoning_channel_markers)
)
- think_prefix = detect_think_prefill(
+ think_prefix = _think_prefix(
prompt,
getattr(self._tokenizer, "all_special_tokens", None),
+ messages,
+ continue_final_message,
preserves_think_close = think_close_survives,
)
constraint = _build_grammar_constraint(
@@ -5479,8 +5501,6 @@ class MLXInferenceBackend:
) -> "_TextRowPlan":
from mlx_lm.sample_utils import make_sampler
- from core.inference.chat_template_helpers import detect_think_prefill
-
render_result = self._render_text_prompt(
messages,
tools = tools,
@@ -5493,9 +5513,11 @@ class MLXInferenceBackend:
reasoning_channel_markers = render_result.reasoning_channel_markers
_resumed_partial = bool(continue_final_message and trailing_assistant_text(messages))
- think_prefix = detect_think_prefill(
+ think_prefix = _think_prefix(
prompt,
getattr(self._tokenizer, "all_special_tokens", None),
+ messages,
+ continue_final_message,
preserves_think_close = (
bool(tools) or tool_protocol_active or reasoning_channel_markers is not None
)
@@ -5755,10 +5777,11 @@ class MLXInferenceBackend:
continue_final_message = continue_final_message,
)
- from core.inference.chat_template_helpers import detect_think_prefill
-
- think_prefix = detect_think_prefill(
- prompt, getattr(chat_target, "all_special_tokens", None)
+ think_prefix = _think_prefix(
+ prompt,
+ getattr(chat_target, "all_special_tokens", None),
+ messages,
+ continue_final_message,
)
normalizer = (
make_reasoning_normalizer(
@@ -5872,9 +5895,11 @@ class MLXInferenceBackend:
from core.inference.chat_template_helpers import detect_think_prefill
# Detected once: the decoder keeps the delimiters the normalizer below consumes.
- prefill = detect_think_prefill(
+ prefill = _think_prefix(
prompt,
getattr(chat_target, "all_special_tokens", None),
+ messages,
+ continue_final_message,
# The same activation the decoder below uses: in unrestricted mode ``tools`` is
# empty while the protocol is live, so ``bool(tools)`` said the closer would be
# stripped, the opener was suppressed, and the stream ran on to an orphan
diff --git a/studio/backend/core/inference/safetensors_agentic.py b/studio/backend/core/inference/safetensors_agentic.py
index ffc2b6b3aa5..b5b894ba629 100644
--- a/studio/backend/core/inference/safetensors_agentic.py
+++ b/studio/backend/core/inference/safetensors_agentic.py
@@ -73,7 +73,7 @@ from core.inference.tool_loop_controller import (
)
from core.inference.chat_template_helpers import (
append_assistant_turn,
- trailing_assistant_text,
+ trailing_assistant_resume_kind,
)
from core.inference.passthrough_healing import nudge_enabled
from state.tool_approvals import (
@@ -357,6 +357,26 @@ def _status_for_tool(tool_name: str, arguments: dict) -> str:
return status_for_tool(tool_name, arguments)
+def _append_raw_turn(conversation: list, assistant_msg: dict, *, continue_final_message: bool):
+ """``append_assistant_turn`` for raw-text turns: a resumed thought is replayed whole after the
+ re-emitted ``<think>``; text without that opener drops it."""
+ if not (
+ continue_final_message
+ and trailing_assistant_resume_kind(conversation) == "reasoning_content"
+ and isinstance(assistant_msg.get("content"), str)
+ ):
+ append_assistant_turn(
+ conversation, assistant_msg, continue_final_message = continue_final_message
+ )
+ return
+ thought = conversation[-1]["reasoning_content"]
+ merged = {**conversation[-1], **assistant_msg}
+ merged.pop("reasoning_content", None)
+ if merged["content"].startswith("<think>"):
+ merged["content"] = f"<think>{thought}{merged['content'][len('<think>'):]}"
+ conversation[-1] = merged
+
+
def _reprompt_intent_text(
text: str,
*,
@@ -677,7 +697,7 @@ def run_safetensors_tool_loop(
# plus its result, moving the boundary so the model opens a fresh answer.
_skip_autoinject = (
confirm_tool_calls and not bypass_permissions and permission_mode not in ("auto", "off")
- ) or bool(continue_final_message and trailing_assistant_text(conversation))
+ ) or bool(continue_final_message and trailing_assistant_resume_kind(conversation) is not None)
_auto = None if _skip_autoinject else build_rag_autoinject(conversation, rag_scope)
if _auto:
for _ev in _auto["events"]:
@@ -1241,7 +1261,7 @@ def run_safetensors_tool_loop(
)
# Merges into a resumed partial: the nudge that follows is a user
# turn, so a second assistant turn breaks alternation.
- append_assistant_turn(
+ _append_raw_turn(
conversation,
{"role": "assistant", "content": intent_text},
continue_final_message = continue_final_message,
@@ -1414,7 +1434,7 @@ def run_safetensors_tool_loop(
if not decision.should_execute:
if content_text and not assistant_appended:
- append_assistant_turn(
+ _append_raw_turn(
conversation,
assistant_msg,
continue_final_message = continue_final_message,
@@ -1444,7 +1464,7 @@ def run_safetensors_tool_loop(
assistant_msg["tool_calls"] = [decision.as_assistant_tool_call()]
# Merges into a resumed partial, so a continued turn that calls a tool
# stays one assistant message.
- append_assistant_turn(
+ _append_raw_turn(
conversation,
assistant_msg,
continue_final_message = continue_final_message,
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index b5886c9725c..fc0aa5666bc 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -445,7 +445,7 @@ def _continue_final_message(payload, *, thought: bool = False) -> bool:
Nothing resumable (no assistant turn, or one holding tool calls) degrades to an
ordinary new turn rather than erroring. ``thought`` also resumes a turn holding only
- ``reasoning_content``, which only llama-server can continue inside its reasoning block.
+ ``reasoning_content``, which only llama-server and MLX reopen inside the reasoning block.
"""
if not getattr(payload, "continue_final_message", None):
return False
@@ -27038,7 +27038,7 @@ async def produce_openai_chat_completions(
"call: guided decoding admits only the document, so no tool would be "
"called. Use tool_choice 'auto' or 'none'.",
)
- if _continue_final_message(payload):
+ if _continue_final_message(payload, thought = True):
_raise_unsupported_openai_parameter(
"response_format",
"response_format cannot resume a partial assistant message: guided "
@@ -29745,8 +29745,9 @@ async def produce_openai_chat_completions(
# A continued turn renders no generation prompt, so nothing is prefilled and the
# resumed text is the visible answer. Only the first turn continues; later tool-loop
- # turns render a fresh generation prompt and prefill as usual.
- _sf_continue = _continue_final_message(payload)
+ # turns render a fresh generation prompt and prefill as usual. MLX also resumes a thought.
+ _sf_continue = _continue_final_message(payload, thought = bool(_sf_model_info.get("is_mlx")))
+ _sf_resumes_thought = _sf_continue and not _continue_final_message(payload)
_sf_continued_turn = [_sf_continue]
_sf_single_block = _response_format_constrains_decoding(payload)
@@ -29755,7 +29756,7 @@ async def produce_openai_chat_completions(
prefilled = _sf_reasoning_prefilled
if _sf_continued_turn[0]:
_sf_continued_turn[0] = False
- prefilled = False
+ prefilled = _sf_resumes_thought
return _ResponsesReasoningExtractor(
parse_think_markers = _sf_parse_think,
reasoning_prefilled = prefilled,
@@ -29895,9 +29896,7 @@ async def produce_openai_chat_completions(
# Strip stale tool-call XML from prior assistant turns. The continuation target
# keeps its exact whitespace (see the GGUF path).
- _sf_continue_target = (
- chat_messages[-1] if _continue_final_message(payload) and chat_messages else None
- )
+ _sf_continue_target = chat_messages[-1] if _sf_continue and chat_messages else None
_sf_chat_messages = []
for _msg in chat_messages:
if _msg.get("role") == "assistant" and isinstance(_msg.get("content"), str):
@@ -29984,7 +29983,7 @@ async def produce_openai_chat_completions(
enable_thinking = payload.enable_thinking,
reasoning_effort = payload.reasoning_effort,
preserve_thinking = payload.preserve_thinking,
- continue_final_message = _continue_final_message(payload),
+ continue_final_message = _sf_continue,
auto_heal_tool_calls = _sf_auto_heal_tool_calls,
nudge_tool_calls = payload.nudge_tool_calls,
max_tool_iterations = _sf_tool_budget,
@@ -30258,7 +30257,7 @@ async def produce_openai_chat_completions(
# boundary and pin the mode of the turn that text came from (the same events
# reset the streaming path's extractor).
continued = _sf_continue
- prefilled = _sf_reasoning_prefilled and not continued
+ prefilled = _sf_resumes_thought if continued else _sf_reasoning_prefilled
gen = sf_generate_with_tools()
for event in gen:
if cancel_event.is_set():
@@ -30279,7 +30278,7 @@ async def produce_openai_chat_completions(
auto_heal_tool_calls = _sf_auto_heal_tool_calls,
enabled_tool_names = _sf_display_tool_names,
)
- prefilled = _sf_reasoning_prefilled and not continued
+ prefilled = _sf_resumes_thought if continued else _sf_reasoning_prefilled
elif _event_type == "tool_start" or (
_event_type == "status" and not event.get("text")
):
@@ -30406,7 +30405,7 @@ async def produce_openai_chat_completions(
gen_kwargs["reasoning_effort"] = payload.reasoning_effort
if payload.preserve_thinking is not None:
gen_kwargs["preserve_thinking"] = payload.preserve_thinking
- if _continue_final_message(payload):
+ if _sf_continue:
gen_kwargs["continue_final_message"] = True
# Client-tool passthrough (safetensors + MLX): render client tools, generate one turn,
@@ -31024,7 +31023,9 @@ async def produce_openai_chat_completions(
_reasoning_text, _visible_text = _extract_responses_reasoning(
full_text,
parse_think_markers = _sf_parse_think,
- reasoning_prefilled = _sf_reasoning_prefilled and not _sf_continue,
+ reasoning_prefilled = (
+ _sf_resumes_thought if _sf_continue else _sf_reasoning_prefilled
+ ),
single_block = _sf_single_block,
)
# Client-tool passthrough: promote text-form calls; opt-in single
diff --git a/studio/backend/tests/test_chat_template_continuation.py b/studio/backend/tests/test_chat_template_continuation.py
index 54dca4253dc..d1dfacb9e6d 100644
--- a/studio/backend/tests/test_chat_template_continuation.py
+++ b/studio/backend/tests/test_chat_template_continuation.py
@@ -11,6 +11,7 @@ assistant turn can be resumed, so anything else renders normally instead of rais
from __future__ import annotations
+import copy
import importlib
import sys
import types
@@ -25,8 +26,10 @@ if str(_BACKEND) not in sys.path:
from core.inference.chat_template_helpers import ( # noqa: E402
append_assistant_turn,
apply_chat_template_for_generation,
+ detect_think_prefill,
last_user_text,
render_prompt_with_boundary,
+ ThoughtUnresumableError,
trailing_assistant_text,
)
@@ -835,6 +838,9 @@ def _sf_completion(
monkeypatch,
events,
stats = None,
+ messages = None,
+ is_mlx = False,
+ seen = None,
**body,
):
"""POST a non-streaming safetensors tool-loop chat and return the assistant message."""
@@ -851,9 +857,11 @@ def _sf_completion(
class _Safetensors:
active_model_name = "qwen3"
- models = {"qwen3": {"chat_template_info": {"template": _THINK_TEMPLATE}}}
+ models = {"qwen3": {"chat_template_info": {"template": _THINK_TEMPLATE}, "is_mlx": is_mlx}}
def generate_chat_completion_with_tools(self, **kwargs):
+ if seen is not None:
+ seen.update(kwargs)
# The worker fills this on gen_done; the route reads it for usage and
# for finish_reason "length".
if stats is not None:
@@ -879,7 +887,8 @@ def _sf_completion(
resp = TestClient(app).post(
"/v1/chat/completions",
json = {
- "messages": [
+ "messages": messages
+ or [
{"role": "user", "content": "weather?"},
{"role": "assistant", "content": "Let me check the "},
],
@@ -912,6 +921,150 @@ def test_a_resumed_turn_without_a_tool_keeps_the_partial_visible(monkeypatch):
assert not message["reasoning_content"]
+_CUT_THOUGHT = [
+ {"role": "user", "content": "weather?"},
+ {"role": "assistant", "content": "", "reasoning_content": "\nThe user wants the "},
+]
+# The MLX backend re-emits a bare opener ahead of a resumed thought's tail.
+_RESUMED_THOUGHT_EVENTS = [
+ {"type": "content", "text": "<think>forecast.</think>\n\nIt is 21C."},
+ {"type": "status", "text": ""},
+]
+
+
+def test_mlx_resumes_a_thought_and_reports_its_tail_as_reasoning(monkeypatch):
+ seen = {}
+ message = _sf_completion(
+ monkeypatch,
+ _RESUMED_THOUGHT_EVENTS,
+ messages = _CUT_THOUGHT,
+ is_mlx = True,
+ seen = seen,
+ # The turn is reasoning whatever the toggle now says.
+ enable_thinking = False,
+ )["message"]
+ assert seen["continue_final_message"] is True
+ assert message["reasoning_content"] == "forecast."
+ assert message["content"] == "\n\nIt is 21C."
+
+
+def test_transformers_answers_after_a_thought_instead_of_resuming_it(monkeypatch):
+ seen = {}
+ _sf_completion(monkeypatch, _RESUMED_THOUGHT_EVENTS, messages = _CUT_THOUGHT, seen = seen)
+ assert seen["continue_final_message"] is False
+
+
+class _ThinkTokenizer(_ChatMLTokenizer):
+ chat_template = "{{ '<think>' }}"
+
+ def __init__(self, prefill):
+ self.prefill = prefill
+
+ def apply_chat_template(
+ self,
+ messages,
+ *,
+ add_generation_prompt = True,
+ **kw,
+ ):
+ out = super().apply_chat_template(
+ messages, add_generation_prompt = add_generation_prompt, **kw
+ )
+ return out + self.prefill if add_generation_prompt else out
+
+
+ "prefill", ["", "<think>\n", "<think>\n\n</think>\n\n"], ids = ["none", "open", "closed"]
+)
+def test_a_thought_resumes_inside_its_reasoning_block(prefill):
+ thought = "\nTry 2 then"
+ prompt = apply_chat_template_for_generation(
+ _ThinkTokenizer(prefill),
+ [
+ {"role": "user", "content": "q"},
+ {"role": "assistant", "content": "", "reasoning_content": thought},
+ ],
+ continue_final_message = True,
+ )
+ assert prompt == "<|im_start|>user\nq<|im_end|>\n<|im_start|>assistant\n<think>\nTry 2 then"
+ assert detect_think_prefill(prompt, resumes_thought = True) == "<think>"
+ assert detect_think_prefill(prompt) == ""
+
+
+def test_a_think_in_history_is_not_taken_for_the_prefill():
+ history = [
+ {"role": "user", "content": "q"},
+ {"role": "assistant", "content": "<think>a</think>b"},
+ {"role": "user", "content": "r"},
+ ]
+ prompt = apply_chat_template_for_generation(
+ _ThinkTokenizer(""),
+ history + [{"role": "assistant", "content": "", "reasoning_content": "c"}],
+ continue_final_message = True,
+ )
+ assert prompt.endswith("<|im_start|>user\nr<|im_end|>\n<|im_start|>assistant\n<think>c")
+ assert "<think>a</think>b" in prompt
+
+
+class _GemmaTokenizer(_ChatMLTokenizer):
+ chat_template = "{{ '<think>' }}<|channel>thought\n"
+
+
+ "tokenizer", [_ChatMLTokenizer, _GemmaTokenizer], ids = ["no-think", "native"]
+)
+def test_a_template_without_a_think_opener_refuses_to_resume_a_thought(tokenizer):
+ with pytest.raises(ThoughtUnresumableError) as refused:
+ apply_chat_template_for_generation(
+ tokenizer(),
+ [
+ {"role": "user", "content": "q"},
+ {"role": "assistant", "content": "", "reasoning_content": "x"},
+ ],
+ continue_final_message = True,
+ )
+ # The worker forwards only a public refusal, and the route answers its parameter with a 400.
+ assert refused.value.public and refused.value.openai_param == "continue_final_message"
+
+
+def test_mlx_refuses_a_thought_its_decoder_would_never_close():
+ from core.inference.mlx_inference import _think_prefix
+
+ prompt = "<|im_start|>assistant\n<think>\nThe user wants the "
+ closer = ["</think>"]
+ assert (
+ _think_prefix(prompt, closer, _CUT_THOUGHT, True, preserves_think_close = True) == "<think>"
+ )
+ with pytest.raises(ThoughtUnresumableError):
+ _think_prefix(prompt, closer, _CUT_THOUGHT, True)
+ assert _think_prefix(prompt, closer, _CUT_THOUGHT, False) == ""
+
+
+def test_the_tool_loop_replays_a_resumed_thought_whole():
+ from core.inference.safetensors_agentic import _append_raw_turn
+
+ conversation = copy.deepcopy(_CUT_THOUGHT)
+ _append_raw_turn(
+ conversation,
+ {"role": "assistant", "content": "<think>forecast.</think>Checking.", "tool_calls": [{}]},
+ continue_final_message = True,
+ )
+ assert conversation[1:] == [
+ {
+ "role": "assistant",
+ "content": "<think>\nThe user wants the forecast.</think>Checking.",
+ "tool_calls": [{}],
+ }
+ ]
+ conversation = copy.deepcopy(_CUT_THOUGHT)
+ _append_raw_turn(
+ conversation,
+ {"role": "assistant", "content": "I will search."},
+ continue_final_message = True,
+ )
+ assert conversation[1:] == [{"role": "assistant", "content": "I will search."}]
+
+
@pytest.mark.parametrize("gguf", [False, True], ids = ["safetensors", "gguf"])
def test_a_tts_model_refuses_a_continuation(monkeypatch, gguf):
"""Both TTS branches re-speak the newest user text before any continuation handling,
diff --git a/studio/backend/tests/test_safetensors_reasoning_stream.py b/studio/backend/tests/test_safetensors_reasoning_stream.py
index 58b63241697..1941a5e0514 100644
--- a/studio/backend/tests/test_safetensors_reasoning_stream.py
+++ b/studio/backend/tests/test_safetensors_reasoning_stream.py
@@ -14,6 +14,7 @@ loop against synthetic events, and covers the split through the route itself.
from __future__ import annotations
+import json
import sys
from pathlib import Path
@@ -633,9 +634,11 @@ def _sf_route_message(
is_mlx = False,
features = None,
model_info = None,
+ seen = None,
+ status = 200,
**body,
):
- """POST a non-streaming safetensors chat completion and return the assistant message."""
+ """POST a safetensors chat completion and return the assistant message, joined if streamed."""
from fastapi import FastAPI
from fastapi.testclient import TestClient
@@ -659,6 +662,8 @@ def _sf_route_message(
}
def generate_chat_response(self, **kwargs):
+ if seen is not None:
+ seen.update(kwargs)
yield from snapshots
def reset_generation_state(self, *_args):
@@ -687,8 +692,59 @@ def _sf_route_message(
**body,
},
)
- assert resp.status_code == 200, resp.text
- return resp.json()["choices"][0]["message"]
+ assert resp.status_code == status, resp.text
+ if status != 200:
+ return resp.json()["error"]
+ if not body.get("stream"):
+ return resp.json()["choices"][0]["message"]
+ message = {"content": "", "reasoning_content": ""}
+ for line in resp.text.splitlines():
+ if line.startswith("data: {"):
+ for choice in json.loads(line[6:]).get("choices") or []:
+ for key in message:
+ message[key] += (choice.get("delta") or {}).get(key) or ""
+ return message
+
+
[email protected]("stream", [False, True], ids = ["json", "sse"])
+def test_route_resumes_an_mlx_thought_as_reasoning(monkeypatch, stream):
+ seen = {}
+ message = _sf_route_message(
+ monkeypatch,
+ _TEMPLATE_DEFAULT_OFF_TPL,
+ # The MLX backend re-emits a bare opener ahead of the resumed tail.
+ ["<think>tail", "<think>tail.</think>\n\nanswer"],
+ is_mlx = True,
+ seen = seen,
+ messages = [
+ {"role": "user", "content": "hi"},
+ {"role": "assistant", "content": "", "reasoning_content": "The head, then the "},
+ ],
+ continue_final_message = True,
+ # The turn is reasoning whatever the toggle now says.
+ enable_thinking = False,
+ stream = stream,
+ )
+ assert seen["continue_final_message"] is True
+ assert message["reasoning_content"] == "tail."
+ assert message["content"] == "\n\nanswer"
+
+
+def test_route_refuses_response_format_on_an_mlx_thought_resume(monkeypatch):
+ error = _sf_route_message(
+ monkeypatch,
+ _TEMPLATE_DEFAULT_OFF_TPL,
+ [],
+ is_mlx = True,
+ status = 400,
+ messages = [
+ {"role": "user", "content": "hi"},
+ {"role": "assistant", "content": "", "reasoning_content": "The head"},
+ ],
+ continue_final_message = True,
+ response_format = {"type": "json_object"},
+ )
+ assert error["param"] == "response_format"
def test_route_returns_the_answer_as_content_when_the_template_closes_its_block(monkeypatch):
diff --git a/studio/frontend/src/components/assistant-ui/thread.tsx b/studio/frontend/src/components/assistant-ui/thread.tsx
index 8067dd8fa9c..cba3c72a73f 100644
--- a/studio/frontend/src/components/assistant-ui/thread.tsx
+++ b/studio/frontend/src/components/assistant-ui/thread.tsx
@@ -154,7 +154,7 @@ import {
attachLibraryChatFiles,
useLibraryChatHandoffStore,
} from "@/features/library/chat-handoff-store";
-import { isServedByLlamaCpp } from "@/features/model-picker";
+import { resumesThought } from "@/features/model-picker";
import { cancelResearchRun } from "@/features/chat/api/research-api";
import {
ingestResearchUpdate,
@@ -7429,10 +7429,10 @@ function useContinuation() {
const researchActive = useThreadResearchActive();
const status = useAuiState(({ message }) => message.status);
const metadata = useAuiState(({ message }) => message.metadata);
- // Only llama-server can resume a thought.
const thoughtResumable = useChatRuntimeStore((s) =>
- isServedByLlamaCpp({
+ resumesThought({
loadedIsGguf: s.loadedIsGguf,
+ loadedIsMlx: s.loadedIsMlx,
activeGgufVariant: s.activeGgufVariant,
activeNativePathToken: s.activeNativePathToken,
checkpoint: s.params.checkpoint,
diff --git a/studio/frontend/src/features/chat/api/chat-adapter.ts b/studio/frontend/src/features/chat/api/chat-adapter.ts
index c57b436dd62..aecec6f0fe9 100644
--- a/studio/frontend/src/features/chat/api/chat-adapter.ts
+++ b/studio/frontend/src/features/chat/api/chat-adapter.ts
@@ -38,6 +38,7 @@ import {
import { isHiddenModelId } from "@/features/hub/lib/hidden-models";
import {
isServedByLlamaCpp,
+ resumesThought,
isServedByMlx,
loadedContextFields,
resolveInitialConfig,
@@ -2405,6 +2406,7 @@ type QueuedResolvedModelRuntime = {
preserveThinking: boolean;
loadedContextLength: number | null;
loadedIsGguf: boolean | null;
+ loadedIsMlx: boolean | null;
loadedIsMultimodal: boolean;
modelCapabilities: QueuedModelCapabilities | null;
};
@@ -2557,6 +2559,7 @@ function queuedResolvedModelFromStore(
preserveThinking: state.preserveThinking,
loadedContextLength: state.loadedContextLength,
loadedIsGguf: state.loadedIsGguf,
+ loadedIsMlx: state.loadedIsMlx,
loadedIsMultimodal: state.loadedIsMultimodal,
modelCapabilities: activeModel
? {
@@ -4195,6 +4198,7 @@ async function resolveQueuedEmptyLocalModel(abortSignal: AbortSignal): Promise<{
preserveThinking: resolvePreserveThinkingOnLoad(status),
loadedContextLength: loadedContextFields(status).loadedContextLength,
loadedIsGguf: loadedContextFields(status).loadedIsGguf,
+ loadedIsMlx: loadedContextFields(status).loadedIsMlx,
loadedIsMultimodal: isMultimodalResponse(status),
modelCapabilities: {
isVision: status.is_vision ?? false,
@@ -4427,6 +4431,7 @@ export function createOpenAIStreamAdapter(
preserveThinking: queuedEmptyModelRuntime.preserveThinking,
loadedContextLength: queuedEmptyModelRuntime.loadedContextLength,
loadedIsGguf: queuedEmptyModelRuntime.loadedIsGguf,
+ loadedIsMlx: queuedEmptyModelRuntime.loadedIsMlx,
models: mergeQueuedModelCapabilities(
base.models,
queuedEmptyModelRuntime.checkpoint,
@@ -4827,6 +4832,10 @@ export function createOpenAIStreamAdapter(
queuedEmptyModelRuntime !== null
? queuedEmptyModelRuntime.loadedIsGguf
: liveRuntime.loadedIsGguf,
+ loadedIsMlx:
+ queuedEmptyModelRuntime !== null
+ ? queuedEmptyModelRuntime.loadedIsMlx
+ : liveRuntime.loadedIsMlx,
loadedIsMultimodal:
queuedEmptyModelRuntime?.loadedIsMultimodal ??
liveRuntime.loadedIsMultimodal,
@@ -5058,11 +5067,12 @@ export function createOpenAIStreamAdapter(
);
}
- // Carry reasoning only to llama-server, the one backend that can resume it.
+ // Carry reasoning only to a backend that can resume it.
const resumedThought =
continuation &&
- isServedByLlamaCpp({
+ resumesThought({
loadedIsGguf: runtime.loadedIsGguf,
+ loadedIsMlx: runtime.loadedIsMlx,
activeGgufVariant: runtime.activeGgufVariant,
activeNativePathToken: runtime.activeNativePathToken,
checkpoint: params.checkpoint,
@@ -5072,7 +5082,7 @@ export function createOpenAIStreamAdapter(
if (continuation && !continuation.partial && !resumedThought) {
toast.error("This response cannot be resumed", {
description:
- "It stopped mid-thought, and only GGUF models can resume a thought. Use Retry instead.",
+ "It stopped mid-thought, and only GGUF and MLX models can resume a thought. Use Retry instead.",
});
throw new Error("A response that stopped mid-thought cannot be resumed here.");
}
diff --git a/studio/frontend/src/features/chat/utils/continuation.ts b/studio/frontend/src/features/chat/utils/continuation.ts
index c5d3a2da7df..b471560c043 100644
--- a/studio/frontend/src/features/chat/utils/continuation.ts
+++ b/studio/frontend/src/features/chat/utils/continuation.ts
@@ -359,7 +359,7 @@ export const CONTINUATION_RUN_CONFIG_KEY = "unslothContinuation";
export type ContinuationRequest = {
/** The partial answer exactly as rendered; empty when stopped mid-thought. */
partial: string;
- /** Carried only to llama-server. */
+ /** Carried only to a backend that resumes a thought. */
reasoning?: string;
/** Seconds, so the resumed turn keeps its timer. */
reasoningDuration?: number;
diff --git a/studio/frontend/src/features/chat/utils/queued-chat-run-settings.ts b/studio/frontend/src/features/chat/utils/queued-chat-run-settings.ts
index 5df89a03c98..f686ba9a209 100644
--- a/studio/frontend/src/features/chat/utils/queued-chat-run-settings.ts
+++ b/studio/frontend/src/features/chat/utils/queued-chat-run-settings.ts
@@ -40,6 +40,7 @@ const QUEUED_SETTING_KEYS = [
// queued against. Without this an Ollama or native-path GGUF, which reports no quant and
// no .gguf suffix, reads as non-GGUF and loses its compaction policy.
"loadedIsGguf",
+ "loadedIsMlx",
"autoHealToolCalls",
"nudgeToolCalls",
"maxToolCallsPerMessage",
diff --git a/studio/frontend/src/features/model-picker/index.ts b/studio/frontend/src/features/model-picker/index.ts
index 69f5f5248fd..5556630866d 100644
--- a/studio/frontend/src/features/model-picker/index.ts
+++ b/studio/frontend/src/features/model-picker/index.ts
@@ -73,6 +73,7 @@ export {
adoptCachedRepoConfig,
adoptLegacyConfigKey,
isServedByLlamaCpp,
+ resumesThought,
contextPinPatch,
listPerModelConfigs,
isServedByMlx,
diff --git a/studio/frontend/src/features/model-picker/model-config/per-model-config.ts b/studio/frontend/src/features/model-picker/model-config/per-model-config.ts
index dc1497f0b7f..993249b7cf0 100644
--- a/studio/frontend/src/features/model-picker/model-config/per-model-config.ts
+++ b/studio/frontend/src/features/model-picker/model-config/per-model-config.ts
@@ -190,6 +190,20 @@ export function isServedByLlamaCpp(x: {
);
}
+/** Whether the backend can resume a reply stopped mid-thought: llama-server or reported MLX. */
+export function resumesThought(x: {
+ loadedIsGguf?: boolean | null;
+ loadedIsMlx?: boolean | null;
+ activeGgufVariant?: string | null;
+ activeNativePathToken?: string | null;
+ checkpoint?: string | null;
+}): boolean {
+ return (
+ isServedByLlamaCpp(x) ||
+ (!isExternalModelId(x.checkpoint) && x.loadedIsMlx === true)
+ );
+}
+
/** The store's record of the context window a load left behind.
*
* A window counts when the backend that reported it sized one. MLX always does, so its `context_length` stands
diff --git a/studio/frontend/tests/mlx-context-helpers.test.ts b/studio/frontend/tests/mlx-context-helpers.test.ts
index a4bdd7a59b5..5473190e15e 100644
--- a/studio/frontend/tests/mlx-context-helpers.test.ts
+++ b/studio/frontend/tests/mlx-context-helpers.test.ts
@@ -22,6 +22,7 @@ const {
isServedByMlx,
loadedContextFields,
residentIsServedByMlx,
+ resumesThought,
} = await import(
"../src/features/model-picker/model-config/per-model-config.ts"
);
@@ -136,6 +137,18 @@ test("a load that reported a non-GGUF backend outranks a stale variant", () => {
assert.equal(isServedByLlamaCpp({ checkpoint: "external::openai/gpt-4" }), false);
});
+test("a thought resumes on llama-server and on a load MLX reports serving", () => {
+ assert.equal(resumesThought({ loadedIsGguf: true }), true);
+ assert.equal(resumesThought({ loadedIsGguf: false, loadedIsMlx: true }), true);
+ // Transformers, an unloaded pick, and an external provider cannot.
+ assert.equal(resumesThought({ loadedIsGguf: false, loadedIsMlx: false }), false);
+ assert.equal(resumesThought({ loadedIsMlx: null }), false);
+ assert.equal(
+ resumesThought({ loadedIsMlx: true, checkpoint: "external::openai/gpt-4" }),
+ false,
+ );
+});
+
test("MLX is a Mac non-GGUF load, and the reasons that rule it out", () => {
assert.equal(isServedByMlx(false, "mac", null), true);
assert.equal(isServedByMlx(true, "mac", null), false);
@@ -376,3 +389,31 @@ test("a background load cannot leave the visible model reading another model's w
assert.match(list, new RegExp(`"${key}"`), `${key} is not preserved`);
}
});
+
+test("a queued run keeps its own model's MLX thought-resume verdict", async () => {
+ const { snapshotQueuedChatRunSettings } = await import(
+ "../src/features/chat/utils/queued-chat-run-settings.ts"
+ );
+ const resident = {
+ params: { checkpoint: "mlx-community/Qwen3-0.6B-4bit" },
+ activeGgufVariant: null,
+ activeNativePathToken: null,
+ loadedIsGguf: false,
+ loadedIsMlx: true,
+ };
+ const queued = snapshotQueuedChatRunSettings(
+ resident as unknown as Parameters<typeof snapshotQueuedChatRunSettings>[0],
+ );
+ const live = { ...resident, ...loadedContextFields(null) };
+ const runtime = { ...live, ...queued };
+ assert.equal(
+ resumesThought({
+ loadedIsGguf: runtime.loadedIsGguf,
+ loadedIsMlx: runtime.loadedIsMlx,
+ activeGgufVariant: runtime.activeGgufVariant,
+ activeNativePathToken: runtime.activeNativePathToken,
+ checkpoint: runtime.params.checkpoint,
+ }),
+ true,
+ );
+});