Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: continue finished replies and resume GGUF reasoning (#12371)

oobabooga1250bf1b8c66
diff --git a/studio/backend/core/inference/chat_template_helpers.py b/studio/backend/core/inference/chat_template_helpers.py
index 202c64d90a1..377f1073811 100644
--- a/studio/backend/core/inference/chat_template_helpers.py
+++ b/studio/backend/core/inference/chat_template_helpers.py
@@ -2825,6 +2825,19 @@ def trailing_assistant_text(messages: list) -> Optional[str]:
return None
+def trailing_assistant_resume_kind(messages: list) -> Optional[str]:
+ """Return the trailing assistant field to resume, or None; prefer content over reasoning."""
+ text = trailing_assistant_text(messages)
+ if text:
+ return "content"
+ if text is None:
+ return None
+ reasoning = messages[-1].get("reasoning_content")
+ if isinstance(reasoning, str) and reasoning.strip():
+ return "reasoning_content"
+ return None
+
+
def last_user_text(messages: list) -> str:
"""Text of the newest user turn, with any ``<img>`` markup stripped. Scans back rather than
reading ``messages[-1]``: a continuation ends on the assistant partial. Stops at the newest
@@ -3084,6 +3097,14 @@ def append_assistant_turn(
# Copy rather than mutate: the caller owns assistant_msg and may still read it.
merged_msg = {**conversation[-1], **assistant_msg}
merged_msg["content"] = f"{prev_text}{assistant_msg['content']}"
+ added_reasoning = assistant_msg.get("reasoning_content")
+ if (
+ isinstance(added_reasoning, str)
+ and trailing_assistant_resume_kind(conversation) == "reasoning_content"
+ ):
+ merged_msg["reasoning_content"] = (
+ f"{conversation[-1]['reasoning_content']}{added_reasoning}"
+ )
conversation[-1] = merged_msg
return
conversation.append(assistant_msg)
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index 84326ad27b2..6c484eb838a 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -2146,7 +2146,11 @@ _RESPAWN_REAP_GRACE_S = 1.0
def _finalize_reasoning_only_cumulative(
- cumulative: str, reasoning_text: str, finish_reason: Optional[str], promote_reasoning_only: bool
+ cumulative: str,
+ reasoning_text: str,
+ finish_reason: Optional[str],
+ promote_reasoning_only: bool,
+ resumed_reasoning: str = "",
) -> str:
"""Close a live thinking block and promote it only after a clean stop.
@@ -2157,9 +2161,13 @@ def _finalize_reasoning_only_cumulative(
promotion and let the client surface the ``length`` terminal state. Raw
consumers that do not split reasoning from visible content can disable the
fallback to avoid returning the same reasoning twice.
+
+ Include ``resumed_reasoning`` in the promoted reply; the stream contains only the new tail.
"""
visible_fallback = (
- reasoning_text if promote_reasoning_only and finish_reason != "length" else ""
+ resumed_reasoning + reasoning_text
+ if promote_reasoning_only and finish_reason != "length"
+ else ""
)
return cumulative + "</think>" + visible_fallback
@@ -7689,6 +7697,7 @@ class LlamaCppBackend:
self._vocab_size: Optional[int] = None
# Set from reserves_micro_batch_outputs() by the callers that know the binary.
self._reserves_micro_batch_outputs: bool = False
+ self._resumes_thoughts: bool = True
# Architecture-aware KV fields for 5-path estimation
self._kv_key_length: Optional[int] = None
self._kv_value_length: Optional[int] = None
@@ -9317,6 +9326,20 @@ class LlamaCppBackend:
return False
return build is not None and build < cls._OUTPUT_ROWS_CAPPED_BUILD
+ # ggml-org/llama.cpp#23089 added continuation inside an unfinished reasoning block.
+ _THOUGHT_CONTINUATION_BUILD = 9200
+
+ @classmethod
+ def resumes_thoughts(cls, binary: Optional[str] = None) -> bool:
+ """Whether this llama-server continues a trailing turn that holds only reasoning.
+
+ Unknown builds are treated as current, matching Studio's prebuilts."""
+ try:
+ build = cls.probe_build_number(binary)
+ except Exception:
+ return True
+ return build is None or build >= cls._THOUGHT_CONTINUATION_BUILD
+
_ADVERTISED_DEFAULT_RE = re.compile(r"\(default:\s*(-?\d+)")
@classmethod
@@ -24149,6 +24172,7 @@ class LlamaCppBackend:
server_caps = _launch_caps(binary)
self._reserves_micro_batch_outputs = self.reserves_micro_batch_outputs(binary)
+ self._resumes_thoughts = self.resumes_thoughts(binary)
# Outside ``self._lock`` so /unload, /cancel, /status aren't
# blocked. ``unload_model`` also records the kill, so the
@@ -35453,12 +35477,16 @@ class LlamaCppBackend:
from core.inference.chat_template_helpers import (
neutralize_control_markup_in_messages,
- trailing_assistant_text,
+ trailing_assistant_resume_kind,
)
openai_messages = self._build_openai_messages(messages, image_b64)
- continue_final_message = continue_final_message and bool(
- trailing_assistant_text(openai_messages)
+ _resume_kind = (
+ trailing_assistant_resume_kind(openai_messages) if continue_final_message else None
+ )
+ continue_final_message = _resume_kind is not None
+ resumed_reasoning = (
+ openai_messages[-1]["reasoning_content"] if _resume_kind == "reasoning_content" else ""
)
payload = {
@@ -35658,6 +35686,7 @@ class LlamaCppBackend:
reasoning_text,
_metadata_finish_reason,
promote_reasoning_only,
+ resumed_reasoning,
)
_prov_entry = None
yield cumulative
@@ -35979,7 +36008,11 @@ class LlamaCppBackend:
# retrieval call would actually prompt (ask mode); auto never gates the
# safe search_knowledge_base tool, so retrieval must still run there.
# off never prompts either, so it also keeps first-pass retrieval.
- from core.inference.chat_template_helpers import forced_tool_name, trailing_assistant_text
+ from core.inference.chat_template_helpers import (
+ forced_tool_name,
+ trailing_assistant_resume_kind,
+ trailing_assistant_text,
+ )
initial_forced_name = forced_tool_name(tool_choice)
if initial_forced_name and initial_forced_name not in _gguf_active_tool_names(tools):
@@ -35996,7 +36029,9 @@ class LlamaCppBackend:
and not bypass_permissions
and permission_mode not in ("auto", "off")
)
- ) or bool(continue_final_message and trailing_assistant_text(conversation))
+ ) or bool(
+ continue_final_message and trailing_assistant_resume_kind(conversation) is not None
+ )
_auto = None if _skip_autoinject else build_rag_autoinject(conversation, rag_scope)
if _auto:
for _ev in _auto["events"]:
@@ -36497,7 +36532,8 @@ class LlamaCppBackend:
strict = True,
chat_template_kwargs = _reasoning_kw,
continue_final_message = bool(
- continue_final_message and trailing_assistant_text(fitted)
+ continue_final_message
+ and trailing_assistant_resume_kind(fitted) is not None
),
should_abort = lambda: bool(cancel_event and cancel_event.is_set()),
),
@@ -36622,9 +36658,15 @@ class LlamaCppBackend:
payload["thinking_budget_tokens"] = thinking_budget_tokens
# Re-checked per iteration: once a tool result is appended the partial is
# no longer trailing, so later turns are normal.
- if continue_final_message and trailing_assistant_text(conversation):
+ _resume_kind = (
+ trailing_assistant_resume_kind(conversation) if continue_final_message else None
+ )
+ if _resume_kind is not None:
payload["continue_final_message"] = True
payload["add_generation_prompt"] = False
+ _resumed_reasoning = (
+ conversation[-1]["reasoning_content"] if _resume_kind == "reasoning_content" else ""
+ )
payload["max_tokens"] = (
max_tokens
if max_tokens is not None
@@ -36673,7 +36715,8 @@ class LlamaCppBackend:
strict = True,
chat_template_kwargs = _reasoning_kw,
continue_final_message = bool(
- continue_final_message and trailing_assistant_text(fitted)
+ continue_final_message
+ and trailing_assistant_resume_kind(fitted) is not None
),
should_abort = lambda: bool(cancel_event and cancel_event.is_set()),
),
@@ -36879,6 +36922,7 @@ class LlamaCppBackend:
reasoning_accum,
_iter_finish_reason,
promote_reasoning_only,
+ _resumed_reasoning,
)
_prov_entry = None
if not _suppress_visible_output:
@@ -37451,6 +37495,7 @@ class LlamaCppBackend:
reasoning_accum,
_iter_finish_reason,
promote_reasoning_only,
+ _resumed_reasoning,
)
_prov_entry = None
if not _suppress_visible_output:
@@ -37654,10 +37699,15 @@ class LlamaCppBackend:
_cand_l,
{
"role": "assistant",
- "content": _unfinished_thought_progress(reasoning_accum),
+ # The stream holds only the resumed thought's tail.
+ "content": _unfinished_thought_progress(
+ _resumed_reasoning + reasoning_accum
+ ),
},
continue_final_message = continue_final_message,
)
+ if _resumed_reasoning:
+ _cand_l[-1].pop("reasoning_content", None)
_cand_l.append(
{
"role": "user",
@@ -39011,11 +39061,12 @@ class LlamaCppBackend:
# the field above so a whitespace-only split never appends an
# empty assistant message.
if not assistant_appended and (
- content_text or assistant_msg.get("reasoning_content")
+ content_text or assistant_msg.get("reasoning_content") or _resumed_reasoning
):
- if assistant_msg.get("reasoning_content"):
+ if assistant_msg.get("reasoning_content") or _resumed_reasoning:
+ # Whole thought: the nudge below hides the original's field.
assistant_msg["content"] = neutralize_control_markup(
- reasoning_accum,
+ _resumed_reasoning + reasoning_accum,
self.markup_profile,
)
_continued_partial = (
@@ -39027,12 +39078,14 @@ class LlamaCppBackend:
assistant_msg["content"] = f"\n{assistant_msg['content']}"
if content_text:
assistant_msg["content"] += f"\n{content_text}"
- del assistant_msg["reasoning_content"]
+ assistant_msg.pop("reasoning_content", None)
append_assistant_turn(
conversation,
assistant_msg,
continue_final_message = _merge_into_partial,
)
+ if _resumed_reasoning:
+ conversation[-1].pop("reasoning_content", None)
assistant_appended = True
append_deferred_nudges(conversation, deferred_noop_msgs)
if _final_over_cap:
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 5b8bb33f46d..b52fe00c900 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -440,11 +440,12 @@ def _clamp_finish_reason(value) -> str:
)
-def _continue_final_message(payload) -> bool:
+def _continue_final_message(payload, *, thought: bool = False) -> bool:
"""Whether this request resumes the trailing assistant turn.
Nothing resumable (no assistant turn, or one holding tool calls) degrades to an
- ordinary new turn rather than erroring.
+ ordinary new turn rather than erroring. ``thought`` also resumes a turn holding only
+ ``reasoning_content``, which only llama-server can continue inside its reasoning block.
"""
if not getattr(payload, "continue_final_message", None):
return False
@@ -462,8 +463,8 @@ def _continue_final_message(payload) -> bool:
return False
content = last.get("content") if isinstance(last, dict) else getattr(last, "content", None)
if isinstance(content, str):
- return bool(content)
- if isinstance(content, list):
+ has_text = bool(content)
+ elif isinstance(content, list):
# No resume point inside an image or tool-result part.
texts = []
for part in content:
@@ -473,8 +474,31 @@ def _continue_final_message(payload) -> bool:
texts.append(
part.get("text") if isinstance(part, dict) else getattr(part, "text", None)
)
- return any(texts)
- return False
+ has_text = any(texts)
+ else:
+ has_text = False
+ if has_text or not thought:
+ return has_text
+ reasoning = (
+ last.get("reasoning_content")
+ if isinstance(last, dict)
+ else getattr(last, "reasoning_content", None)
+ )
+ return isinstance(reasoning, str) and bool(reasoning.strip())
+
+
+def _reject_unresumable_thought(payload, llama_backend, reject) -> None:
+ """An older llama-server would treat the thought as finished and answer after it."""
+ if (
+ _continue_final_message(payload, thought = True)
+ and not _continue_final_message(payload)
+ and not getattr(llama_backend, "_resumes_thoughts", True)
+ ):
+ raise reject(
+ 400,
+ "This llama.cpp build cannot resume a response that stopped mid-thought. "
+ "Update Unsloth Studio for a newer llama.cpp, or use Retry.",
+ )
def _reject_audio_output_continuation(payload) -> None:
@@ -2520,7 +2544,7 @@ def _count_gguf_admission_prompt(
) -> int:
"""Exact prompt count plus media allowance; the whole pool if counting fails
(the character estimate undercounts numeric text, #10671)."""
- from core.inference.chat_template_helpers import trailing_assistant_text
+ from core.inference.chat_template_helpers import trailing_assistant_resume_kind
budget = _openai_llama_admission_budget(llama_backend) or 0
try:
@@ -2535,8 +2559,8 @@ def _count_gguf_admission_prompt(
chat_template_kwargs = llama_backend._request_reasoning_kwargs(
payload.enable_thinking, payload.reasoning_effort, payload.preserve_thinking
),
- continue_final_message = _continue_final_message(payload)
- and bool(trailing_assistant_text(messages)),
+ continue_final_message = _continue_final_message(payload, thought = True)
+ and trailing_assistant_resume_kind(messages) is not None,
**({"should_abort": cancel_event.is_set} if cancel_event is not None else {}),
)
if type(count) is not int or count <= 0:
@@ -27375,6 +27399,7 @@ async def produce_openai_chat_completions(
if using_gguf and _takes_tool_passthrough(payload, llama_backend):
if _wants_multiple_choices(payload):
raise _reject_unsupported_n("GGUF tool or response_format passthrough")
+ _reject_unresumable_thought(payload, llama_backend, _reject)
if payload.audio_base64:
# This path forwards the request verbatim, so the transcoded audio
# never gets injected. (The agentic tool loop below does support
@@ -27543,6 +27568,7 @@ async def produce_openai_chat_completions(
# has to hand it these: seeded empty it counted only the current run and let a
# replayed eight sit beside a fresh eight.
_gguf_replayed_image_parts: list = []
+ _reject_unresumable_thought(payload, llama_backend, _reject)
gguf_messages, _ = await _openai_messages_for_gguf_chat_async(
payload,
llama_backend.is_vision,
@@ -27783,7 +27809,9 @@ async def produce_openai_chat_completions(
# the model's output to the partial it holds, so trimming the tail here would
# resume from a different boundary.
_gguf_continue_target = (
- gguf_messages[-1] if _continue_final_message(payload) and gguf_messages else None
+ gguf_messages[-1]
+ if _continue_final_message(payload, thought = True) and gguf_messages
+ else None
)
for _msg in gguf_messages:
if _msg.get("role") == "assistant" and isinstance(_msg.get("content"), str):
@@ -27818,7 +27846,7 @@ async def produce_openai_chat_completions(
enable_thinking = payload.enable_thinking,
reasoning_effort = payload.reasoning_effort,
preserve_thinking = payload.preserve_thinking,
- continue_final_message = _continue_final_message(payload),
+ continue_final_message = _continue_final_message(payload, thought = True),
auto_heal_tool_calls = _gguf_auto_heal_tool_calls,
nudge_tool_calls = payload.nudge_tool_calls,
tool_choice = payload.tool_choice,
@@ -28663,7 +28691,7 @@ async def produce_openai_chat_completions(
enable_thinking = payload.enable_thinking,
reasoning_effort = payload.reasoning_effort,
preserve_thinking = payload.preserve_thinking,
- continue_final_message = _continue_final_message(payload),
+ continue_final_message = _continue_final_message(payload, thought = True),
seed = _seed,
perf_callback = _gguf_perf_callback,
context_overflow = _rolling_context_policy(payload),
@@ -40054,7 +40082,7 @@ def _build_openai_passthrough_body(
stream_options = payload.stream_options,
markup = getattr(llama_backend, "markup_profile", None),
)
- if _continue_final_message(payload):
+ if _continue_final_message(payload, thought = True):
# llama-server rejects both flags set true.
body["continue_final_message"] = True
body["add_generation_prompt"] = False
diff --git a/studio/backend/tests/test_thought_continuation.py b/studio/backend/tests/test_thought_continuation.py
new file mode 100644
index 00000000000..78172daf87a
--- /dev/null
+++ b/studio/backend/tests/test_thought_continuation.py
@@ -0,0 +1,584 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""GGUF continuations preserve and extend the trailing assistant's reasoning.
+
+Fake streams match b11160 with Qwen3.5-4B: deltas exclude the supplied prefill.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import copy
+import json
+import sys
+from pathlib import Path
+from types import SimpleNamespace
+
+import pytest
+
+_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)
+if _BACKEND_DIR not in sys.path:
+ sys.path.insert(0, _BACKEND_DIR)
+
+from core.inference.chat_template_helpers import (
+ append_assistant_turn,
+ trailing_assistant_resume_kind,
+)
+from core.inference.llama_cpp import LlamaCppBackend, _finalize_reasoning_only_cumulative
+
+_THOUGHT = "The user wants three primes. Small ones are"
+_QUESTION = {"role": "user", "content": "Name three primes."}
+_CUT_MID_THOUGHT = {"role": "assistant", "content": "", "reasoning_content": _THOUGHT}
+
+
+def _sse(delta: dict) -> str:
+ return "data: " + json.dumps({"choices": [{"index": 0, "delta": delta}]}) + "\n"
+
+
+def _finish(reason: str) -> str:
+ return (
+ "data: "
+ + json.dumps({"choices": [{"index": 0, "delta": {}, "finish_reason": reason}]})
+ + "\n"
+ )
+
+
+def _done() -> str:
+ return "data: [DONE]\n"
+
+
+_WEB_SEARCH_TOOL = {
+ "type": "function",
+ "function": {
+ "name": "web_search",
+ "description": "Search the web.",
+ "parameters": {
+ "type": "object",
+ "properties": {"query": {"type": "string"}},
+ "required": ["query"],
+ },
+ },
+}
+
+
+def _make_backend(monkeypatch, streams: list[list[str]], payloads: list[dict]):
+ backend = LlamaCppBackend.__new__(LlamaCppBackend)
+ backend._process = object()
+ backend._healthy = True
+ backend._port = 48858
+ backend._api_key = None
+ backend._effective_context_length = 4096
+ backend._supports_reasoning = True
+ backend._reasoning_always_on = False
+ backend._reasoning_style = "enable_thinking"
+ backend._supports_preserve_thinking = False
+
+ @contextlib.contextmanager
+ def fake_stream_with_retry(
+ _client,
+ _url,
+ payload,
+ _cancel_event,
+ headers = None,
+ first_token_deadline = None,
+ ):
+ payloads.append(copy.deepcopy(payload))
+ yield SimpleNamespace(status_code = 200, chunks = streams.pop(0))
+
+ def fake_iter_text_cancellable(
+ response,
+ _cancel_event,
+ first_token_deadline = None,
+ ):
+ yield from response.chunks
+
+ monkeypatch.setattr(backend, "_stream_with_retry", fake_stream_with_retry)
+ monkeypatch.setattr(backend, "_iter_text_cancellable", fake_iter_text_cancellable)
+ monkeypatch.setattr(backend, "_maybe_recover_from_mtp_crash", lambda *_a, **_k: False)
+ return backend
+
+
+def _texts(events) -> list[str]:
+ return [
+ event if isinstance(event, str) else event["text"]
+ for event in events
+ if isinstance(event, str) or event.get("type") == "content"
+ ]
+
+
+# ── The shape the UI sends ─────────────────────────────────────────
+
+
+ "last, kind",
+ [
+ ({"role": "assistant", "content": "2, 3 and"}, "content"),
+ ({"role": "assistant", "content": "2, 3", "reasoning_content": "Easy."}, "content"),
+ (_CUT_MID_THOUGHT, "reasoning_content"),
+ ({"role": "assistant", "content": [], "reasoning_content": _THOUGHT}, "reasoning_content"),
+ # Whitespace is no thought, and an empty turn has nothing to resume.
+ ({"role": "assistant", "content": "", "reasoning_content": " \n"}, None),
+ ({"role": "assistant", "content": ""}, None),
+ # A tool call has no resume point, whatever reasoning it carries.
+ (
+ {
+ "role": "assistant",
+ "content": "",
+ "reasoning_content": _THOUGHT,
+ "tool_calls": [{"id": "c", "type": "function", "function": {"name": "x"}}],
+ },
+ None,
+ ),
+ (_QUESTION, None),
+ ],
+)
+def test_the_resume_kind_matches_what_llama_server_resolves(last, kind):
+ assert trailing_assistant_resume_kind([_QUESTION, last]) == kind
+
+
+def test_the_route_resumes_a_thought_only_where_asked():
+ from routes.inference import _continue_final_message
+
+ payload = SimpleNamespace(
+ continue_final_message = True,
+ messages = [
+ SimpleNamespace(role = "user", content = "Name three primes."),
+ SimpleNamespace(
+ role = "assistant", content = "", reasoning_content = _THOUGHT, tool_calls = None
+ ),
+ ],
+ )
+ # Other backends cannot resume inside a thought, so for them it stays a new turn.
+ assert _continue_final_message(payload) is False
+ assert _continue_final_message(payload, thought = True) is True
+ payload.messages[-1].reasoning_content = " "
+ assert _continue_final_message(payload, thought = True) is False
+ payload.messages[-1].content = "2, 3"
+ assert _continue_final_message(payload) is True
+
+
+# ── Plain GGUF path ──────────────────────────────────────────────
+
+
+def test_a_thought_only_turn_goes_out_as_a_reasoning_continuation(monkeypatch):
+ payloads: list[dict] = []
+ backend = _make_backend(
+ monkeypatch,
+ [[_sse({"reasoning_content": " 2, 3 and 5."}), _sse({"content": "2, 3 and 5."}), _done()]],
+ payloads,
+ )
+
+ events = list(
+ backend.generate_chat_completion(
+ messages = [_QUESTION, dict(_CUT_MID_THOUGHT)],
+ continue_final_message = True,
+ )
+ )
+
+ sent = payloads[0]
+ assert sent["continue_final_message"] is True
+ assert sent["add_generation_prompt"] is False
+ assert sent["messages"][-1] == _CUT_MID_THOUGHT
+ # Only what llama-server streamed: the UI already holds the thought it sent.
+ assert _texts(events)[-1] == "<think> 2, 3 and 5.</think>2, 3 and 5."
+
+
+def test_without_the_flag_the_thought_is_history(monkeypatch):
+ payloads: list[dict] = []
+ backend = _make_backend(monkeypatch, [[_sse({"content": "2, 3, 5."}), _done()]], payloads)
+ list(backend.generate_chat_completion(messages = [_QUESTION, dict(_CUT_MID_THOUGHT)]))
+ assert "continue_final_message" not in payloads[0]
+
+
+def test_a_resumed_thought_promoted_as_the_reply_is_the_whole_thought(monkeypatch):
+ """Promotion includes the original thought and its streamed continuation."""
+ payloads: list[dict] = []
+ backend = _make_backend(
+ monkeypatch,
+ [[_sse({"reasoning_content": " 2, 3 and 5."}), _finish("stop"), _done()]],
+ payloads,
+ )
+
+ events = list(
+ backend.generate_chat_completion(
+ messages = [_QUESTION, dict(_CUT_MID_THOUGHT)],
+ continue_final_message = True,
+ )
+ )
+
+ assert _texts(events)[-1] == f"<think> 2, 3 and 5.</think>{_THOUGHT} 2, 3 and 5."
+
+
+def test_promotion_is_unchanged_without_a_resumed_thought():
+ assert _finalize_reasoning_only_cumulative("<think>abc", "abc", "stop", True) == (
+ "<think>abc</think>abc"
+ )
+ assert _finalize_reasoning_only_cumulative("<think>c", "c", "length", True, "ab") == (
+ "<think>c</think>"
+ )
+
+
+# ── Tool loop ────────────────────────────────────────────────────
+
+
+def test_the_tool_loop_resumes_the_thought_and_keeps_it_whole(monkeypatch):
+ """Tool calls replay the full resumed thought in a single assistant turn."""
+ payloads: list[dict] = []
+ backend = _make_backend(
+ monkeypatch,
+ [
+ [
+ _sse({"reasoning_content": " best looked up."}),
+ _sse(
+ {
+ "tool_calls": [
+ {
+ "index": 0,
+ "id": "call_0",
+ "type": "function",
+ "function": {
+ "name": "web_search",
+ "arguments": json.dumps({"query": "small primes"}),
+ },
+ }
+ ]
+ }
+ ),
+ _finish("tool_calls"),
+ _done(),
+ ],
+ [_sse({"content": "2, 3 and 5."}), _done()],
+ ],
+ payloads,
+ )
+ monkeypatch.setattr("core.inference.tools.execute_tool", lambda *_a, **_k: "2, 3, 5, 7, 11")
+
+ list(
+ backend.generate_chat_completion_with_tools(
+ messages = [_QUESTION, dict(_CUT_MID_THOUGHT)],
+ tools = [_WEB_SEARCH_TOOL],
+ continue_final_message = True,
+ max_tool_iterations = 2,
+ auto_heal_tool_calls = False,
+ )
+ )
+
+ first = payloads[0]
+ assert first["continue_final_message"] is True
+ assert first["add_generation_prompt"] is False
+ assert first["messages"][-1]["reasoning_content"] == _THOUGHT
+
+ second = payloads[1]
+ assert "continue_final_message" not in second, "after a tool result the turn is new"
+ calls = [m for m in second["messages"] if m.get("role") == "assistant" and m.get("tool_calls")]
+ assert len(calls) == 1
+ assert calls[0]["reasoning_content"] == f"{_THOUGHT} best looked up."
+ assert [m["role"] for m in second["messages"]].count("assistant") == 1
+
+
+def _replayed_turn(payload: dict) -> dict:
+ """The resumed assistant turn, replayed ahead of the user turn the loop appended."""
+ assert payload["messages"][-1]["role"] == "user"
+ turn = payload["messages"][-2]
+ assert turn["role"] == "assistant"
+ return turn
+
+
[email protected]("tail", [" worth checking.", ""])
+def test_a_no_op_tool_call_after_a_resumed_thought_replays_the_whole_thought(monkeypatch, tail):
+ """The nudge hides reasoning_content, so the thought travels as content, all of it."""
+ payloads: list[dict] = []
+ backend = _make_backend(
+ monkeypatch,
+ [
+ [
+ *([_sse({"reasoning_content": tail})] if tail else []),
+ # Not enabled for this request, so the call is a no-op and a nudge follows.
+ _sse(
+ {
+ "tool_calls": [
+ {
+ "index": 0,
+ "id": "call_0",
+ "type": "function",
+ "function": {"name": "python", "arguments": '{"code": "1"}'},
+ }
+ ]
+ }
+ ),
+ _finish("tool_calls"),
+ _done(),
+ ],
+ [_sse({"content": "2, 3 and 5."}), _finish("stop"), _done()],
+ ],
+ payloads,
+ )
+
+ list(
+ backend.generate_chat_completion_with_tools(
+ messages = [_QUESTION, dict(_CUT_MID_THOUGHT)],
+ tools = [_WEB_SEARCH_TOOL],
+ continue_final_message = True,
+ max_tool_iterations = 3,
+ auto_heal_tool_calls = False,
+ )
+ )
+
+ turn = _replayed_turn(payloads[1])
+ assert turn["content"] == f"{_THOUGHT}{tail}"
+ assert "reasoning_content" not in turn
+
+
+def test_a_resumed_thought_cut_by_length_notes_the_whole_thought(monkeypatch):
+ payloads: list[dict] = []
+ backend = _make_backend(
+ monkeypatch,
+ [
+ [_sse({"reasoning_content": " 2, then 3, then"}), _finish("length"), _done()],
+ [_sse({"content": "2, 3 and 5."}), _finish("stop"), _done()],
+ ],
+ payloads,
+ )
+
+ list(
+ backend.generate_chat_completion_with_tools(
+ messages = [_QUESTION, dict(_CUT_MID_THOUGHT)],
+ tools = [_WEB_SEARCH_TOOL],
+ continue_final_message = True,
+ max_tool_iterations = 3,
+ auto_heal_tool_calls = False,
+ )
+ )
+
+ turn = _replayed_turn(payloads[1])
+ assert f"{_THOUGHT} 2, then 3, then" in turn["content"]
+ assert "reasoning_content" not in turn
+
+
+def test_a_merged_thought_joins_the_old_reasoning():
+ conversation = [_QUESTION, dict(_CUT_MID_THOUGHT)]
+ append_assistant_turn(
+ conversation,
+ {"role": "assistant", "content": "2, 3, 5.", "reasoning_content": " 2, 3 and 5."},
+ continue_final_message = True,
+ )
+ assert conversation[-1] == {
+ "role": "assistant",
+ "content": "2, 3, 5.",
+ "reasoning_content": f"{_THOUGHT} 2, 3 and 5.",
+ }
+
+
+def test_a_resumed_answer_keeps_its_own_reasoning():
+ """Resuming the answer text never grows the thought before it."""
+ conversation = [
+ _QUESTION,
+ {"role": "assistant", "content": "2, 3", "reasoning_content": "Easy."},
+ ]
+ append_assistant_turn(
+ conversation, {"role": "assistant", "content": " and 5."}, continue_final_message = True
+ )
+ assert conversation[-1] == {
+ "role": "assistant",
+ "content": "2, 3 and 5.",
+ "reasoning_content": "Easy.",
+ }
+
+
+# ── Older llama-server builds ─────────────────────────────────────
+
+
+ "build, resumes",
+ [(9199, False), (9200, True), (11160, True), (None, True)],
+)
+def test_thought_resumption_needs_llama_cpp_b9200(monkeypatch, build, resumes):
+ monkeypatch.setattr(
+ LlamaCppBackend, "probe_build_number", classmethod(lambda cls, b = None: build)
+ )
+ assert LlamaCppBackend.resumes_thoughts() is resumes
+
+
+class _LiveProcess:
+ """Stands in for the llama-server process: alive, and nothing to kill."""
+
+ pid = None
+ returncode = None
+
+ def poll(self):
+ return None
+
+
+def _route_backend(monkeypatch, streams: list[list[str]], payloads: list[dict], *, tools: bool):
+ """A constructed backend behind the real route, so every attribute the route reads exists."""
+ backend = LlamaCppBackend()
+ backend._process = _LiveProcess()
+ backend._healthy = True
+ backend._port = 48859
+ backend._effective_context_length = 4096
+ backend._supports_reasoning = True
+ backend._reasoning_style = "enable_thinking"
+ backend._supports_tools = tools
+ backend._model_identifier = "unsloth/Qwen3.5-4B-GGUF"
+
+ @contextlib.contextmanager
+ def fake_stream_with_retry(
+ _client,
+ _url,
+ payload,
+ _cancel_event,
+ headers = None,
+ first_token_deadline = None,
+ ):
+ payloads.append(copy.deepcopy(payload))
+ yield SimpleNamespace(status_code = 200, chunks = streams.pop(0))
+
+ monkeypatch.setattr(backend, "_stream_with_retry", fake_stream_with_retry)
+ monkeypatch.setattr(
+ backend,
+ "_iter_text_cancellable",
+ lambda response, _e, first_token_deadline = None: iter(response.chunks),
+ )
+ monkeypatch.setattr(backend, "_maybe_recover_from_mtp_crash", lambda *_a, **_k: False)
+ monkeypatch.setattr(backend, "count_chat_tokens", lambda *_a, **_k: 64)
+ return backend
+
+
+def _route_client(monkeypatch, backend):
+ from fastapi import FastAPI
+ from fastapi.testclient import TestClient
+
+ import routes.inference as inference_route
+ from auth.authentication import get_current_subject
+ from utils.api_errors import install_api_error_handlers
+
+ monkeypatch.setattr(inference_route, "get_llama_cpp_backend", lambda: backend)
+ app = FastAPI()
+ app.include_router(inference_route.router, prefix = "/v1")
+ install_api_error_handlers(app)
+ app.dependency_overrides[get_current_subject] = lambda: "test-user"
+ return TestClient(app)
+
+
+def _stream_deltas(resp) -> list[dict]:
+ deltas = []
+ for line in resp.text.splitlines():
+ if line.startswith("data: ") and line != "data: [DONE]":
+ choice = (json.loads(line[6:]).get("choices") or [{}])[0]
+ if choice.get("delta"):
+ deltas.append(choice["delta"])
+ return deltas
+
+
[email protected]("tools", [False, True], ids = ["plain", "tool-loop"])
+def test_the_chat_route_resumes_the_thought_the_ui_sends(monkeypatch, tools):
+ """The request the chat UI sends for Resume on a turn stopped mid-thought, end to end."""
+ payloads: list[dict] = []
+ backend = _route_backend(
+ monkeypatch,
+ [
+ [
+ _sse({"reasoning_content": " 2, 3 and 5."}),
+ _sse({"content": "2, 3 and 5."}),
+ _finish("stop"),
+ _done(),
+ ]
+ ],
+ payloads,
+ tools = tools,
+ )
+ body = {
+ "messages": [_QUESTION, _CUT_MID_THOUGHT],
+ "continue_final_message": True,
+ "stream": True,
+ }
+ if tools:
+ body.update(enable_tools = True, enabled_tools = ["web_search"])
+ resp = _route_client(monkeypatch, backend).post(
+ "/v1/chat/completions", json = body, headers = {"X-Unsloth-Events": "1"}
+ )
+
+ assert resp.status_code == 200, resp.text
+ (sent,) = payloads
+ assert ("tools" in sent) is tools
+ assert sent["continue_final_message"] is True
+ assert sent["add_generation_prompt"] is False
+ assert sent["messages"][-1] == {**_CUT_MID_THOUGHT}
+ deltas = _stream_deltas(resp)
+ assert "".join(d.get("reasoning_content") or "" for d in deltas) == " 2, 3 and 5."
+ assert "".join(d.get("content") or "" for d in deltas) == "2, 3 and 5."
+
+
+def test_an_older_llama_server_refuses_to_resume_a_thought(monkeypatch):
+ """Older builds close the thought instead of continuing inside it."""
+ payloads: list[dict] = []
+ backend = _route_backend(monkeypatch, [], payloads, tools = False)
+ backend._resumes_thoughts = False
+ resp = _route_client(monkeypatch, backend).post(
+ "/v1/chat/completions",
+ json = {
+ "messages": [_QUESTION, _CUT_MID_THOUGHT],
+ "continue_final_message": True,
+ "stream": False,
+ },
+ )
+ assert resp.status_code == 400, resp.text
+ assert "mid-thought" in resp.text
+ assert payloads == []
+
+
+def test_an_older_llama_server_still_resumes_an_answer(monkeypatch):
+ payloads: list[dict] = []
+ backend = _route_backend(
+ monkeypatch,
+ [[_sse({"content": " and 5."}), _finish("stop"), _done()]],
+ payloads,
+ tools = False,
+ )
+ backend._resumes_thoughts = False
+ resp = _route_client(monkeypatch, backend).post(
+ "/v1/chat/completions",
+ json = {
+ "messages": [_QUESTION, {"role": "assistant", "content": "2, 3"}],
+ "continue_final_message": True,
+ "stream": True,
+ },
+ )
+ assert resp.status_code == 200, resp.text
+ assert payloads[0]["continue_final_message"] is True
+
+
+def test_the_tool_passthrough_resumes_the_thought(monkeypatch):
+ """Client tools skip Studio's loop, and the forwarded body must still continue the thought."""
+ from models.inference import ChatCompletionRequest
+ from routes.inference import _build_openai_passthrough_body
+
+ backend = _route_backend(monkeypatch, [], [], tools = True)
+ payload = ChatCompletionRequest(
+ messages = [_QUESTION, _CUT_MID_THOUGHT],
+ tools = [_WEB_SEARCH_TOOL],
+ continue_final_message = True,
+ )
+ body = _build_openai_passthrough_body(payload, llama_backend = backend)
+ assert body["continue_final_message"] is True
+ assert body["add_generation_prompt"] is False
+ assert body["messages"][-1]["reasoning_content"] == _THOUGHT
+
+
+def test_an_older_llama_server_refuses_a_thought_in_the_tool_passthrough(monkeypatch):
+ payloads: list[dict] = []
+ backend = _route_backend(monkeypatch, [], payloads, tools = True)
+ backend._resumes_thoughts = False
+ resp = _route_client(monkeypatch, backend).post(
+ "/v1/chat/completions",
+ json = {
+ "messages": [_QUESTION, _CUT_MID_THOUGHT],
+ "tools": [_WEB_SEARCH_TOOL],
+ "continue_final_message": True,
+ "stream": False,
+ },
+ )
+ assert resp.status_code == 400, resp.text
+ assert "mid-thought" in resp.text
+ assert payloads == []
diff --git a/studio/frontend/src/components/assistant-ui/thread.tsx b/studio/frontend/src/components/assistant-ui/thread.tsx
index 6ee2df6aa3d..4de852e4eec 100644
--- a/studio/frontend/src/components/assistant-ui/thread.tsx
+++ b/studio/frontend/src/components/assistant-ui/thread.tsx
@@ -151,6 +151,7 @@ import {
attachLibraryChatFiles,
useLibraryChatHandoffStore,
} from "@/features/library/chat-handoff-store";
+import { isServedByLlamaCpp } from "@/features/model-picker";
import { cancelResearchRun } from "@/features/chat/api/research-api";
import {
ingestResearchUpdate,
@@ -166,11 +167,13 @@ import { replySourceMarkdown } from "@/features/chat/utils/reply-source-markdown
import { toolResultModelText } from "@/features/chat/api/chat-adapter";
import {
CONTINUATION_RUN_CONFIG_KEY,
+ type ContinuationRequest,
incompleteLabel,
incompleteRemedy,
isContinuableContent,
isProviderReportedReason,
modeAllowsContinuation,
+ readContinuationSource,
readIncompleteInfo,
readTextThoughtSignature,
claimAutoContinue,
@@ -342,6 +345,7 @@ import {
Image03Icon,
McpServerIcon,
PencilRulerIcon,
+ PlayIcon,
Scroll01Icon,
Telescope02Icon,
VolumeMute02Icon,
@@ -7370,49 +7374,16 @@ const CancelledIndicator: FC = () => {
);
};
-/** Text of an assistant turn: what a continuation resumes from.
- *
- * Text parts only: a continuation resumes the visible answer, not its private reasoning.
- * Joined with nothing, like the backend's `trailing_assistant_text`: a turn split around
- * a reasoning part never had a newline between its halves, and inventing one moves the
- * boundary. */
-function assistantMessageText(content: readonly unknown[] | undefined): string {
- if (!content) {
- return "";
- }
- return content
- .filter(
- (part): part is { type: "text"; text: string } =>
- (part as { type?: string })?.type === "text" &&
- typeof (part as { text?: unknown })?.text === "string",
- )
- .map((part) => part.text)
- .join("");
+function readThoughtDuration(metadata: unknown): number | undefined {
+ const custom = (metadata as { custom?: Record<string, unknown> } | undefined)
+ ?.custom;
+ const durations = custom?.reasoningDurations;
+ const value = Array.isArray(durations) ? durations[0] : custom?.reasoningDuration;
+ return typeof value === "number" && Number.isFinite(value) ? value : undefined;
}
-/**
- * Resume a response that stopped early instead of regenerating it. Shown under the last
- * assistant turn when Max Tokens ran out, Stop was pressed, or the stream dropped.
- * Retry keeps its old meaning: drop the partial and start over.
- */
-const ContinueMessageBar: FC = () => {
- // One subscription, not ten, on every message that is not the newest.
- //
- // The bar mounts under every assistant message and returns null unless it is the last, but the
- // ten `useAuiState` calls below ran first, each a subscription whose selector re-runs on EVERY
- // store update -- one per character typed (220 messages, 300K characters: 10,193 subscriptions,
- // 10,258 selector runs per keystroke).
- //
- // `isLast` is the same condition the body below already gates on, asked before the work rather
- // than after it, so nothing that used to render stops rendering.
- const isLast = useAuiState(({ message }) => message.isLast);
- if (!isLast) {
- return null;
- }
- return <ContinueMessageBarForLastMessage />;
-};
-
-const ContinueMessageBarForLastMessage: FC = () => {
+/** Shared eligibility and run setup for Resume and Continue response. */
+function useContinuation() {
const aui = useAui();
const messageId = useAuiState(({ message }) => message.id);
const isLast = useAuiState(({ message }) => message.isLast);
@@ -7421,13 +7392,25 @@ const ContinueMessageBarForLastMessage: FC = () => {
const researchActive = useThreadResearchActive();
const status = useAuiState(({ message }) => message.status);
const metadata = useAuiState(({ message }) => message.metadata);
- const partial = useAuiState(({ message }) =>
- assistantMessageText(message.content),
+ // Only llama-server can resume a thought.
+ const thoughtResumable = useChatRuntimeStore((s) =>
+ isServedByLlamaCpp({
+ loadedIsGguf: s.loadedIsGguf,
+ activeGgufVariant: s.activeGgufVariant,
+ activeNativePathToken: s.activeNativePathToken,
+ checkpoint: s.params.checkpoint,
+ }),
+ );
+ const partial = useAuiState(
+ ({ message }) => readContinuationSource(message.content).partial,
+ );
+ const reasoning = useAuiState(
+ ({ message }) => readContinuationSource(message.content).reasoning,
);
// A tool-calling turn cannot be resumed: the continuation runs as a sibling, so the
// call and its result would be missing from the outbound history.
const continuable = useAuiState(({ message }) =>
- isContinuableContent(message.content),
+ isContinuableContent(message.content, { thought: thoughtResumable }),
);
// Gemini signs its text parts, and the resumed turn is replayed from this branch,
// so the signature travels with the partial.
@@ -7454,11 +7437,9 @@ const ContinueMessageBarForLastMessage: FC = () => {
cancelled && !isProviderReportedReason(stamped?.reason)
? ("cancelled" as const)
: stamped?.reason;
+ const carriedReasoning = thoughtResumable ? reasoning : "";
- // Every gate the bar itself answers to. Resuming without asking has to clear the same
- // ones, or it would resume a turn the bar would have refused to offer.
- const resumable =
- Boolean(reason) &&
+ const canResume =
isLast &&
!isRunning &&
!researchRunId &&
@@ -7468,18 +7449,9 @@ const ContinueMessageBarForLastMessage: FC = () => {
fromAudioInput,
audioOutputModel,
}) &&
- Boolean(partial.trim());
-
- // A cut with a remedy is one resuming cannot undo, so the way out replaces the button.
- const remedy = reason ? incompleteRemedy(reason) : null;
-
- // The parent is what every round of one logical turn shares; the message id changes
- // each round, because a continuation runs as a sibling.
- const parentId = useAuiState(({ thread, message }) => {
- const index = thread.messages.findIndex((m) => m.id === message.id);
- return index > 0 ? thread.messages[index - 1].id : null;
- });
+ Boolean(partial.trim() || carriedReasoning.trim());
+ const reasoningDuration = readThoughtDuration(metadata);
// Hands the started run back, untyped: the only handle identified with THIS run.
const startContinuation = useCallback((): unknown => {
const messages = aui.thread().getState().messages;
@@ -7487,17 +7459,56 @@ const ContinueMessageBarForLastMessage: FC = () => {
if (index < 0) {
return undefined;
}
- // Sibling of the truncated turn, so the branch picker can still reach the partial.
+ // Sibling of the resumed turn, so the branch picker can still reach the original.
const parent = index > 0 ? messages[index - 1].id : null;
+ const request: ContinuationRequest = {
+ partial,
+ ...(carriedReasoning ? { reasoning: carriedReasoning, reasoningDuration } : {}),
+ ...(thoughtSignature ? { thoughtSignature } : {}),
+ };
return aui.thread().startRun({
parentId: parent,
runConfig: {
- custom: {
- [CONTINUATION_RUN_CONFIG_KEY]: { partial, thoughtSignature },
- },
+ custom: { [CONTINUATION_RUN_CONFIG_KEY]: request },
},
});
- }, [aui, messageId, partial, thoughtSignature]);
+ }, [aui, messageId, partial, carriedReasoning, reasoningDuration, thoughtSignature]);
+
+ return {
+ messageId,
+ reason,
+ completed: status?.type === "complete",
+ canResume,
+ resumedChars: partial.length + carriedReasoning.length,
+ startContinuation,
+ };
+}
+
+const ContinueMessageBar: FC = () => {
+ // Mount the full subscriptions only for the newest message to keep typing responsive.
+ const isLast = useAuiState(({ message }) => message.isLast);
+ if (!isLast) {
+ return null;
+ }
+ return <ContinueMessageBarForLastMessage />;
+};
+
+const ContinueMessageBarForLastMessage: FC = () => {
+ const aui = useAui();
+ const { messageId, reason, canResume, resumedChars, startContinuation } =
+ useContinuation();
+
+ const resumable = Boolean(reason) && canResume;
+
+ // A cut with a remedy is one resuming cannot undo, so the way out replaces the button.
+ const remedy = reason ? incompleteRemedy(reason) : null;
+
+ // The parent is what every round of one logical turn shares; the message id changes
+ // each round, because a continuation runs as a sibling.
+ const parentId = useAuiState(({ thread, message }) => {
+ const index = thread.messages.findIndex((m) => m.id === message.id);
+ return index > 0 ? thread.messages[index - 1].id : null;
+ });
// The resumed turn's own fit. Resuming replays the partial as the final assistant turn,
// which the fit protects, so a partial too big to sit beside the system turn makes the
@@ -7543,7 +7554,7 @@ const ContinueMessageBarForLastMessage: FC = () => {
fits: truncation?.fits,
// The same cheap estimator the backend fit uses, which is all that is needed to
// spot a partial that has already eaten the whole budget.
- partialTokens: Math.ceil(partial.length / 4),
+ partialTokens: Math.ceil(resumedChars / 4),
promptTarget: truncation?.prompt_target,
});
useEffect(() => {
@@ -7645,24 +7656,6 @@ const ContinueMessageBarForLastMessage: FC = () => {
);
}
- const handleContinue = () => {
- const messages = aui.thread().getState().messages;
- const index = messages.findIndex((message) => message.id === messageId);
- if (index < 0) {
- return;
- }
- // Sibling of the truncated turn, so the branch picker can still reach the partial.
- const parentId = index > 0 ? messages[index - 1].id : null;
- aui.thread().startRun({
- parentId,
- runConfig: {
- custom: {
- [CONTINUATION_RUN_CONFIG_KEY]: { partial, thoughtSignature },
- },
- },
- });
- };
-
return (
<div className="aui-continue-bar mt-2 flex flex-wrap items-center gap-x-3 gap-y-2 rounded-md border border-border/70 bg-muted/50 p-2.5 text-sm">
<span className="min-w-0 flex-1 text-muted-foreground">
@@ -7674,7 +7667,9 @@ const ContinueMessageBarForLastMessage: FC = () => {
size="sm"
variant="secondary"
className="h-7 shrink-0 gap-1.5 text-xs"
- onClick={handleContinue}
+ onClick={() => {
+ startContinuation();
+ }}
>
<QueueResumeIcon className="size-3.5" />
Resume
@@ -8356,6 +8351,38 @@ const EditAssistantMessageButton: FC = () => {
);
};
+/** Continue the newest finished reply; incomplete replies use the Resume bar. */
+const ContinueResponseButton: FC = () => {
+ const isLast = useAuiState(({ message }) => message.isLast);
+ if (!isLast) {
+ return null;
+ }
+ return <ContinueResponseButtonForLastMessage />;
+};
+
+const ContinueResponseButtonForLastMessage: FC = () => {
+ const { messageId, reason, completed, canResume, startContinuation } =
+ useContinuation();
+ const editing = useChatRuntimeStore((s) => s.editingMessageId === messageId);
+ // The sibling carries no citations, and a continuation restarts their [N] numbering.
+ const cited = useAuiState(({ message }) =>
+ message.content.some((part) => part.type === "source"),
+ );
+ if (!completed || reason || !canResume || editing || cited) {
+ return null;
+ }
+ return (
+ <TooltipIconButton
+ tooltip="Continue response"
+ onClick={() => {
+ startContinuation();
+ }}
+ >
+ <HugeiconsIcon icon={PlayIcon} strokeWidth={1.75} className="size-icon" />
+ </TooltipIconButton>
+ );
+};
+
// The More menu's Edit response, shown when the button is not pinned to the bar.
const EditAssistantMessageMenuItem: FC = () => {
const messageId = useAuiState(({ message }) => message.id);
@@ -8436,6 +8463,7 @@ const AssistantActionBar: FC = () => {
>
<CopyButton />
{inlineEdit && <EditAssistantMessageButton />}
+ <ContinueResponseButton />
{!researchRunId && !researchActive && (
<ActionBarPrimitive.Reload asChild={true}>
<TooltipIconButton tooltip="Refresh">
diff --git a/studio/frontend/src/features/chat/api/chat-adapter.ts b/studio/frontend/src/features/chat/api/chat-adapter.ts
index d4b43dea47e..362fd34210e 100644
--- a/studio/frontend/src/features/chat/api/chat-adapter.ts
+++ b/studio/frontend/src/features/chat/api/chat-adapter.ts
@@ -301,6 +301,7 @@ import type { CachedGgufRepo, CachedModelRepo } from "./chat-api";
import {
budgetImpliesTruncation,
CONTINUE_INSTRUCTION,
+ continuationSeed,
createContinuationMerger,
hasRenderableContent,
incompleteLabel,
@@ -5022,11 +5023,30 @@ export function createOpenAIStreamAdapter(
);
}
+ // Carry reasoning only to llama-server, the one backend that can resume it.
+ const resumedThought =
+ continuation &&
+ isServedByLlamaCpp({
+ loadedIsGguf: runtime.loadedIsGguf,
+ activeGgufVariant: runtime.activeGgufVariant,
+ activeNativePathToken: runtime.activeNativePathToken,
+ checkpoint: params.checkpoint,
+ })
+ ? (continuation.reasoning ?? "")
+ : "";
+ if (continuation && !continuation.partial && !resumedThought) {
+ toast.error("This response cannot be resumed", {
+ description:
+ "It stopped mid-thought, and only GGUF models can resume a thought. Use Retry instead.",
+ });
+ throw new Error("A response that stopped mid-thought cannot be resumed here.");
+ }
// The run's messages stop at the user turn, so the partial is appended here for the backend to resume.
if (continuation) {
outboundMessages.push({
role: "assistant",
content: continuation.partial,
+ ...(resumedThought ? { reasoning_content: resumedThought } : {}),
});
// The original assistant message is not in this branch, so without its signature the history
// goes back unsigned.
@@ -5434,9 +5454,12 @@ export function createOpenAIStreamAdapter(
local: !isExternalRequest,
owner: serverCancel,
});
- // Seeded with the partial so the bubble reads as one response; the boundary lets the
- // finalizers repair a repeat or restart.
- let cumulativeText = continuation ? continuation.partial : "";
+ const continuationPartial = continuation
+ ? continuationSeed(continuation.partial, resumedThought)
+ : "";
+ // A seed that ends inside the thought: the next reasoning delta extends it.
+ const resumesInsideThought = Boolean(resumedThought) && !continuation?.partial;
+ let cumulativeText = continuationPartial;
// Reading `cumulativeText` costs O(reply): each `+=` builds a cons string that the first read
// flattens, so one charCodeAt per arrival is as expensive as a scan. Everything below is fed
// the delta through `appendCumulative` and the buffer is read only where the reply is
@@ -5450,7 +5473,6 @@ export function createOpenAIStreamAdapter(
// Whether this run appended reply text of its own: a continuation is SEEDED with the previous
// run's partial, so a run that adds nothing must not have its tail trimmed.
let producedReplyText = false;
- const continuationPartial = continuation?.partial ?? "";
// Local backends resume at the exact token boundary, so trimming could only delete words the
// model meant; the repair is for providers that repeat or restart.
const repairContinuation =
@@ -5537,9 +5559,15 @@ export function createOpenAIStreamAdapter(
let requestedMaxTokens: number | undefined;
const isMlxRequest = !isExternalRequest && activeModel?.isMlx === true;
const reasoningDurationTracker = createReasoningDurationTracker();
- // True while wrapping a `delta.reasoning_content` stream in <think> for parseAssistantContent;
- // outside the SSE loop because the close tag fires when content arrives.
- let reasoningContentOpen = false;
+ if (resumedThought) {
+ reasoningDurationTracker.seedThought({
+ duration: continuation?.reasoningDuration,
+ open: resumesInsideThought,
+ textLength: resumedThought.length,
+ });
+ }
+ // Keep the <think> block open across reasoning deltas; answer content closes it.
+ let reasoningContentOpen = resumesInsideThought;
type ToolCallProvenance = {
source?: string;
healed?: boolean;
@@ -5953,10 +5981,13 @@ export function createOpenAIStreamAdapter(
: { thinking: { type: reasoningEnabled ? "enabled" : "disabled" } }
: {};
// Decided before the continuation yield below, which an abort during load saves as is.
+ // A carried thought is reasoning whatever this request's thinking setting says.
setParseThink(
isExternalRequest
? requestParsesThinkTags(externalReasoningFields)
- : reasoningAlwaysOn || requestParsesThinkTags(localReasoningFields),
+ : reasoningAlwaysOn ||
+ Boolean(resumedThought) ||
+ requestParsesThinkTags(localReasoningFields),
);
// Yielded before the request starts: an abort during load skips the partial-content yield
// below, saving an empty message.
@@ -8181,6 +8212,11 @@ export function createOpenAIStreamAdapter(
// A run can stop cleanly on its first token and leave nothing behind.
// Saved as complete that is a blank bubble with no way out.
(hasRenderableContent(finalContent) ? null : "empty");
+ if (continuation && !producedReplyText && !finalIncompleteReason) {
+ toast("The model had nothing to add", {
+ description: "It ended the reply where it already stopped.",
+ });
+ }
yield {
content: finalContent,
metadata: {
diff --git a/studio/frontend/src/features/chat/utils/continuation.ts b/studio/frontend/src/features/chat/utils/continuation.ts
index d1ffcf9e044..c5d3a2da7df 100644
--- a/studio/frontend/src/features/chat/utils/continuation.ts
+++ b/studio/frontend/src/features/chat/utils/continuation.ts
@@ -230,29 +230,64 @@ export function budgetImpliesTruncation({
);
}
-/** Whether an assistant turn can be resumed at all. A turn that called a tool cannot: the
- * continuation runs as a sibling, so the call and its result are absent from the outbound
- * history. Matches the backend guard. */
+/** Mirrors the backend guard: tool calls block; reasoning-only needs `thought`. */
export function isContinuableContent(
content: readonly unknown[] | undefined,
+ { thought = false }: { thought?: boolean } = {},
): boolean {
if (!content) {
return false;
}
let hasText = false;
+ let hasReasoning = false;
for (const part of content) {
const type = (part as { type?: string })?.type;
if (type === "text") {
hasText = hasText || ((part as { text?: string }).text ?? "").length > 0;
continue;
}
- // Reasoning and citations are never replayed, so they neither block nor enable.
- if (type === "reasoning" || type === "source") {
+ if (type === "reasoning") {
+ hasReasoning =
+ hasReasoning || ((part as { text?: string }).text ?? "").trim().length > 0;
+ continue;
+ }
+ // Citations are never replayed, so they neither block nor enable.
+ if (type === "source") {
continue;
}
return false;
}
- return hasText;
+ return hasText || (thought && hasReasoning);
+}
+
+/** Reasoning is kept only when it all precedes the answer, as reasoning_content does. */
+export function readContinuationSource(
+ content: readonly unknown[] | undefined,
+): { partial: string; reasoning: string } {
+ let partial = "";
+ const thoughts: string[] = [];
+ let ordered = true;
+ for (const part of content ?? []) {
+ const { type, text } = (part ?? {}) as { type?: string; text?: unknown };
+ if (typeof text !== "string") {
+ continue;
+ }
+ if (type === "text") {
+ partial += text;
+ } else if (type === "reasoning") {
+ ordered = ordered && partial.length === 0;
+ thoughts.push(text);
+ }
+ }
+ return { partial, reasoning: ordered ? thoughts.join("\n") : "" };
+}
+
+/** Seed the adapter buffer, leaving <think> open when there is no answer yet. */
+export function continuationSeed(partial: string, thought: string): string {
+ if (!thought) {
+ return partial;
+ }
+ return partial ? `<think>${thought}</think>${partial}` : `<think>${thought}`;
}
/** The newest Gemini text-part thoughtSignature on an assistant turn, carried so the resumed turn
@@ -322,8 +357,12 @@ export const CONTINUE_INSTRUCTION =
export const CONTINUATION_RUN_CONFIG_KEY = "unslothContinuation";
export type ContinuationRequest = {
- /** The partial answer to resume, exactly as it was rendered. */
+ /** The partial answer exactly as rendered; empty when stopped mid-thought. */
partial: string;
+ /** Carried only to llama-server. */
+ reasoning?: string;
+ /** Seconds, so the resumed turn keeps its timer. */
+ reasoningDuration?: number;
/** Gemini text-part thoughtSignature from the turn being resumed: the sibling run drops the
* original assistant message, so replaying it here keeps the history signed. */
thoughtSignature?: string;
@@ -336,16 +375,36 @@ export function readContinuationRequest(
const custom = (runConfig as { custom?: Record<string, unknown> } | undefined)
?.custom;
const request = custom?.[CONTINUATION_RUN_CONFIG_KEY] as
- | { partial?: unknown; thoughtSignature?: unknown }
+ | {
+ partial?: unknown;
+ reasoning?: unknown;
+ reasoningDuration?: unknown;
+ thoughtSignature?: unknown;
+ }
| undefined;
- const partial = request?.partial;
- if (typeof partial === "string" && partial.length > 0) {
- const signature = request?.thoughtSignature;
- return typeof signature === "string" && signature
- ? { partial, thoughtSignature: signature }
- : { partial };
+ const partial = typeof request?.partial === "string" ? request.partial : "";
+ const reasoning =
+ typeof request?.reasoning === "string" && request.reasoning.trim()
+ ? request.reasoning
+ : "";
+ if (!partial && !reasoning) {
+ return null;
}
- return null;
+ const duration = request?.reasoningDuration;
+ const signature = request?.thoughtSignature;
+ return {
+ partial,
+ ...(reasoning ? { reasoning } : {}),
+ ...(reasoning &&
+ typeof duration === "number" &&
+ Number.isFinite(duration) &&
+ duration >= 0
+ ? { reasoningDuration: duration }
+ : {}),
+ ...(typeof signature === "string" && signature
+ ? { thoughtSignature: signature }
+ : {}),
+ };
}
/** Resuming a Max Tokens cut WITHOUT asking: hitting the cap is not a decision the user made.
diff --git a/studio/frontend/src/features/chat/utils/reasoning-duration.ts b/studio/frontend/src/features/chat/utils/reasoning-duration.ts
index 009e89a9e48..a29177e68f7 100644
--- a/studio/frontend/src/features/chat/utils/reasoning-duration.ts
+++ b/studio/frontend/src/features/chat/utils/reasoning-duration.ts
@@ -176,6 +176,28 @@ export function createReasoningDurationTracker(
finishGroup() {
finishGroupAt(now());
},
+ /** Preserve the original thought's duration. Open thoughts keep timing and ignore server
+ * summaries of just the new tail; closed thoughts reopen only if their text grows. */
+ seedThought({
+ duration,
+ open,
+ textLength,
+ }: {
+ duration: number | undefined;
+ open: boolean;
+ textLength: number;
+ }) {
+ const known = asDuration(duration);
+ if (known !== undefined) {
+ setDuration(0, known);
+ }
+ reasoningLength[0] = textLength;
+ groupCount = 1;
+ if (open) {
+ startedAt[0] = now() - (known ?? 0) * 1000;
+ activeIndex = 0;
+ }
+ },
recordServerDuration(reasoningMs: unknown): boolean {
if (
typeof reasoningMs !== "number" ||
diff --git a/studio/frontend/tests/chat-thought-continuation.test.ts b/studio/frontend/tests/chat-thought-continuation.test.ts
new file mode 100644
index 00000000000..bc4e384bb89
--- /dev/null
+++ b/studio/frontend/tests/chat-thought-continuation.test.ts
@@ -0,0 +1,208 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+/** GGUF continuation seeds preserve reasoning and join it with new streamed deltas. */
+
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import {
+ continuationSeed,
+ isContinuableContent,
+ readContinuationRequest,
+ readContinuationSource,
+} from "../src/features/chat/utils/continuation.ts";
+import { parseAssistantContent } from "../src/features/chat/utils/parse-assistant-content.ts";
+import { createReasoningDurationTracker } from "../src/features/chat/utils/reasoning-duration.ts";
+
+import { readSrc } from "./helpers/kit.ts";
+
+const THOUGHT = "The user wants three primes. Small ones are";
+
+test("the source splits a reply into the answer and the thought before it", () => {
+ assert.deepEqual(
+ readContinuationSource([
+ { type: "reasoning", text: THOUGHT },
+ { type: "text", text: "2, 3" },
+ { type: "text", text: " and" },
+ ]),
+ { partial: "2, 3 and", reasoning: THOUGHT },
+ );
+ assert.deepEqual(readContinuationSource([{ type: "reasoning", text: THOUGHT }]), {
+ partial: "",
+ reasoning: THOUGHT,
+ });
+ // A thought after the answer cannot be replayed before it, so it is not carried.
+ assert.deepEqual(
+ readContinuationSource([
+ { type: "text", text: "2, 3" },
+ { type: "reasoning", text: "Maybe 5 too." },
+ { type: "text", text: " and 5" },
+ ]),
+ { partial: "2, 3 and 5", reasoning: "" },
+ );
+ assert.deepEqual(readContinuationSource(undefined), { partial: "", reasoning: "" });
+});
+
+test("a thought-only turn is continuable only where a thought resumes", () => {
+ const cut = [{ type: "reasoning", text: THOUGHT }];
+ assert.equal(isContinuableContent(cut), false);
+ assert.equal(isContinuableContent(cut, { thought: true }), true);
+ assert.equal(
+ isContinuableContent([{ type: "reasoning", text: " " }], { thought: true }),
+ false,
+ );
+ // A tool call still blocks it: the sibling would replay the call without its result.
+ assert.equal(
+ isContinuableContent([...cut, { type: "tool-call", toolName: "web_search" }], {
+ thought: true,
+ }),
+ false,
+ );
+});
+
+test("a thought-only request is read, and its duration only travels with a thought", () => {
+ assert.deepEqual(
+ readContinuationRequest({
+ custom: {
+ unslothContinuation: { partial: "", reasoning: THOUGHT, reasoningDuration: 7 },
+ },
+ }),
+ { partial: "", reasoning: THOUGHT, reasoningDuration: 7 },
+ );
+ assert.deepEqual(
+ readContinuationRequest({
+ custom: { unslothContinuation: { partial: "2, 3", reasoningDuration: 7 } },
+ }),
+ { partial: "2, 3" },
+ );
+ assert.equal(
+ readContinuationRequest({
+ custom: { unslothContinuation: { partial: "", reasoning: " \n" } },
+ }),
+ null,
+ );
+});
+
+/** What the adapter's stream loop appends, per delta, to a seeded buffer. */
+function stream(
+ seed: string,
+ openAtStart: boolean,
+ deltas: Array<{ reasoning?: string; content?: string }>,
+): string {
+ let text = seed;
+ let open = openAtStart;
+ for (const { reasoning, content } of deltas) {
+ if (reasoning) {
+ text += open ? reasoning : `<think>${reasoning}`;
+ open = true;
+ }
+ if (content) {
+ if (open) {
+ text += "</think>";
+ open = false;
+ }
+ text += content;
+ }
+ }
+ return open ? `${text}</think>` : text;
+}
+
+test("a resumed thought and its answer render as one thought and one answer", () => {
+ const seed = continuationSeed("", THOUGHT);
+ assert.equal(seed, `<think>${THOUGHT}`);
+ // Exactly what llama-server streamed for this request on b11160 (Qwen3.5-4B).
+ const final = stream(seed, true, [
+ { reasoning: " 2, 3, 5." },
+ { content: "2, 3 and 5." },
+ ]);
+ assert.deepEqual(parseAssistantContent(final), [
+ { type: "reasoning", text: `${THOUGHT} 2, 3, 5.` },
+ { type: "text", text: "2, 3 and 5." },
+ ]);
+});
+
+test("a continued answer keeps the thought it followed", () => {
+ const seed = continuationSeed("2, 3", "Easy.");
+ assert.deepEqual(parseAssistantContent(seed), [
+ { type: "reasoning", text: "Easy." },
+ { type: "text", text: "2, 3" },
+ ]);
+ assert.deepEqual(parseAssistantContent(stream(seed, false, [{ content: " and 5." }])), [
+ { type: "reasoning", text: "Easy." },
+ { type: "text", text: "2, 3 and 5." },
+ ]);
+ // Without a thought the seed is the partial, byte for byte, as before.
+ assert.equal(continuationSeed("2, 3 ", ""), "2, 3 ");
+});
+
+function clock(start = 1_000_000) {
+ let now = start;
+ return {
+ now: () => now,
+ advance: (seconds: number) => {
+ now += seconds * 1000;
+ },
+ };
+}
+
+test("a resumed thought keeps counting from the time it already took", () => {
+ const time = clock();
+ const tracker = createReasoningDurationTracker(time.now);
+ tracker.seedThought({ duration: 12, open: true, textLength: THOUGHT.length });
+ time.advance(3);
+ tracker.finishGroup();
+ assert.deepEqual(tracker.metadata(), { reasoningDuration: 15, reasoningDurations: [15] });
+});
+
+test("a server summary of the resumed tail does not replace the whole thought's time", () => {
+ const time = clock();
+ const tracker = createReasoningDurationTracker(time.now);
+ tracker.seedThought({ duration: 12, open: true, textLength: THOUGHT.length });
+ time.advance(3);
+ tracker.recordServerDuration(3000);
+ tracker.finishGroup();
+ assert.equal(tracker.metadata().reasoningDuration, 15);
+});
+
+test("a closed thought keeps its duration while the answer grows", () => {
+ const time = clock();
+ const tracker = createReasoningDurationTracker(time.now);
+ tracker.seedThought({ duration: 12, open: false, textLength: 5 });
+ time.advance(4);
+ // The adapter re-offers the group on every chunk; unchanged text must not reopen it.
+ tracker.resumeGroup(0, 5);
+ tracker.finishGroup();
+ assert.deepEqual(tracker.metadata(), { reasoningDuration: 12, reasoningDurations: [12] });
+ // A new thought after it is a group of its own.
+ tracker.startGroup();
+ time.advance(2);
+ tracker.finishGroup();
+ assert.deepEqual(tracker.metadata().reasoningDurations, [12, 2]);
+});
+
+test("a closed thought with no recorded time stays untimed", () => {
+ const time = clock();
+ const tracker = createReasoningDurationTracker(time.now);
+ tracker.seedThought({ duration: undefined, open: false, textLength: 5 });
+ time.advance(4);
+ tracker.resumeGroup(0, 5);
+ tracker.finishGroup();
+ assert.deepEqual(tracker.metadata(), {});
+});
+
+test("Continue response sits after Edit response and yields to the Resume bar", () => {
+ const thread = readSrc("components/assistant-ui/thread.tsx");
+ assert.match(
+ thread,
+ /\{inlineEdit && <EditAssistantMessageButton \/>\}\n\s*<ContinueResponseButton \/>\n\s*\{!researchRunId && !researchActive && \(\n\s*<ActionBarPrimitive\.Reload/,
+ );
+ // A stopped reply already offers Resume; a reply being edited continues from the saved text;
+ // a cited reply would lose its sources in the sibling.
+ assert.match(
+ thread,
+ /if \(!completed \|\| reason \|\| !canResume \|\| editing \|\| cited\) \{\n\s*return null;/,
+ );
+ // Both controls start the same run.
+ assert.equal(thread.match(/\[CONTINUATION_RUN_CONFIG_KEY\]/g)?.length, 1);
+});
diff --git a/studio/frontend/tests/composer-keystroke-subscription-budget.test.ts b/studio/frontend/tests/composer-keystroke-subscription-budget.test.ts
index c86a71bdcc2..893f901f518 100644
--- a/studio/frontend/tests/composer-keystroke-subscription-budget.test.ts
+++ b/studio/frontend/tests/composer-keystroke-subscription-budget.test.ts
@@ -115,14 +115,17 @@ test("the newest message still gets the whole continue bar", () => {
"const ContinueMessageBarForLastMessage: FC = () => {",
"\n};",
);
+ // The reads live in the hook the bar shares with the Continue response button.
+ assert.match(full, /useContinuation\(\);/);
+ const shared = body(thread, "function useContinuation() {", "\n}\n");
for (const marker of [
/useAuiState\(\(\{ message \}\) => message\.status\)/,
/useAuiState\(\(\{ message \}\) => message\.metadata\)/,
- /assistantMessageText\(message\.content\)/,
- /isContinuableContent\(message\.content\)/,
+ /readContinuationSource\(message\.content\)/,
+ /isContinuableContent\(message\.content, \{ thought: thoughtResumable \}\)/,
/findLatestUserAudioBase64\(thread\.messages, false\)/,
/modeAllowsContinuation\(\{/,
]) {
- assert.match(full, marker);
+ assert.match(shared, marker);
}
});
diff --git a/studio/frontend/tests/literal-think-when-thinking-off.test.ts b/studio/frontend/tests/literal-think-when-thinking-off.test.ts
index 1262783bc30..59eb4d41f22 100644
--- a/studio/frontend/tests/literal-think-when-thinking-off.test.ts
+++ b/studio/frontend/tests/literal-think-when-thinking-off.test.ts
@@ -165,7 +165,8 @@ test("the adapter and recovery follow the turn's think parse state", () => {
assert.match(adapter, /parseThink && thinkTags\.endsInsideThink\(\)/);
assert.match(
adapter,
- /setParseThink\(\s*isExternalRequest\s*\? requestParsesThinkTags\(externalReasoningFields\)\s*: reasoningAlwaysOn \|\| requestParsesThinkTags\(localReasoningFields\),\s*\);/,
+ // A resumed thought is reasoning whatever the request's own setting says.
+ /setParseThink\(\s*isExternalRequest\s*\? requestParsesThinkTags\(externalReasoningFields\)\s*: reasoningAlwaysOn \|\|\s*Boolean\(resumedThought\) \|\|\s*requestParsesThinkTags\(localReasoningFields\),\s*\);/,
"the live stream and recovery must read thinking off from the same request fields",
);
assert.match(adapter, /\.\.\.externalReasoningFields,/);