Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: flag responses that appear to stop while quoting a token (#12259)

oobaboogad654d718bf82
diff --git a/studio/backend/core/inference/chat_generation_runs.py b/studio/backend/core/inference/chat_generation_runs.py
index 2dcd6253c4f..953de0a5f28 100644
--- a/studio/backend/core/inference/chat_generation_runs.py
+++ b/studio/backend/core/inference/chat_generation_runs.py
@@ -703,6 +703,8 @@ class ChatGenerationSupervisor:
last_flush = time.monotonic()
finish_reason: str | None = None
error: str | None = None
+ # Sent on its own chunk, since the finish chunk still reads `stop`.
+ quote_cut = False
saw_done = False
worker_token: str | None = None
next_raw_task: asyncio.Task | None = None
@@ -828,6 +830,7 @@ class ChatGenerationSupervisor:
pending.append(("chunk", chunk, db.now_ms()))
finish_reason = _chunk_finish_reason(chunk) or finish_reason
error = _chunk_error(chunk) or error
+ quote_cut = quote_cut or chunk.get("quote_cut") is True
now = time.monotonic()
if (
len(pending) >= _EVENT_BATCH_SIZE
@@ -873,6 +876,7 @@ class ChatGenerationSupervisor:
finish_reason = finish_reason,
error = error,
pending_events = pending,
+ quote_cut = quote_cut,
)
pending = []
except asyncio.CancelledError:
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index 14c8ccff98e..e95fd762321 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -2150,6 +2150,40 @@ def _finalize_reasoning_only_cumulative(
return cumulative + "</think>" + visible_fallback
+def _ends_inside_quote(text: str) -> bool:
+ """Heuristic: text ends on a quote opener left unmatched on its line, or a lone quote."""
+ stripped = (text or "").rstrip()
+ if not stripped or stripped[-1] not in "`\"'":
+ return False
+ if len(stripped) > 1 and not (stripped[-2].isspace() or stripped[-2] in "*_([{"):
+ return False
+ last = stripped[-1]
+ line = stripped.rsplit("\n", 1)[-1]
+ # An apostrophe between letters (Qwen2's, don't) is not a quote mark.
+ marks = sum(
+ 1
+ for i, ch in enumerate(line)
+ if ch == last
+ and not (
+ ch == "'" and 0 < i < len(line) - 1 and line[i - 1].isalnum() and line[i + 1].isalnum()
+ )
+ )
+ return marks % 2 == 1
+
+
+def _quote_cut_event(
+ reasoning_text: str,
+ answer_text: str,
+ finish_reason: Optional[str],
+ promote_reasoning_only: bool,
+) -> Optional[dict]:
+ """Flag a mid-quote `stop` in the shown answer; off for Anthropic, which cannot relay it."""
+ if not promote_reasoning_only or finish_reason != "stop":
+ return None
+ shown = answer_text if answer_text.strip() else reasoning_text
+ return {"type": "quote_cut"} if _ends_inside_quote(shown) else None
+
+
# Only large streamed tool payloads get an early provisional card; render_html
# is exempt because it needs immediate artifact feedback.
_PROVISIONAL_ARGS_MIN_CHARS = 256
@@ -35415,6 +35449,8 @@ class LlamaCppBackend:
):
buffer = ""
has_content_tokens = False
+ # Track content separately: a literal `</think>` can appear in the answer.
+ answer_text = ""
reasoning_text = ""
_prov_entry = None
for raw_chunk in self._iter_text_cancellable(
@@ -35512,6 +35548,7 @@ class LlamaCppBackend:
token = delta.get("content", "")
if token:
has_content_tokens = True
+ answer_text += token
if in_thinking:
cumulative += "</think>"
in_thinking = False
@@ -35522,6 +35559,14 @@ class LlamaCppBackend:
logger.debug(f"Skipping malformed SSE line: {line[:100]}")
if _stream_done:
break # exit outer for
+ _cut = _quote_cut_event(
+ reasoning_text,
+ answer_text,
+ _metadata_finish_reason,
+ promote_reasoning_only,
+ )
+ if _cut is not None:
+ yield _cut
if _metadata_usage or _metadata_timings or _metadata_finish_reason:
_metadata_usage = _backfill_usage_from_timings(
_metadata_usage, _metadata_timings
@@ -37648,6 +37693,14 @@ class LlamaCppBackend:
# Content was already streamed. Yield metadata.
yield {"type": "status", "text": ""}
+ _cut = _quote_cut_event(
+ reasoning_accum,
+ _visible,
+ _iter_finish_reason,
+ promote_reasoning_only,
+ )
+ if _cut is not None:
+ yield _cut
_meta = _build_metadata_event(
_iter_usage, _iter_timings, _iter_finish_reason
)
@@ -39223,6 +39276,8 @@ class LlamaCppBackend:
in_thinking = False
has_content_tokens = False
reasoning_text = ""
+ # Track content separately to preserve literal `</think>` in the answer.
+ answer_text = ""
_prov_entry = None
_final_reasoning_started_at: Optional[float] = None
_final_reasoning_summary_emitted = False
@@ -39439,6 +39494,7 @@ class LlamaCppBackend:
_final_reasoning_started_at
)
has_content_tokens = True
+ answer_text += token
if in_thinking:
cumulative += "</think>"
in_thinking = False
@@ -39750,6 +39806,14 @@ class LlamaCppBackend:
if _meta is not None:
yield _meta
else:
+ _cut = _quote_cut_event(
+ reasoning_text,
+ answer_text,
+ _metadata_finish_reason,
+ promote_reasoning_only,
+ )
+ if _cut is not None:
+ yield _cut
_meta = _build_metadata_event(
_metadata_usage, _metadata_timings, _metadata_finish_reason
)
diff --git a/studio/backend/core/inference/sse_control_frames.py b/studio/backend/core/inference/sse_control_frames.py
index 699050dd253..c9c5563e01d 100644
--- a/studio/backend/core/inference/sse_control_frames.py
+++ b/studio/backend/core/inference/sse_control_frames.py
@@ -50,6 +50,7 @@ _CONTROL_KEYS = (
"_diffusionFrame",
"_reasoningDurationMs",
"_mcp_provenance",
+ "quote_cut",
)
# A stripped frame is only worth relaying if it still says something in the provider's own vocabulary.
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 8e31eaf4c8e..4efb5953582 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -965,6 +965,19 @@ def _context_truncated_sse_chunk(completion_id: str, model_name: str, truncation
return f"data: {json.dumps(data)}\n\n"
+def _quote_cut_sse_chunk(completion_id: str, model_name: str) -> str:
+ # Report the warning separately to preserve llama-server's finish reason.
+ data = {
+ "id": completion_id,
+ "object": "chat.completion.chunk",
+ "created": int(time.time()),
+ "model": model_name,
+ "choices": [],
+ "quote_cut": True,
+ }
+ return f"data: {json.dumps(data)}\n\n"
+
+
def _accumulate_context_truncation(current: Optional[dict], event: dict) -> dict:
incoming = {key: value for key, value in event.items() if key != "type"}
# The drains accumulate rather than forwarding each event, so record here. The per-fit
@@ -27874,6 +27887,14 @@ async def produce_openai_chat_completions(
_stream_finish = event.get("finish_reason")
continue
+ if event["type"] == "quote_cut":
+ # Bypass content handling to avoid resetting the text cursor.
+ if _ui_events:
+ yield _quote_cut_sse_chunk(completion_id, model_name)
+ elif _drop_keepalive.due():
+ yield _OPENAI_PASSTHROUGH_SSE_KEEPALIVE
+ continue
+
if event["type"] == "reasoning_summary":
# Forward server-side reasoning timing to the UI.
if _ui_events:
@@ -28491,6 +28512,11 @@ async def produce_openai_chat_completions(
_stream_usage = cumulative.get("usage")
_stream_timings = cumulative.get("timings")
_stream_finish = cumulative.get("finish_reason")
+ elif cumulative.get("type") == "quote_cut":
+ if _ui_events:
+ yield _quote_cut_sse_chunk(completion_id, model_name)
+ elif _drop_keepalive.due():
+ yield _OPENAI_PASSTHROUGH_SSE_KEEPALIVE
elif cumulative.get("type") == "diffusion_frame":
# Diffusion frame (per-step canvas): pass through as a raw SSE line on the
# tool_status channel. No assistant text, so it never enters the cumulative diff.
diff --git a/studio/backend/storage/chat_generation_runs_db.py b/studio/backend/storage/chat_generation_runs_db.py
index 0244c10561c..8d784631ea2 100644
--- a/studio/backend/storage/chat_generation_runs_db.py
+++ b/studio/backend/storage/chat_generation_runs_db.py
@@ -490,7 +490,12 @@ def _commit(conn: sqlite3.Connection, *, notify: bool = False) -> None:
_EVENTS_CHANGED.notify_all()
-def _sync_assistant_status_locked(conn: sqlite3.Connection, run_id: str, status: str) -> None:
+def _sync_assistant_status_locked(
+ conn: sqlite3.Connection,
+ run_id: str,
+ status: str,
+ quote_cut: bool = False,
+) -> None:
row = conn.execute(
"""SELECT r.assistant_message_id, r.finish_reason, m.metadata_json
FROM chat_generation_runs r
@@ -517,6 +522,8 @@ def _sync_assistant_status_locked(conn: sqlite3.Connection, run_id: str, status:
elif status == "completed":
if row["finish_reason"] == "length":
metadata["incomplete"] = {"reason": "length"}
+ elif quote_cut:
+ metadata["incomplete"] = {"reason": "quote_cut"}
else:
metadata.pop("incomplete", None)
conn.execute(
@@ -922,6 +929,7 @@ def finish_run(
finish_reason: str | None = None,
error: str | None = None,
pending_events: Iterable[ChatGenerationEventInput] = (),
+ quote_cut: bool = False,
) -> dict[str, Any] | None:
if status not in TERMINAL_STATUSES:
raise ValueError(f"Invalid terminal status: {status}")
@@ -957,7 +965,7 @@ def finish_run(
WHERE id=?""",
(status, finish_reason, error, completed, completed, run_id),
)
- _sync_assistant_status_locked(conn, run_id, status)
+ _sync_assistant_status_locked(conn, run_id, status, quote_cut)
updated = conn.execute(
"SELECT * FROM chat_generation_runs WHERE id=?",
(run_id,),
diff --git a/studio/backend/tests/test_chat_generation_supervisor.py b/studio/backend/tests/test_chat_generation_supervisor.py
index d13517504ed..2829e533a3e 100644
--- a/studio/backend/tests/test_chat_generation_supervisor.py
+++ b/studio/backend/tests/test_chat_generation_supervisor.py
@@ -291,6 +291,45 @@ async def test_background_producer_persists_chunks_and_completes(durable_run, mo
] == chunks
+ "finish, cut, expected",
+ [
+ ("stop", True, {"reason": "quote_cut"}),
+ ("stop", False, None),
+ # Length takes precedence over the heuristic.
+ ("length", True, {"reason": "length"}),
+ ],
+ ids = ["cut", "clean-stop", "length-wins"],
+)
+async def test_producer_stamps_a_reported_quote_cut(
+ durable_run, monkeypatch, finish, cut, expected
+):
+ """The producer must persist the reason before the client can settle with it."""
+ chunks = [
+ {"choices": [{"delta": {"reasoning_content": "The tokens are `"}, "finish_reason": None}]},
+ *([{"choices": [], "quote_cut": True}] if cut else []),
+ {"choices": [{"delta": {}, "finish_reason": finish}]},
+ ]
+
+ async def body():
+ for chunk in chunks:
+ yield f"data: {json.dumps(chunk)}\n\n"
+ yield "data: [DONE]\n\n"
+
+ async def fake(_payload, _request, _subject, *, cancel_on_disconnect):
+ return SimpleNamespace(status_code = 200, body_iterator = body())
+
+ monkeypatch.setattr(inference, "produce_openai_chat_completions", fake)
+ supervisor = ChatGenerationSupervisor(SimpleNamespace(state = SimpleNamespace()))
+ await supervisor._produce("run-1")
+
+ run = runs_db.get_run("run-1", "alice")
+ assert (run["status"], run["finishReason"]) == ("completed", finish)
+ message = studio_db.get_chat_message("thread-1", "assistant-1")
+ assert message["metadata"].get("incomplete") == expected
+
+
@pytest.mark.asyncio
async def test_producer_dates_the_prompt_in_the_browser_timezone(monkeypatch):
studio_db.upsert_chat_thread(
diff --git a/studio/backend/tests/test_provider_control_frame_spoofing.py b/studio/backend/tests/test_provider_control_frame_spoofing.py
index 8e601b9432f..72dab08a86f 100644
--- a/studio/backend/tests/test_provider_control_frame_spoofing.py
+++ b/studio/backend/tests/test_provider_control_frame_spoofing.py
@@ -55,6 +55,7 @@ def test_every_control_type_is_dropped(frame_type):
"_diffusionFrame",
"_reasoningDurationMs",
"_mcp_provenance",
+ "quote_cut",
],
)
def test_every_studio_private_key_is_stripped(key):
diff --git a/studio/backend/tests/test_quote_cut.py b/studio/backend/tests/test_quote_cut.py
new file mode 100644
index 00000000000..54fd24441ca
--- /dev/null
+++ b/studio/backend/tests/test_quote_cut.py
@@ -0,0 +1,321 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Detect possible mid-quote stops without changing response text or finish reasons."""
+
+from __future__ import annotations
+
+import ast
+import contextlib
+import inspect
+import json
+import sys
+from pathlib import Path
+
+import pytest
+
+_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)
+if _BACKEND_DIR not in sys.path:
+ sys.path.insert(0, _BACKEND_DIR)
+
+from core.inference.llama_cpp import (
+ LlamaCppBackend,
+ _ends_inside_quote,
+ _quote_cut_event,
+)
+from routes.inference import _quote_cut_sse_chunk, produce_openai_chat_completions
+
+# Reported and observed Qwen3.8 cuts, plus each other opener context.
+_CUT_TAILS = [
+ "Actually I need to check whether the actual file contains `",
+ "that's actually the canonical Qwen behavior (keeps `",
+ "Note the special tokens (in the actual Qwen3, the tokens are `",
+ "```\nActually Qwen1.5/2? tokens: `",
+ "might be `<|im_start|>` but in tokenizer, the token ID string may be `",
+ "Let's recall Qwen2 tokenizer: added tokens include:\n\"",
+ 'Need consider if Qwen3 uses "',
+ 'Could be special token "\n',
+ "Character by character:\n\n- `<` `|` `i` `",
+ '"',
+ "Qwen2's chat template uses `{{- '",
+ 'The token is:\n\n```python\n"',
+ "The special tokens that open and close each turn are **`",
+ "the tokens (`",
+ "the _`",
+ "tokens: [`",
+ '{"',
+]
+
+# Ways finished text ends, including ones that end on a quote or backtick.
+_FINISHED_TAILS = [
+ "The answer is 4.",
+ "Four",
+ "Use `print()` to show it.",
+ "def f():\n return 1\n```",
+ "```python",
+ 'He said "hello".',
+ 'the sum of the two preceding ones (e.g., $3 + 5 = 8$)."',
+ "the students'",
+ '```python\n"<|im_end|>"\n```',
+ '"<|im_end|>"',
+ "`<|im_start|>`\n`<|im_end|>`",
+ "It is called a backtick, and here it is: **`**",
+ "Use `x` for inline code ```",
+ 'He called it "**bold**"',
+ "{}",
+ "",
+ " \n",
+]
+
+
[email protected]("tail", _CUT_TAILS)
+def test_text_cut_inside_an_opened_quote_is_detected(tail):
+ assert _ends_inside_quote("Some earlier text.\n\n" + tail)
+
+
[email protected]("tail", _FINISHED_TAILS)
+def test_finished_text_is_not(tail):
+ assert not _ends_inside_quote(tail)
+
+
+_CUT = _CUT_TAILS[0]
+_DONE = "The answer is 4."
+
+
+ "reasoning, answer, finish, promote, expected",
+ [
+ (_CUT, "", "stop", True, True),
+ (_DONE, _CUT, "stop", True, True),
+ (_DONE, "", "stop", True, False),
+ (_DONE, _DONE, "stop", True, False),
+ # A complete answer overrides a suspicious reasoning tail.
+ (_CUT, _DONE, "stop", True, False),
+ # `length` has its own recovery and terminal state.
+ (_CUT, _CUT, "length", True, False),
+ (_CUT, _CUT, None, True, False),
+ # The Anthropic route does not forward the event.
+ (_CUT, _CUT, "stop", False, False),
+ ],
+ ids = [
+ "cut-thought",
+ "cut-answer",
+ "finished-thought",
+ "finished-answer",
+ "cut-thought-answered",
+ "length",
+ "no-finish",
+ "anthropic",
+ ],
+)
+def test_the_event_reads_the_answer_else_the_promoted_thought(
+ reasoning, answer, finish, promote, expected
+):
+ event = _quote_cut_event(reasoning, answer, finish, promote)
+ assert event == ({"type": "quote_cut"} if expected else None)
+
+
+def test_the_sse_chunk_is_an_empty_chunk_carrying_the_flag():
+ line = _quote_cut_sse_chunk("chatcmpl-1", "qwen")
+ assert line.startswith("data: ") and line.endswith("\n\n")
+ data = json.loads(line[len("data: ") :])
+ assert data["choices"] == []
+ assert data["quote_cut"] is True
+ assert data["object"] == "chat.completion.chunk"
+
+
+def test_both_gguf_streams_forward_the_flag_only_to_the_ui():
+ tree = ast.parse(inspect.getsource(produce_openai_chat_completions))
+ forwards = [
+ node
+ for node in ast.walk(tree)
+ if isinstance(node, ast.If)
+ and "_ui_events" in ast.dump(node.test)
+ and "_quote_cut_sse_chunk" in ast.dump(ast.Module(node.body, []))
+ ]
+ assert len(forwards) == 2
+
+
+def _sse(delta: dict) -> str:
+ return "data: " + json.dumps({"choices": [{"index": 0, "delta": delta}]}) + "\n"
+
+
+def _finish(reason: str) -> str:
+ return (
+ "data: "
+ + json.dumps({"choices": [{"index": 0, "delta": {}, "finish_reason": reason}]})
+ + "\n"
+ )
+
+
+def _done() -> str:
+ return "data: [DONE]\n"
+
+
+def _make_backend(monkeypatch, streams: list[list[str]]):
+ backend = LlamaCppBackend.__new__(LlamaCppBackend)
+ backend._process = object()
+ backend._healthy = True
+ backend._port = 48853
+ backend._api_key = None
+ backend._effective_context_length = 4096
+ backend._supports_reasoning = True
+ backend._reasoning_always_on = False
+ backend._reasoning_style = "enable_thinking"
+ backend._supports_preserve_thinking = False
+
+ @contextlib.contextmanager
+ def fake_stream_with_retry(
+ _client,
+ _url,
+ _payload,
+ _cancel_event,
+ headers = None,
+ first_token_deadline = None,
+ ):
+ yield type("FakeResponse", (), {"status_code": 200, "chunks": streams.pop(0)})()
+
+ def fake_iter_text_cancellable(
+ response,
+ _cancel_event,
+ first_token_deadline = None,
+ ):
+ yield from response.chunks
+
+ monkeypatch.setattr(backend, "_stream_with_retry", fake_stream_with_retry)
+ monkeypatch.setattr(backend, "_iter_text_cancellable", fake_iter_text_cancellable)
+ monkeypatch.setattr(backend, "_maybe_recover_from_mtp_crash", lambda *_a, **_k: False)
+ return backend
+
+
+_PREFIX = "Compare the two templates. " * 20
+_CUT_THOUGHT = _PREFIX + "In Qwen3 the tokens are `"
+_WHOLE_REPLY_IN_THOUGHT = _PREFIX + "They differ only in spacing."
+_CUT_ANSWER = "The exact strings are:\n\n- `<` `|` `i` `"
+_FINISHED_ANSWER = "The exact strings are `<|im_start|>` and `<|im_end|>`."
+# Literal `</think>` in an answer must not be parsed as a reasoning boundary.
+_CUT_ANSWER_QUOTING_CLOSER = "The tokens are `</think>` and `"
+_FINISHED_ANSWER_QUOTING_CLOSER = "The closing tag is `</think>`"
+
+_SEARCH_TOOL = {
+ "type": "function",
+ "function": {
+ "name": "web_search",
+ "description": "Search the web.",
+ "parameters": {
+ "type": "object",
+ "properties": {"query": {"type": "string"}},
+ "required": ["query"],
+ },
+ },
+}
+
+
+def _stream(
+ thought: str = "",
+ content: str = "",
+ finish: str = "stop",
+) -> list[str]:
+ chunks = [_sse({"reasoning_content": thought})] if thought else []
+ if content:
+ chunks.append(_sse({"content": content}))
+ return chunks + [_finish(finish), _done()]
+
+
+def _plain(backend, **kwargs) -> list:
+ return list(
+ backend.generate_chat_completion(
+ messages = [{"role": "user", "content": "compare these templates"}],
+ enable_thinking = True,
+ **kwargs,
+ )
+ )
+
+
+def _tool_loop(backend, **kwargs) -> list:
+ return list(
+ backend.generate_chat_completion_with_tools(
+ messages = [{"role": "user", "content": "compare these templates"}],
+ tools = [_SEARCH_TOOL],
+ max_tool_iterations = 3,
+ enable_thinking = True,
+ **kwargs,
+ )
+ )
+
+
+def _final_pass(backend, **kwargs) -> list:
+ """The pass taken once the tool loop is done."""
+ return list(
+ backend.generate_chat_completion_with_tools(
+ messages = [{"role": "user", "content": "compare these templates"}],
+ tools = [],
+ max_tool_iterations = 0,
+ enable_thinking = True,
+ **kwargs,
+ )
+ )
+
+
+_EXITS = {"plain": _plain, "tool-loop": _tool_loop, "final-pass": _final_pass}
+
+_FLAGGED = {
+ "cut-thought": _stream(_CUT_THOUGHT),
+ "cut-answer-after-thought": _stream(_WHOLE_REPLY_IN_THOUGHT, _CUT_ANSWER),
+ "cut-answer-thinking-off": _stream(content = _CUT_ANSWER),
+ "cut-answer-quoting-closer": _stream(_WHOLE_REPLY_IN_THOUGHT, _CUT_ANSWER_QUOTING_CLOSER),
+ "cut-answer-quoting-closer-thinking-off": _stream(content = _CUT_ANSWER_QUOTING_CLOSER),
+}
+
+_UNFLAGGED = {
+ "whole-reply-in-thought": (_stream(_WHOLE_REPLY_IN_THOUGHT), {}),
+ "finished-answer": (_stream(_CUT_THOUGHT, _FINISHED_ANSWER), {}),
+ "finished-answer-thinking-off": (_stream(content = _FINISHED_ANSWER), {}),
+ "answer-quoting-closer": (
+ _stream(_WHOLE_REPLY_IN_THOUGHT, _FINISHED_ANSWER_QUOTING_CLOSER),
+ {},
+ ),
+ "answer-quoting-closer-thinking-off": (
+ _stream(content = _FINISHED_ANSWER_QUOTING_CLOSER),
+ {},
+ ),
+ "length": (_stream(_CUT_THOUGHT, _CUT_ANSWER, finish = "length"), {}),
+ "anthropic": (_stream(_CUT_THOUGHT), {"promote_reasoning_only": False}),
+}
+
+
+# A `length` turn is continued, so it needs a second stream to finish on.
+_SPARE = _stream(content = _FINISHED_ANSWER)
+
+
+def _events(kind) -> list:
+ return [e for e in kind if isinstance(e, dict)]
+
+
[email protected]("exit_name", list(_EXITS))
[email protected]("case", list(_FLAGGED))
+def test_every_exit_reports_a_cut_before_its_metadata(monkeypatch, exit_name, case):
+ events = _events(_EXITS[exit_name](_make_backend(monkeypatch, [list(_FLAGGED[case]), _SPARE])))
+
+ kinds = [e.get("type") for e in events]
+ assert kinds.count("quote_cut") == 1
+ assert kinds.index("quote_cut") < kinds.index("metadata")
+
+
[email protected]("exit_name", list(_EXITS))
[email protected]("case", list(_UNFLAGGED))
+def test_every_exit_leaves_other_endings_alone(monkeypatch, exit_name, case):
+ stream, kwargs = _UNFLAGGED[case]
+ events = _events(
+ _EXITS[exit_name](_make_backend(monkeypatch, [list(stream), _SPARE]), **kwargs)
+ )
+
+ assert not [e for e in events if e.get("type") == "quote_cut"]
+
+
+def test_the_shown_text_is_unchanged(monkeypatch):
+ texts = [
+ e for e in _plain(_make_backend(monkeypatch, [_stream(_CUT_THOUGHT)])) if isinstance(e, str)
+ ]
+ # The warning preserves the promoted thought.
+ assert texts[-1].endswith("</think>" + _CUT_THOUGHT)
diff --git a/studio/frontend/src/features/chat/api/chat-adapter.ts b/studio/frontend/src/features/chat/api/chat-adapter.ts
index e6fa4647312..661ad7a4086 100644
--- a/studio/frontend/src/features/chat/api/chat-adapter.ts
+++ b/studio/frontend/src/features/chat/api/chat-adapter.ts
@@ -5487,6 +5487,7 @@ export function createOpenAIStreamAdapter(
buildAssistantContent(mergeContinuation(cumulativeText));
// Declared above the live metadata that reads it, or it is in its temporal dead zone.
let contextWindowExceeded = false;
+ let quoteCut = false;
// Provisional reason on every streamed yield: an abort skips the terminal yields and a reload
// rebuilds messages as "complete". Stop is only the guess; a reported window outranks it.
const liveCustom = () => ({
@@ -6787,6 +6788,11 @@ export function createOpenAIStreamAdapter(
continue;
}
+ if (chunk.quote_cut) {
+ quoteCut = true;
+ continue;
+ }
+
if (chunk.context_truncated) {
contextTruncation = mergeContextTruncation(
contextTruncation,
@@ -8155,6 +8161,7 @@ export function createOpenAIStreamAdapter(
];
const finalIncompleteReason =
resolveIncompleteReason(incompleteReason, contextWindowExceeded) ??
+ (quoteCut ? "quote_cut" : null) ??
// A run can stop cleanly on its first token and leave nothing behind.
// Saved as complete that is a blank bubble with no way out.
(hasRenderableContent(finalContent) ? null : "empty");
diff --git a/studio/frontend/src/features/chat/runtime-provider.tsx b/studio/frontend/src/features/chat/runtime-provider.tsx
index 6eacab2de00..abeeceb7e31 100644
--- a/studio/frontend/src/features/chat/runtime-provider.tsx
+++ b/studio/frontend/src/features/chat/runtime-provider.tsx
@@ -1219,6 +1219,10 @@ function scheduleGenerationRecovery(
: undefined;
let totalChunks = Number(metadata.generationChunkCount ?? 0);
if (!Number.isSafeInteger(totalChunks) || totalChunks < 0) totalChunks = 0;
+ // The cut arrives on its own chunk, which a resumed cursor may already be past.
+ let quoteCut =
+ (metadata.incomplete as { reason?: unknown } | undefined)?.reason ===
+ "quote_cut";
let currentMetadata = { ...metadata };
const serverCancel = () => {
void cancelChatGenerationRun(runId).catch(() => {});
@@ -1322,6 +1326,7 @@ function scheduleGenerationRecovery(
cursor,
lastEventSeq: run.lastEventSeq,
lengthLimited,
+ quoteCut,
firstChunkAt,
totalChunks,
usage: recoveryUsage,
@@ -1442,6 +1447,7 @@ function scheduleGenerationRecovery(
};
}>;
context_truncated?: OpenAIChatChunk["context_truncated"];
+ quote_cut?: boolean;
};
if ("_reasoningDurationMs" in chunk) {
currentMetadata = recoveredReasoningSummaryMetadata(
@@ -1467,6 +1473,7 @@ function scheduleGenerationRecovery(
),
};
}
+ if (chunk.quote_cut === true) quoteCut = true;
if (chunk.usage) recoveryUsage = chunk.usage;
if (chunk.timings) recoveryTimings = chunk.timings;
if (typeof chunk.usage?.completion_tokens === "number") {
diff --git a/studio/frontend/src/features/chat/types/api.ts b/studio/frontend/src/features/chat/types/api.ts
index 73060e561b4..029491032ef 100644
--- a/studio/frontend/src/features/chat/types/api.ts
+++ b/studio/frontend/src/features/chat/types/api.ts
@@ -735,6 +735,8 @@ export interface OpenAIChatChunk {
total_tokens: number;
};
timings?: Record<string, number>;
+ /** Studio heuristic: the response may have stopped mid-quote. */
+ quote_cut?: boolean;
context_truncated?: {
dropped_messages: number;
prompt_tokens_before?: number;
diff --git a/studio/frontend/src/features/chat/utils/chat-generation-recovery.ts b/studio/frontend/src/features/chat/utils/chat-generation-recovery.ts
index c2b96d754a6..0c00f4f8f23 100644
--- a/studio/frontend/src/features/chat/utils/chat-generation-recovery.ts
+++ b/studio/frontend/src/features/chat/utils/chat-generation-recovery.ts
@@ -49,13 +49,16 @@ export function generationChunkCountsTowardTiming(payload: unknown): boolean {
| {
_reasoningDurationMs?: unknown;
context_truncated?: unknown;
+ quote_cut?: unknown;
usage?: unknown;
choices?: unknown[];
}
| null
| undefined;
if (!chunk || typeof chunk !== "object") return false;
- if ("_reasoningDurationMs" in chunk || chunk.context_truncated) return false;
+ if ("_reasoningDurationMs" in chunk || chunk.context_truncated || chunk.quote_cut) {
+ return false;
+ }
return !(chunk.usage && Array.isArray(chunk.choices) && chunk.choices.length === 0);
}
@@ -564,6 +567,7 @@ export function generationRecoveryMetadata(options: {
cursor: number;
lastEventSeq: number;
lengthLimited: boolean;
+ quoteCut?: boolean;
firstChunkAt?: number;
totalChunks?: number;
usage?: unknown;
@@ -576,6 +580,7 @@ export function generationRecoveryMetadata(options: {
cursor,
lastEventSeq,
lengthLimited,
+ quoteCut = false,
firstChunkAt,
totalChunks,
usage,
@@ -593,6 +598,9 @@ export function generationRecoveryMetadata(options: {
if (status === "completed") {
if (lengthLimited) {
next.incomplete = { reason: "length" };
+ } else if (quoteCut) {
+ // Must match the producer's stamp, or the server refuses the settle.
+ next.incomplete = { reason: "quote_cut" };
} else {
next.incomplete = undefined;
}
diff --git a/studio/frontend/src/features/chat/utils/continuation.ts b/studio/frontend/src/features/chat/utils/continuation.ts
index 36f2a1592f9..d1ffcf9e044 100644
--- a/studio/frontend/src/features/chat/utils/continuation.ts
+++ b/studio/frontend/src/features/chat/utils/continuation.ts
@@ -7,13 +7,15 @@
/** Why a turn ended before the model was done. `context_window` is a `length` cut the same
* request can never fit into, hence its own reason. `empty` is a clean finish that produced
- * nothing, which is a failure to report rather than an answer. */
+ * nothing, which is a failure to report rather than an answer. `quote_cut` flags a
+ * possible mid-quote stop. */
export type IncompleteReason =
| "length"
| "cancelled"
| "interrupted"
| "context_window"
- | "empty";
+ | "empty"
+ | "quote_cut";
/** Metadata stamped on an assistant message that stopped early. */
export type IncompleteInfo = {
@@ -37,6 +39,7 @@ const INCOMPLETE_REASONS: readonly IncompleteReason[] = [
"interrupted",
"context_window",
"empty",
+ "quote_cut",
];
/** Below this a shared boundary is likely coincidence, and trimming would eat output. */
@@ -96,6 +99,7 @@ const STATUS_REASON: Record<
// reason, losing the explanation on reload. `context_window` maps here for the same
// reason. Not `error` either, which would paint a red box over the bar.
empty: "length",
+ quote_cut: "length",
};
/** Restore assistant-ui's status without losing the product-specific stop reason. */
@@ -115,6 +119,7 @@ const INCOMPLETE_LABELS: Record<IncompleteReason, string> = {
interrupted: "Response interrupted",
context_window: "Response filled the model's context window",
empty: "The model returned an empty response",
+ quote_cut: "This response may have ended early",
};
/** The user-facing explanation of why a turn stopped. */
@@ -128,6 +133,9 @@ const INCOMPLETE_REMEDIES: Partial<Record<IncompleteReason, string>> = {
context_window: "Start a new chat, or shorten this one, to keep going",
// There is no partial to resume from, so the way out is another attempt.
empty: "Try again, or pick a different model",
+ // Detection is heuristic, and continuation may repeat the cut.
+ quote_cut:
+ "The model may have emitted a special token while quoting it. Write special tokens with a space inside, like < |im_end|> or < end_of_turn>, ask the model to do the same, and try again",
};
/** What to do about a turn that stopped early, or `null` when resuming is the answer. */
diff --git a/studio/frontend/tests/chat-continuation.test.ts b/studio/frontend/tests/chat-continuation.test.ts
index 749b3c6353b..3da219a3d79 100644
--- a/studio/frontend/tests/chat-continuation.test.ts
+++ b/studio/frontend/tests/chat-continuation.test.ts
@@ -264,6 +264,39 @@ test("the adapter marks a finish that rendered nothing", () => {
assert.match(ending, /yield \{\s*content: finalContent,/);
});
+test("a reply cut mid-quote is reported, and offers a retry rather than a resume", () => {
+ // A lone backtick or quote can be intentional, so the warning stays tentative.
+ assert.equal(incompleteLabel("quote_cut"), "This response may have ended early");
+ assert.match(incompleteRemedy("quote_cut") ?? "", /may have emitted a special token/);
+ assert.match(incompleteRemedy("quote_cut") ?? "", /< \|im_end\|>/);
+ assert.deepEqual(
+ readIncompleteInfo({ custom: { incomplete: { reason: "quote_cut" } } }),
+ { reason: "quote_cut" },
+ );
+ assert.equal(shouldAutoContinue("quote_cut", "parent-1"), false);
+ // A cancelled status would hide the label.
+ assert.deepEqual(
+ restoredAssistantStatus({ custom: { incomplete: { reason: "quote_cut" } } }),
+ { type: "incomplete", reason: "length" },
+ );
+});
+
+test("the adapter stamps a cut the backend reported on a clean finish", () => {
+ assert.match(
+ CHAT_ADAPTER,
+ /if \(chunk\.quote_cut\) \{\s*quoteCut = true;\s*continue;\s*\}/,
+ );
+ const ending = CHAT_ADAPTER.slice(
+ CHAT_ADAPTER.indexOf("const finalIncompleteReason ="),
+ );
+ const reason = ending.slice(0, ending.indexOf("yield {"));
+ // After the reported finish (`length` wins) and before the empty-turn fallback.
+ assert.match(
+ reason,
+ /resolveIncompleteReason\(incompleteReason, contextWindowExceeded\) \?\?\s*\(quoteCut \? "quote_cut" : null\) \?\?/,
+ );
+});
+
test("the provider's own reason outranks every reason the client infers", () => {
// The event ends Anthropic's turn, so the model has already stopped. A null reason
// matters most: it reads as a completed answer.
diff --git a/studio/frontend/tests/chat-runtime-reload-status.test.ts b/studio/frontend/tests/chat-runtime-reload-status.test.ts
index 5cf8317e42f..0888fada3a9 100644
--- a/studio/frontend/tests/chat-runtime-reload-status.test.ts
+++ b/studio/frontend/tests/chat-runtime-reload-status.test.ts
@@ -48,10 +48,11 @@ test("first-token recovery ignores role and control chunks", () => {
[
{ choices: [{ delta: { role: "assistant" } }] },
{ context_truncated: { checkpoint: true } },
+ { choices: [], quote_cut: true },
{ choices: [], usage: { completion_tokens: 1 } },
{ choices: [{ delta: { content: "token" } }] },
].map(generationChunkCountsTowardTiming),
- [true, false, false, true],
+ [true, false, false, false, true],
);
assert.equal(
generationChunkHasSubstantiveDelta({
@@ -239,6 +240,21 @@ test("reload, wake, and stale-tab recovery stays monotonic and truthful", () =>
],
);
+ // Recovery must match the producer's reason, with length taking precedence.
+ const recovered = (lengthLimited: boolean, quoteCut: boolean) =>
+ generationRecoveryMetadata({
+ current: { generationRunId: "run-1" },
+ runId: "run-1",
+ status: "completed",
+ cursor: 4,
+ lastEventSeq: 4,
+ lengthLimited,
+ quoteCut,
+ }).incomplete;
+ assert.deepEqual(recovered(false, true), { reason: "quote_cut" });
+ assert.deepEqual(recovered(true, true), { reason: "length" });
+ assert.equal(recovered(false, false), undefined);
+
const windowTarget = new EventTarget();
const documentTarget = Object.assign(new EventTarget(), {
visibilityState: "hidden",
diff --git a/studio/frontend/tests/generation-tool-recovery.test.ts b/studio/frontend/tests/generation-tool-recovery.test.ts
index 62653f0b378..dbfc069b040 100644
--- a/studio/frontend/tests/generation-tool-recovery.test.ts
+++ b/studio/frontend/tests/generation-tool-recovery.test.ts
@@ -320,7 +320,11 @@ const executable = ts.transpileModule(scheduler, {
async function recoverRun(
content: unknown[],
payloads: unknown[],
- options: { cursor?: number; viewContent?: unknown[] } = {},
+ options: {
+ cursor?: number;
+ viewContent?: unknown[];
+ metadata?: Record<string, unknown>;
+ } = {},
) {
let shown = {
messages: [
@@ -403,6 +407,7 @@ async function recoverRun(
generationSeq: options.cursor ?? 0,
generationStatus: "running",
generationSettled: false,
+ ...options.metadata,
},
},
{
@@ -454,6 +459,36 @@ test("the recovery scheduler persists later tool events between reasoning groups
);
});
+test("a recovered turn keeps the reasoning cut the backend reported", async () => {
+ // The producer stamps the cut on the saved turn; a settle without it is refused.
+ const { snapshots } = await recoverRun(
+ [],
+ [
+ { choices: [{ delta: { reasoning_content: "The tokens are `" } }] },
+ { choices: [], quote_cut: true },
+ { choices: [{ delta: {}, finish_reason: "stop" }] },
+ ],
+ );
+ assert.deepEqual(snapshots.at(-1)?.metadata.incomplete, {
+ reason: "quote_cut",
+ });
+});
+
+test("a recovery resumed past the cut keeps the stamp the producer saved", async () => {
+ const { snapshots } = await recoverRun(
+ [],
+ [
+ { choices: [{ delta: { content: "The token is `" } }] },
+ { choices: [], quote_cut: true },
+ { choices: [{ delta: {}, finish_reason: "stop" }] },
+ ],
+ { cursor: 2, metadata: { incomplete: { reason: "quote_cut" } } },
+ );
+ assert.deepEqual(snapshots.at(-1)?.metadata.incomplete, {
+ reason: "quote_cut",
+ });
+});
+
test("the recovery scheduler completes a saved pending card", async () => {
const { content } = await recoverRun(
[