Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: turn the generation prompt off on the final answer continuation (#10219)

Nilaye5bf6e65dcb9
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index ecb4bc92e36..eb15d8d172f 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -28811,6 +28811,7 @@ class LlamaCppBackend:
tools,
reasoning_kw,
continue_flag = False,
+ protect = None,
):
"""Drop older turns from a continuation candidate that is one eviction short.
@@ -28824,6 +28825,8 @@ class LlamaCppBackend:
preflight: that one archives, recalls and moves this thread's sticky boundary,
and none of that should happen a second time in the middle of one turn.
Returns the evicted candidate, or None when there was nothing to evict.
+
+ ``protect`` keeps the current turn intact while older history is evicted.
"""
if context_overflow != "truncate_oldest" or not self._effective_context_length:
return None
@@ -28851,7 +28854,7 @@ class LlamaCppBackend:
continue_final_message = continue_flag,
),
estimate_message = estimate_message_tokens_without_unpriced_media,
- anchor_ids = _rolling_anchor_ids,
+ anchor_ids = (_rolling_anchor_ids | protect) if protect else _rolling_anchor_ids,
)
except Exception:
logger.debug("continuation eviction: fit failed", exc_info = True)
@@ -31666,6 +31669,27 @@ class LlamaCppBackend:
stream_payload["timings_per_token"] = True
_final_respawn_truncations: list[dict] = []
+ # Messages appended to the payload but absent from `conversation`.
+ _refit_tail: list = []
+ _refit_tail_merged = False
+
+ def _record_refit_tail(committed: list, appended: list, merged: bool) -> None:
+ """Track appended messages that a respawn refit must replay."""
+ nonlocal _refit_tail, _refit_tail_merged
+ _live = {id(_message) for _message in committed}
+ _new = [_message for _message in appended if id(_message) in _live]
+ if len(_new) != len(appended):
+ # An eviction split the tail, so fall back to `conversation` alone.
+ _refit_tail = []
+ _refit_tail_merged = False
+ return
+ if not merged:
+ _refit_tail = _refit_tail + _new
+ elif _refit_tail:
+ _refit_tail = _refit_tail[:-1] + _new
+ else:
+ _refit_tail = _new
+ _refit_tail_merged = True
def _refit_final_after_respawn() -> None:
nonlocal conversation
@@ -31724,9 +31748,41 @@ class LlamaCppBackend:
from core.inference import context_refusal # noqa: PLC0415
context_refusal.record_fit(truncation)
- stream_payload["messages"] = neutralize_control_markup_in_messages(
- conversation, None, self.markup_profile
+ # Copy because neutralization may return `conversation` itself.
+ _refit_messages = list(
+ neutralize_control_markup_in_messages(conversation, None, self.markup_profile)
)
+ if _refit_tail:
+ # A merged tail replaces the trailing assistant prefill.
+ if _refit_tail_merged and trailing_assistant_text(_refit_messages) is not None:
+ _refit_messages[-1:] = _refit_tail
+ else:
+ _refit_messages.extend(_refit_tail)
+ stream_payload["messages"] = _refit_messages
+ if _refit_tail:
+ # Recheck the restored tail against the replacement window.
+ _refit_continue = bool(stream_payload.get("continue_final_message"))
+ if not _continuation_would_be_served(
+ stream_payload["messages"], _refit_continue
+ ):
+ # Protect the current user turn and its recovery tail.
+ _refit_protect = {id(_message) for _message in _refit_tail}
+ for _message in reversed(
+ _refit_messages[: len(_refit_messages) - len(_refit_tail)]
+ ):
+ _refit_protect.add(id(_message))
+ if _message.get("role") == "user":
+ break
+ _refit_evicted = _evict_until_it_fits(
+ stream_payload["messages"],
+ None,
+ stream_payload.get("chat_template_kwargs"),
+ _refit_continue,
+ _refit_protect,
+ )
+ if _refit_evicted is not None:
+ # Nothing else is safe to evict.
+ stream_payload["messages"] = _refit_evicted
if truncation:
if _records_boundary(truncation):
truncation.update(
@@ -32012,6 +32068,7 @@ class LlamaCppBackend:
# trailing assistant text, so handing it the cumulative value a
# second time yields "fragment1 + fragment1 + continuation1".
_candidate_messages = list(stream_payload["messages"])
+ _merged_f = trailing_assistant_text(_candidate_messages) is not None
_append_assistant_turn(
_candidate_messages,
{
@@ -32028,6 +32085,7 @@ class LlamaCppBackend:
_candidate_messages = neutralize_control_markup_in_messages(
_candidate_messages, None, self.markup_profile
)
+ _continuation_tail = _candidate_messages[-1:]
_next_cap = _remaining_output_budget()
_served = _next_cap != 0 and _continuation_would_be_served(
_candidate_messages, True
@@ -32049,8 +32107,10 @@ class LlamaCppBackend:
_served = True
if _served:
stream_payload["messages"] = _candidate_messages
+ _record_refit_tail(_candidate_messages, _continuation_tail, _merged_f)
_final_replayed_chars = len(_last_emitted)
stream_payload["continue_final_message"] = True
+ stream_payload["add_generation_prompt"] = False
if _next_cap is not None:
stream_payload["max_tokens"] = _next_cap
# Folded in only now, else the reported usage counts the last
@@ -32119,6 +32179,10 @@ class LlamaCppBackend:
# See the answer continuation above: spending the payload and
# the usage before the decision double-counted both.
_candidate_r = list(stream_payload["messages"])
+ _merged_r = (
+ bool(stream_payload.get("continue_final_message"))
+ and trailing_assistant_text(_candidate_r) is not None
+ )
_append_assistant_turn(
_candidate_r,
{
@@ -32144,11 +32208,20 @@ class LlamaCppBackend:
# that would have fit or admits one llama-server then rejects.
_off_kw = self._request_reasoning_kwargs(False, None, preserve_thinking)
_next_cap_r = _remaining_output_budget()
+ # Protect the current user turn and recovery tail.
+ _recovery_tail = _candidate_r[-2:]
+ _recovery_protect = {id(_message) for _message in _recovery_tail}
+ for _message in reversed(_candidate_r[:-2]):
+ _recovery_protect.add(id(_message))
+ if _message.get("role") == "user":
+ break
_served_r = _next_cap_r != 0 and _continuation_would_be_served(
_candidate_r, False, _off_kw
)
if _next_cap_r != 0 and not _served_r:
- _evicted_r = _evict_until_it_fits(_candidate_r, None, _off_kw)
+ _evicted_r = _evict_until_it_fits(
+ _candidate_r, None, _off_kw, False, _recovery_protect
+ )
if _evicted_r is not None and _continuation_would_be_served(
_evicted_r, False, _off_kw
):
@@ -32156,9 +32229,11 @@ class LlamaCppBackend:
_served_r = True
if _served_r:
stream_payload["messages"] = _candidate_r
- # The retry ends on a USER turn, so the flag from any
- # earlier answer continuation no longer describes it.
+ # Preserve both recovery messages across a respawn.
+ _record_refit_tail(_candidate_r, _recovery_tail, _merged_r)
+ # User-ended recovery invalidates continuation flags.
stream_payload.pop("continue_final_message", None)
+ stream_payload.pop("add_generation_prompt", None)
if _next_cap_r is not None:
stream_payload["max_tokens"] = _next_cap_r
if _off_kw is not None:
diff --git a/studio/backend/tests/test_truncated_answer_continuation.py b/studio/backend/tests/test_truncated_answer_continuation.py
index e3d295b0170..a050e318734 100644
--- a/studio/backend/tests/test_truncated_answer_continuation.py
+++ b/studio/backend/tests/test_truncated_answer_continuation.py
@@ -26,10 +26,13 @@ import json
import sys
from pathlib import Path
+import httpx
+
_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)
if _BACKEND_DIR not in sys.path:
sys.path.insert(0, _BACKEND_DIR)
+from core.inference.context_window import _reply_floor
from core.inference.llama_cpp import (
_CONTINUE_TRUNCATED_ANSWER_STATUS,
_MAX_LENGTH_CONTINUATIONS,
@@ -284,6 +287,414 @@ def test_the_final_answer_is_continued_too(monkeypatch):
assert "</html>" in "".join(_texts(events, "content"))
+def test_the_final_continuation_turns_the_generation_prompt_off(monkeypatch):
+ """llama-server rejects a request carrying both flags."""
+
+ payloads: list[dict] = []
+ backend = _make_backend(
+ monkeypatch,
+ _cut_off_then([_sse({"content": ", 0, 6.28);\n</script>\n</html>"}), _done()]),
+ payloads,
+ )
+
+ _run_no_tools(backend)
+
+ assert len(payloads) == 2, "the final answer was left mid-sentence"
+ assert payloads[1]["continue_final_message"] is True
+ assert payloads[1]["add_generation_prompt"] is False
+ # The initial request still needs the default generation prompt.
+ assert "add_generation_prompt" not in payloads[0]
+
+
+def test_a_respawn_refit_during_a_continuation_carries_the_partial(monkeypatch):
+ """The refit must restore the partial absent from `conversation`."""
+
+ payloads: list[dict] = []
+ backend = _make_backend(
+ monkeypatch,
+ _cut_off_then([_sse({"content": ", 0, 6.28);\n</script>\n</html>"}), _done()]),
+ payloads,
+ )
+ monkeypatch.setattr(backend, "count_chat_tokens", lambda *_a, **_k: 64)
+
+ def _respawned() -> bool:
+ # Force a refit against the replacement window.
+ backend._effective_context_length = 2048
+ return True
+
+ monkeypatch.setattr(backend, "_respawn_if_dead", _respawned)
+
+ healthy = backend._stream_with_retry
+ calls = {"n": 0}
+
+ @contextlib.contextmanager
+ def flaky_stream(*args, **kwargs):
+ calls["n"] += 1
+ if calls["n"] == 2:
+ # Fail while opening the continuation.
+ payloads.append(copy.deepcopy(args[2]))
+ raise httpx.RemoteProtocolError("llama-server died before the headers")
+ with healthy(*args, **kwargs) as response:
+ yield response
+
+ monkeypatch.setattr(backend, "_stream_with_retry", flaky_stream)
+
+ _run_no_tools(backend, context_overflow = "truncate_oldest")
+
+ assert calls["n"] == 3, "the continuation has to be opened, die, and be retried"
+ replayed = payloads[-1]
+ assert replayed["messages"][0]["role"] == "user"
+ assert replayed["messages"][-1]["role"] == "assistant"
+ assert "ctx.arc(6, -5, 5, 0" in replayed["messages"][-1]["content"]
+ assert replayed["continue_final_message"] is True
+ assert replayed["add_generation_prompt"] is False
+
+
+def test_a_respawn_refit_prices_the_carried_partial(monkeypatch):
+ """The replacement window must include the restored partial."""
+
+ payloads: list[dict] = []
+ backend = _make_backend(
+ monkeypatch,
+ _cut_off_then([_sse({"content": ", 0, 6.28);\n</script>\n</html>"}), _done()]),
+ payloads,
+ )
+
+ def _count(messages, *_a, **_k) -> int:
+ return sum(len(str(message.get("content", ""))) for message in messages) // 2
+
+ monkeypatch.setattr(backend, "count_chat_tokens", _count)
+
+ def _respawned() -> bool:
+ # The conversation fits, but the restored partial does not.
+ backend._effective_context_length = 1400
+ return True
+
+ monkeypatch.setattr(backend, "_respawn_if_dead", _respawned)
+
+ healthy = backend._stream_with_retry
+ calls = {"n": 0}
+
+ @contextlib.contextmanager
+ def flaky_stream(*args, **kwargs):
+ calls["n"] += 1
+ if calls["n"] == 2:
+ payloads.append(copy.deepcopy(args[2]))
+ raise httpx.RemoteProtocolError("llama-server died before the headers")
+ with healthy(*args, **kwargs) as response:
+ yield response
+
+ monkeypatch.setattr(backend, "_stream_with_retry", flaky_stream)
+
+ history = [
+ {"role": "user", "content": "Older question " + "x" * 400},
+ {"role": "assistant", "content": "Older answer " + "y" * 400},
+ {"role": "user", "content": "Show me the HTML inline"},
+ ]
+ list(
+ backend.generate_chat_completion_with_tools(
+ messages = history,
+ tools = [],
+ max_tool_iterations = 0,
+ context_overflow = "truncate_oldest",
+ )
+ )
+
+ replayed = payloads[-1]
+ assert replayed["messages"][-1]["role"] == "assistant", "the partial still rides across"
+ # The restored partial forces eviction of the older exchange.
+ assert len(replayed["messages"]) == 2, "the older exchange was not evicted for the partial"
+ assert _count(replayed["messages"]) + _reply_floor(1400) <= 1400
+ assert replayed["continue_final_message"] is True
+ assert replayed["add_generation_prompt"] is False
+
+
+def test_a_respawn_refit_does_not_replay_a_caller_prefill_twice(monkeypatch):
+ """A respawn refit must not duplicate a caller prefill."""
+
+ prefill = "Here is the beginning of my answer: "
+ payloads: list[dict] = []
+ backend = _make_backend(
+ monkeypatch,
+ _cut_off_then([_sse({"content": ", 0, 6.28);\n</script>\n</html>"}), _done()]),
+ payloads,
+ )
+ monkeypatch.setattr(backend, "count_chat_tokens", lambda *_a, **_k: 64)
+
+ def _respawned() -> bool:
+ backend._effective_context_length = 2048
+ return True
+
+ monkeypatch.setattr(backend, "_respawn_if_dead", _respawned)
+
+ healthy = backend._stream_with_retry
+ calls = {"n": 0}
+
+ @contextlib.contextmanager
+ def flaky_stream(*args, **kwargs):
+ calls["n"] += 1
+ if calls["n"] == 2:
+ payloads.append(copy.deepcopy(args[2]))
+ raise httpx.RemoteProtocolError("llama-server died before the headers")
+ with healthy(*args, **kwargs) as response:
+ yield response
+
+ monkeypatch.setattr(backend, "_stream_with_retry", flaky_stream)
+
+ prefilled = [
+ {"role": "user", "content": "Show me the HTML inline"},
+ {"role": "assistant", "content": prefill},
+ ]
+ list(
+ backend.generate_chat_completion_with_tools(
+ messages = prefilled,
+ tools = [],
+ max_tool_iterations = 0,
+ continue_final_message = True,
+ context_overflow = "truncate_oldest",
+ )
+ )
+
+ replayed = payloads[-1]
+ assert [message["role"] for message in replayed["messages"]] == ["user", "assistant"]
+ assert json.dumps(replayed["messages"]).count(prefill) == 1
+ assert replayed["messages"][-1]["content"].startswith(prefill)
+ assert "ctx.arc(6, -5, 5, 0" in replayed["messages"][-1]["content"]
+
+
+def test_a_respawn_refit_during_the_reasoning_recovery_keeps_its_request(monkeypatch):
+ """A respawn refit must preserve the reasoning recovery tail."""
+
+ payloads: list[dict] = []
+ backend = _make_backend(
+ monkeypatch,
+ [
+ [_sse({"content": _HALF_AN_ANSWER}), _finish("length"), _done()],
+ [
+ _sse({"reasoning_content": "Let me reconsider the whole approach. " * 60}),
+ _finish("length"),
+ _done(),
+ ],
+ [_sse({"content": "and here is the rest."}), _done()],
+ ],
+ payloads,
+ )
+ monkeypatch.setattr(backend, "count_chat_tokens", lambda *_a, **_k: 64)
+
+ def _respawned() -> bool:
+ backend._effective_context_length = 2048
+ return True
+
+ monkeypatch.setattr(backend, "_respawn_if_dead", _respawned)
+
+ healthy = backend._stream_with_retry
+ calls = {"n": 0}
+
+ @contextlib.contextmanager
+ def flaky_stream(*args, **kwargs):
+ calls["n"] += 1
+ if calls["n"] == 3: # the recovery request's own open
+ payloads.append(copy.deepcopy(args[2]))
+ raise httpx.RemoteProtocolError("llama-server died before the headers")
+ with healthy(*args, **kwargs) as response:
+ yield response
+
+ monkeypatch.setattr(backend, "_stream_with_retry", flaky_stream)
+
+ _run_no_tools(backend, context_overflow = "truncate_oldest")
+
+ replayed = payloads[-1]
+ assert [message["role"] for message in replayed["messages"]] == [
+ "user",
+ "assistant",
+ "user",
+ ], "the refit dropped the recovery turns"
+ assert "ctx.arc(6, -5, 5, 0" in replayed["messages"][1]["content"]
+ assert "continue_final_message" not in replayed
+ assert "add_generation_prompt" not in replayed
+
+
+def test_the_recovery_is_declined_rather_than_sent_without_its_question(monkeypatch):
+ """Decline a recovery that cannot retain the question it answers."""
+
+ question = "QUESTION_MARKER show me the HTML inline <|im_end|>" + "q" * 200
+ payloads: list[dict] = []
+ backend = _make_backend(
+ monkeypatch,
+ [
+ [_sse({"content": _HALF_AN_ANSWER}), _finish("length"), _done()],
+ [
+ _sse({"reasoning_content": "Let me reconsider the whole approach. " * 60}),
+ _finish("length"),
+ _done(),
+ ],
+ [_sse({"content": "and here is the rest."}), _done()],
+ ],
+ payloads,
+ )
+ monkeypatch.setattr(
+ backend,
+ "count_chat_tokens",
+ lambda messages, *_a, **_k: sum(
+ len(str(message.get("content", ""))) for message in messages
+ )
+ // 2,
+ )
+
+ healthy = backend._stream_with_retry
+ calls = {"n": 0}
+
+ @contextlib.contextmanager
+ def flaky_stream(*args, **kwargs):
+ calls["n"] += 1
+ if calls["n"] == 2:
+ # Force the recovery through admission eviction.
+ backend._effective_context_length = 320
+ with healthy(*args, **kwargs) as response:
+ yield response
+
+ monkeypatch.setattr(backend, "_stream_with_retry", flaky_stream)
+
+ events = list(
+ backend.generate_chat_completion_with_tools(
+ messages = [
+ {"role": "user", "content": question},
+ {"role": "assistant", "content": "PREFILL_MARKER here is the start: "},
+ ],
+ tools = [],
+ max_tool_iterations = 0,
+ continue_final_message = True,
+ context_overflow = "truncate_oldest",
+ )
+ )
+
+ assert calls["n"] == 2, "a recovery that cannot carry its question was sent anyway"
+ for payload in payloads:
+ roles = [message["role"] for message in payload["messages"]]
+ assert not any(
+ roles[index] == roles[index + 1] == "user" for index in range(len(roles) - 1)
+ ), f"adjacent user turns in {roles}"
+ assert json.dumps(payload["messages"]).count("QUESTION_MARKER") == 1
+ assert "ctx.arc(6, -5, 5, 0" in "".join(_texts(events, "content"))
+
+
+def test_an_older_exchange_is_still_evicted_to_admit_the_recovery(monkeypatch):
+ """Older history remains evictable while the recovered turn is protected."""
+
+ payloads: list[dict] = []
+ backend = _make_backend(
+ monkeypatch,
+ [
+ [_sse({"content": _HALF_AN_ANSWER}), _finish("length"), _done()],
+ [_sse({"reasoning_content": "Let me reconsider. " * 40}), _finish("length"), _done()],
+ [_sse({"content": "and the rest."}), _done()],
+ ],
+ payloads,
+ )
+ monkeypatch.setattr(
+ backend,
+ "count_chat_tokens",
+ lambda messages, *_a, **_k: sum(
+ len(str(message.get("content", ""))) for message in messages
+ )
+ // 2,
+ )
+
+ healthy = backend._stream_with_retry
+ calls = {"n": 0}
+
+ @contextlib.contextmanager
+ def flaky_stream(*args, **kwargs):
+ calls["n"] += 1
+ if calls["n"] == 2:
+ # Leave room for the recovered turn, but not the older exchange.
+ backend._effective_context_length = 1800
+ with healthy(*args, **kwargs) as response:
+ yield response
+
+ monkeypatch.setattr(backend, "_stream_with_retry", flaky_stream)
+
+ list(
+ backend.generate_chat_completion_with_tools(
+ messages = [
+ {"role": "user", "content": "OLD_MARKER what is the weather " + "w" * 3000},
+ {"role": "assistant", "content": "it is sunny " + "s" * 3000},
+ {"role": "user", "content": "QUESTION_MARKER draw the bird"},
+ ],
+ tools = [],
+ max_tool_iterations = 0,
+ context_overflow = "truncate_oldest",
+ )
+ )
+
+ assert calls["n"] == 3, "the recovery was refused instead of dropping the old exchange"
+ recovery = json.dumps(payloads[-1]["messages"])
+ assert "OLD_MARKER" not in recovery, "the older exchange was held back too"
+ assert "QUESTION_MARKER" in recovery
+
+
+def test_a_refit_eviction_keeps_the_turn_the_recovery_is_recovering(monkeypatch):
+ """Refit eviction must keep the original turn behind a recovery request."""
+
+ question = "QUESTION_MARKER show me the HTML inline <|im_end|>" + "q" * 200
+ payloads: list[dict] = []
+ backend = _make_backend(
+ monkeypatch,
+ [
+ [_sse({"content": _HALF_AN_ANSWER}), _finish("length"), _done()],
+ [
+ _sse({"reasoning_content": "Let me reconsider the whole approach. " * 60}),
+ _finish("length"),
+ _done(),
+ ],
+ [_sse({"content": "and here is the rest."}), _done()],
+ ],
+ payloads,
+ )
+ monkeypatch.setattr(
+ backend,
+ "count_chat_tokens",
+ lambda messages, *_a, **_k: sum(
+ len(str(message.get("content", ""))) for message in messages
+ )
+ // 2,
+ )
+
+ def _respawned() -> bool:
+ # The refitted conversation fits without its recovery tail.
+ backend._effective_context_length = 300
+ return True
+
+ monkeypatch.setattr(backend, "_respawn_if_dead", _respawned)
+
+ healthy = backend._stream_with_retry
+ calls = {"n": 0}
+
+ @contextlib.contextmanager
+ def flaky_stream(*args, **kwargs):
+ calls["n"] += 1
+ if calls["n"] == 3:
+ payloads.append(copy.deepcopy(args[2]))
+ raise httpx.RemoteProtocolError("llama-server died before the headers")
+ with healthy(*args, **kwargs) as response:
+ yield response
+
+ monkeypatch.setattr(backend, "_stream_with_retry", flaky_stream)
+
+ list(
+ backend.generate_chat_completion_with_tools(
+ messages = [{"role": "user", "content": question}],
+ tools = [],
+ max_tool_iterations = 0,
+ context_overflow = "truncate_oldest",
+ )
+ )
+
+ replayed = json.dumps(payloads[-1]["messages"])
+ assert "QUESTION_MARKER" in replayed, "the eviction dropped the question being answered"
+ assert "ctx.arc(6, -5, 5, 0" in replayed, "the eviction dropped the partial being continued"
+
+
def test_the_final_continuation_is_capped(monkeypatch):
payloads: list[dict] = []
backend = _make_backend(
@@ -697,6 +1108,7 @@ def test_a_continuation_that_stalls_in_reasoning_is_not_read_as_more_answer(monk
# apart from the answer continuation: that one replays the partial and extends it.
assert payloads[2]["messages"][-1]["role"] == "user"
assert "continue_final_message" not in payloads[2]
+ assert "add_generation_prompt" not in payloads[2]
def test_an_answer_already_on_screen_is_not_replaced_by_the_explanation(monkeypatch):