Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: keep the first in-loop cut for later tool rounds in chats without a saved thread (#16936)

Nilay1eed2f213a2a
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index 865e8783ad2..89f0501c963 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -37622,6 +37622,8 @@ class LlamaCppBackend:
_conversation_recall_done = False
# Apply the sticky boundary once; refits run on already-fitted history.
_sticky_boundary_applied = False
+ # Later fits run on the cut history, so the cut stays put even without a saved thread.
+ _loop_cut = False
# Estimator misses the tool catalogue and template framing; the offset corrects it.
_prompt_token_offset: Optional[int] = None
# The branch this request is on, kept aside before anything is evicted from or
@@ -38179,7 +38181,7 @@ class LlamaCppBackend:
),
estimate_message = self._estimate_countable_message,
anchor_ids = _rolling_anchor_ids,
- keeps_boundary = _keeps_compaction_boundary(thread_id),
+ keeps_boundary = _loop_cut or _keeps_compaction_boundary(thread_id),
can_reset = _iteration_can_reset,
recall_offered = "search_conversation" in (_enabled_tool_names or ()),
reserve_tokens = _conversation_recall_reserve(thread_id),
@@ -38191,6 +38193,7 @@ class LlamaCppBackend:
)
# Accounted for in this request now, whatever the fit decided.
_sticky_boundary_applied = True
+ _loop_cut = _loop_cut or bool(truncation and truncation.get("dropped_messages"))
if truncation and truncation.get("fits"):
# Before the recall injection, so both sides describe the same
# messages, priced by the fit with the catalogue and template.
@@ -38333,7 +38336,7 @@ class LlamaCppBackend:
_respawn_truncations: list[dict] = []
def _refit_iteration_after_respawn() -> None:
- nonlocal conversation
+ nonlocal conversation, _loop_cut
_before_respawn_fit = conversation
if (
_preflight_context_length is None
@@ -38370,7 +38373,7 @@ class LlamaCppBackend:
),
estimate_message = self._estimate_countable_message,
anchor_ids = _rolling_anchor_ids,
- keeps_boundary = _keeps_compaction_boundary(thread_id),
+ keeps_boundary = _loop_cut or _keeps_compaction_boundary(thread_id),
can_reset = _can_reset_epoch(
thread_id,
_backend_supports_tools(self),
@@ -38379,6 +38382,7 @@ class LlamaCppBackend:
recall_offered = "search_conversation" in (_enabled_tool_names or ()),
**_compaction_fit_kwargs(context_policy, compaction_headroom_ratio),
)
+ _loop_cut = _loop_cut or bool(truncation and truncation.get("dropped_messages"))
# Record here: the reopened stream raises before forwarding sees the refusal.
from core.inference import context_refusal # noqa: PLC0415
@@ -40799,7 +40803,7 @@ class LlamaCppBackend:
),
estimate_message = self._estimate_countable_message,
anchor_ids = _rolling_anchor_ids,
- keeps_boundary = _keeps_compaction_boundary(thread_id),
+ keeps_boundary = _loop_cut or _keeps_compaction_boundary(thread_id),
can_reset = _final_can_reset,
reserve_tokens = _conversation_recall_reserve(thread_id),
sticky_dropped = _final_sticky,
@@ -40958,7 +40962,7 @@ class LlamaCppBackend:
),
estimate_message = self._estimate_countable_message,
anchor_ids = _rolling_anchor_ids,
- keeps_boundary = _keeps_compaction_boundary(thread_id),
+ keeps_boundary = _loop_cut or _keeps_compaction_boundary(thread_id),
can_reset = _can_reset_epoch(
thread_id,
_backend_supports_tools(self),
diff --git a/studio/backend/core/inference/orchestrator.py b/studio/backend/core/inference/orchestrator.py
index 39c175c4378..03a6d57025d 100644
--- a/studio/backend/core/inference/orchestrator.py
+++ b/studio/backend/core/inference/orchestrator.py
@@ -3123,6 +3123,7 @@ class InferenceOrchestrator:
request_branch: Optional[list] = None,
live_branch: Optional[list] = None,
force_reset: bool = False,
+ keeps_boundary: bool = False,
) -> dict:
"""Fit one MLX or managed-engine prompt into the served window under the policy GGUF uses.
@@ -3251,7 +3252,7 @@ class InferenceOrchestrator:
max_tokens = fit_max_tokens,
count_tokens = _count,
anchor_ids = anchor_ids,
- keeps_boundary = _keeps_compaction_boundary(thread_id),
+ keeps_boundary = keeps_boundary or _keeps_compaction_boundary(thread_id),
can_reset = can_reset,
recall_offered = recall_offered,
reserve_tokens = _conversation_recall_reserve(thread_id),
@@ -3796,6 +3797,8 @@ class InferenceOrchestrator:
_request_ids = {id(message) for message in initial}
_sticky_boundary_applied = False
_conversation_recall_done = False
+ # Later fits run on the cut history, so the cut stays put even without a saved thread.
+ _loop_cut = False
# Loop notices are user turns, so the fit stops protecting the question as the newest user turn.
_rolling_anchor_ids: set[int] = set()
for message in reversed(initial):
@@ -3804,7 +3807,7 @@ class InferenceOrchestrator:
break
def _fit_iteration(conversation: list, active_tools: list, live_branch: list) -> dict:
- nonlocal _sticky_boundary_applied, _conversation_recall_done
+ nonlocal _sticky_boundary_applied, _conversation_recall_done, _loop_cut
if loop_images:
# The count cannot price pictures.
return {}
@@ -3836,6 +3839,10 @@ class InferenceOrchestrator:
recall_done = _conversation_recall_done,
request_branch = _request_branch,
live_branch = live_branch,
+ keeps_boundary = _loop_cut,
+ )
+ _loop_cut = _loop_cut or any(
+ event.get("dropped_messages") for event in result.get("events") or ()
)
# The saved boundary applies once, in the first fit that runs (not resumed or failed).
if result.get("boundary_applied"):
diff --git a/studio/backend/tests/test_llama_cpp_tool_loop.py b/studio/backend/tests/test_llama_cpp_tool_loop.py
index b0f7c636cc3..ce8f38ca4fb 100644
--- a/studio/backend/tests/test_llama_cpp_tool_loop.py
+++ b/studio/backend/tests/test_llama_cpp_tool_loop.py
@@ -3463,6 +3463,50 @@ def test_tool_loop_refits_each_preflight_path_after_context_shrinking_respawn(mo
assert len(payloads[1]["messages"]) == 1
+def fresh_cut_memory():
+ from core.inference import prefix_boundary
+
+ prefix_boundary.clear()
+ yield
+ prefix_boundary.clear()
+
+
+def test_a_threadless_tool_loop_keeps_its_cut_between_rounds(monkeypatch, fresh_cut_memory):
+ from core.inference import checkpoint
+
+ monkeypatch.setattr(checkpoint, "CONTEXT_POLICY", "rolling")
+ rounds = 16
+ streams = [
+ _structured_tool_call("python", {"code": f"print(open('f{i}').read())"}, f"call_{i}")
+ for i in range(rounds)
+ ] + [[_sse({"content": "Done."}), _done()]]
+ backend, payloads = _backend_and_payloads(monkeypatch, streams)
+ backend._effective_context_length = 8192
+ monkeypatch.setattr(
+ backend,
+ "count_chat_tokens",
+ lambda candidate, *_a, **_k: sum(len(str(m.get("content") or "")) for m in candidate),
+ )
+ monkeypatch.setattr(
+ "core.inference.tools.execute_tool", lambda name, arguments, **_k: "x" * 1200
+ )
+
+ _run_tool_loop(
+ backend,
+ [{"role": "system", "content": "agent"}, {"role": "user", "content": "read every file"}],
+ [{"type": "function", "function": {"name": "python"}}],
+ max_tool_iterations = rounds,
+ max_tokens = 512,
+ permission_mode = "off",
+ context_overflow = "truncate_oldest",
+ )
+
+ heads = [json.dumps(payload["messages"][:3]) for payload in payloads]
+ moves = sum(before != after for before, after in zip(heads, heads[1:]))
+ assert moves < len(payloads) // 2, moves
+
+
def test_tool_loop_compacts_text_history_around_latest_audio(monkeypatch):
"""Unpriced media must not disable compaction that the text alone requires."""
backend, payloads = _backend_and_payloads(monkeypatch, [[_sse({"content": "OK"}), _done()]])
diff --git a/studio/backend/tests/test_mlx_context_compaction.py b/studio/backend/tests/test_mlx_context_compaction.py
index f387492908c..2240ad5f6d2 100644
--- a/studio/backend/tests/test_mlx_context_compaction.py
+++ b/studio/backend/tests/test_mlx_context_compaction.py
@@ -193,9 +193,8 @@ def _run_loop(monkeypatch, messages, replies, result_chars, **kwargs):
tools = [WEB],
max_tokens = 200,
nudge_tool_calls = True,
- thread_id = "thread-1",
context_overflow = "truncate_oldest",
- **kwargs,
+ **{"thread_id": "thread-1", **kwargs},
)
truncated = [event for event in events if event.get("type") == "context_truncated"]
return truncated, prompts, branches
@@ -289,3 +288,31 @@ def test_the_only_tool_result_is_not_evicted_to_make_room_for_what_the_loop_says
# Dropping the result would fit but leave nothing to answer from.
assert truncation["fits"] is fits and truncation["dropped_messages"] == dropped
assert [m["role"] for m in prompts[-1]].count("tool") == 1
+
+
+def fresh_cut_memory():
+ from core.inference import prefix_boundary
+
+ prefix_boundary.clear()
+ yield
+ prefix_boundary.clear()
+
+
+def test_a_threadless_tool_loop_keeps_its_cut_between_rounds(
+ monkeypatch, archive_calls, fresh_cut_memory
+):
+ rounds = 16
+ _, prompts, _ = _run_loop(
+ monkeypatch,
+ [QUESTION],
+ [CALL % i for i in range(rounds)] + ["done"],
+ 400,
+ max_tool_iterations = rounds,
+ context_policy = "rolling",
+ thread_id = None,
+ )
+
+ heads = [str(prompt[:3]) for prompt in prompts]
+ moves = sum(before != after for before, after in zip(heads, heads[1:]))
+ assert moves < len(prompts) // 2, moves