Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: keep the first in-loop cut for later tool rounds in chats without a saved thread (#16936)
Nilay1eed2f213a2a
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.pyindex 865e8783ad2..89f0501c963 100644--- a/studio/backend/core/inference/llama_cpp.py+++ b/studio/backend/core/inference/llama_cpp.py@@ -37622,6 +37622,8 @@ class LlamaCppBackend:_conversation_recall_done = False# Apply the sticky boundary once; refits run on already-fitted history._sticky_boundary_applied = False+ # Later fits run on the cut history, so the cut stays put even without a saved thread.+ _loop_cut = False# Estimator misses the tool catalogue and template framing; the offset corrects it._prompt_token_offset: Optional[int] = None# The branch this request is on, kept aside before anything is evicted from or@@ -38179,7 +38181,7 @@ class LlamaCppBackend:),estimate_message = self._estimate_countable_message,anchor_ids = _rolling_anchor_ids,- keeps_boundary = _keeps_compaction_boundary(thread_id),+ keeps_boundary = _loop_cut or _keeps_compaction_boundary(thread_id),can_reset = _iteration_can_reset,recall_offered = "search_conversation" in (_enabled_tool_names or ()),reserve_tokens = _conversation_recall_reserve(thread_id),@@ -38191,6 +38193,7 @@ class LlamaCppBackend:)# Accounted for in this request now, whatever the fit decided._sticky_boundary_applied = True+ _loop_cut = _loop_cut or bool(truncation and truncation.get("dropped_messages"))if truncation and truncation.get("fits"):# Before the recall injection, so both sides describe the same# messages, priced by the fit with the catalogue and template.@@ -38333,7 +38336,7 @@ class LlamaCppBackend:_respawn_truncations: list[dict] = []def _refit_iteration_after_respawn() -> None:- nonlocal conversation+ nonlocal conversation, _loop_cut_before_respawn_fit = conversationif (_preflight_context_length is None@@ -38370,7 +38373,7 @@ class LlamaCppBackend:),estimate_message = self._estimate_countable_message,anchor_ids = _rolling_anchor_ids,- keeps_boundary = _keeps_compaction_boundary(thread_id),+ keeps_boundary = _loop_cut or _keeps_compaction_boundary(thread_id),can_reset = _can_reset_epoch(thread_id,_backend_supports_tools(self),@@ -38379,6 +38382,7 @@ class LlamaCppBackend:recall_offered = "search_conversation" in (_enabled_tool_names or ()),**_compaction_fit_kwargs(context_policy, compaction_headroom_ratio),)+ _loop_cut = _loop_cut or bool(truncation and truncation.get("dropped_messages"))# Record here: the reopened stream raises before forwarding sees the refusal.from core.inference import context_refusal # noqa: PLC0415@@ -40799,7 +40803,7 @@ class LlamaCppBackend:),estimate_message = self._estimate_countable_message,anchor_ids = _rolling_anchor_ids,- keeps_boundary = _keeps_compaction_boundary(thread_id),+ keeps_boundary = _loop_cut or _keeps_compaction_boundary(thread_id),can_reset = _final_can_reset,reserve_tokens = _conversation_recall_reserve(thread_id),sticky_dropped = _final_sticky,@@ -40958,7 +40962,7 @@ class LlamaCppBackend:),estimate_message = self._estimate_countable_message,anchor_ids = _rolling_anchor_ids,- keeps_boundary = _keeps_compaction_boundary(thread_id),+ keeps_boundary = _loop_cut or _keeps_compaction_boundary(thread_id),can_reset = _can_reset_epoch(thread_id,_backend_supports_tools(self),diff --git a/studio/backend/core/inference/orchestrator.py b/studio/backend/core/inference/orchestrator.pyindex 39c175c4378..03a6d57025d 100644--- a/studio/backend/core/inference/orchestrator.py+++ b/studio/backend/core/inference/orchestrator.py@@ -3123,6 +3123,7 @@ class InferenceOrchestrator:request_branch: Optional[list] = None,live_branch: Optional[list] = None,force_reset: bool = False,+ keeps_boundary: bool = False,) -> dict:"""Fit one MLX or managed-engine prompt into the served window under the policy GGUF uses.@@ -3251,7 +3252,7 @@ class InferenceOrchestrator:max_tokens = fit_max_tokens,count_tokens = _count,anchor_ids = anchor_ids,- keeps_boundary = _keeps_compaction_boundary(thread_id),+ keeps_boundary = keeps_boundary or _keeps_compaction_boundary(thread_id),can_reset = can_reset,recall_offered = recall_offered,reserve_tokens = _conversation_recall_reserve(thread_id),@@ -3796,6 +3797,8 @@ class InferenceOrchestrator:_request_ids = {id(message) for message in initial}_sticky_boundary_applied = False_conversation_recall_done = False+ # Later fits run on the cut history, so the cut stays put even without a saved thread.+ _loop_cut = False# Loop notices are user turns, so the fit stops protecting the question as the newest user turn._rolling_anchor_ids: set[int] = set()for message in reversed(initial):@@ -3804,7 +3807,7 @@ class InferenceOrchestrator:breakdef _fit_iteration(conversation: list, active_tools: list, live_branch: list) -> dict:- nonlocal _sticky_boundary_applied, _conversation_recall_done+ nonlocal _sticky_boundary_applied, _conversation_recall_done, _loop_cutif loop_images:# The count cannot price pictures.return {}@@ -3836,6 +3839,10 @@ class InferenceOrchestrator:recall_done = _conversation_recall_done,request_branch = _request_branch,live_branch = live_branch,+ keeps_boundary = _loop_cut,+ )+ _loop_cut = _loop_cut or any(+ event.get("dropped_messages") for event in result.get("events") or ())# The saved boundary applies once, in the first fit that runs (not resumed or failed).if result.get("boundary_applied"):diff --git a/studio/backend/tests/test_llama_cpp_tool_loop.py b/studio/backend/tests/test_llama_cpp_tool_loop.pyindex b0f7c636cc3..ce8f38ca4fb 100644--- a/studio/backend/tests/test_llama_cpp_tool_loop.py+++ b/studio/backend/tests/test_llama_cpp_tool_loop.py@@ -3463,6 +3463,50 @@ def test_tool_loop_refits_each_preflight_path_after_context_shrinking_respawn(moassert len(payloads[1]["messages"]) == 1+def fresh_cut_memory():+ from core.inference import prefix_boundary++ prefix_boundary.clear()+ yield+ prefix_boundary.clear()+++def test_a_threadless_tool_loop_keeps_its_cut_between_rounds(monkeypatch, fresh_cut_memory):+ from core.inference import checkpoint++ monkeypatch.setattr(checkpoint, "CONTEXT_POLICY", "rolling")+ rounds = 16+ streams = [+ _structured_tool_call("python", {"code": f"print(open('f{i}').read())"}, f"call_{i}")+ for i in range(rounds)+ ] + [[_sse({"content": "Done."}), _done()]]+ backend, payloads = _backend_and_payloads(monkeypatch, streams)+ backend._effective_context_length = 8192+ monkeypatch.setattr(+ backend,+ "count_chat_tokens",+ lambda candidate, *_a, **_k: sum(len(str(m.get("content") or "")) for m in candidate),+ )+ monkeypatch.setattr(+ "core.inference.tools.execute_tool", lambda name, arguments, **_k: "x" * 1200+ )++ _run_tool_loop(+ backend,+ [{"role": "system", "content": "agent"}, {"role": "user", "content": "read every file"}],+ [{"type": "function", "function": {"name": "python"}}],+ max_tool_iterations = rounds,+ max_tokens = 512,+ permission_mode = "off",+ context_overflow = "truncate_oldest",+ )++ heads = [json.dumps(payload["messages"][:3]) for payload in payloads]+ moves = sum(before != after for before, after in zip(heads, heads[1:]))+ assert moves < len(payloads) // 2, moves++def test_tool_loop_compacts_text_history_around_latest_audio(monkeypatch):"""Unpriced media must not disable compaction that the text alone requires."""backend, payloads = _backend_and_payloads(monkeypatch, [[_sse({"content": "OK"}), _done()]])diff --git a/studio/backend/tests/test_mlx_context_compaction.py b/studio/backend/tests/test_mlx_context_compaction.pyindex f387492908c..2240ad5f6d2 100644--- a/studio/backend/tests/test_mlx_context_compaction.py+++ b/studio/backend/tests/test_mlx_context_compaction.py@@ -193,9 +193,8 @@ def _run_loop(monkeypatch, messages, replies, result_chars, **kwargs):tools = [WEB],max_tokens = 200,nudge_tool_calls = True,- thread_id = "thread-1",context_overflow = "truncate_oldest",- **kwargs,+ **{"thread_id": "thread-1", **kwargs},)truncated = [event for event in events if event.get("type") == "context_truncated"]return truncated, prompts, branches@@ -289,3 +288,31 @@ def test_the_only_tool_result_is_not_evicted_to_make_room_for_what_the_loop_says# Dropping the result would fit but leave nothing to answer from.assert truncation["fits"] is fits and truncation["dropped_messages"] == droppedassert [m["role"] for m in prompts[-1]].count("tool") == 1+++def fresh_cut_memory():+ from core.inference import prefix_boundary++ prefix_boundary.clear()+ yield+ prefix_boundary.clear()+++def test_a_threadless_tool_loop_keeps_its_cut_between_rounds(+ monkeypatch, archive_calls, fresh_cut_memory+):+ rounds = 16+ _, prompts, _ = _run_loop(+ monkeypatch,+ [QUESTION],+ [CALL % i for i in range(rounds)] + ["done"],+ 400,+ max_tool_iterations = rounds,+ context_policy = "rolling",+ thread_id = None,+ )++ heads = [str(prompt[:3]) for prompt in prompts]+ moves = sum(before != after for before, after in zip(heads, heads[1:]))+ assert moves < len(prompts) // 2, moves