Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: keep the tool results of the turn in progress when compaction resets mid-turn (#13951)

oobabooga08fb7bc97491
diff --git a/studio/backend/core/inference/checkpoint.py b/studio/backend/core/inference/checkpoint.py
index 863f727bde4..5df1e3a476b 100644
--- a/studio/backend/core/inference/checkpoint.py
+++ b/studio/backend/core/inference/checkpoint.py
@@ -33,6 +33,8 @@ from collections.abc import Callable
from typing import Any, Optional
from core.inference.context_window import (
+ _COMPACTION_HEADROOM_RATIO,
+ clamp_compaction_headroom_ratio,
estimate_message_tokens,
group_turns,
prompt_budget,
@@ -471,6 +473,21 @@ def _append_to_system(messages: list[dict], block: str) -> list[dict]:
return [{"role": "system", "content": block}, *out]
+def _live_turn_groups(messages: list[dict]) -> list[list[dict]]:
+ """The tool exchanges of the turn in progress, oldest first: the groups after the newest user
+ message, minus the last group, which `truncate_oldest_messages` always keeps."""
+ groups = group_turns(messages)
+ newest = next(
+ (
+ index
+ for index in range(len(groups) - 1, -1, -1)
+ if any(message.get("role") == "user" for message in groups[index])
+ ),
+ None,
+ )
+ return [] if newest is None else groups[newest + 1 : -1]
+
+
def fit_checkpoint_context(
messages: list[dict],
*,
@@ -488,8 +505,7 @@ def fit_checkpoint_context(
can_reset: bool = False,
searchable: bool = True,
estimate_message: Callable[[dict], int] = estimate_message_tokens,
- # Signature compatibility with `fit_rolling_context`. A checkpoint reset already drops to the latest turn plus X; an
- # extra bite of the window would only shrink the standing-instruction block, which is the half worth keeping.
+ # Spent only on the turn in progress a reset holds, so the next tool call does not reset and re-prefill it again.
headroom_ratio: Optional[float] = None,
force_reset: bool = False,
sticky_manual: bool = False,
@@ -584,21 +600,32 @@ def fit_checkpoint_context(
measured = projected
# Phase two: the epoch is full, so start a new one. keep_ratio 0.0 takes every evictable group in one pass; the
- # primitive itself protects system, developer, final and newest user groups.
+ # primitive itself protects system, developer, final and newest user groups. The turn in progress is held too,
+ # released oldest first until the reset leaves the headroom free.
if (current_tokens > prompt_target or force_reset) and _resolved(can_reset):
- candidate, reset_dropped = truncate_oldest_messages(
- messages,
- 0.0,
- protected_message_ids = protected_message_ids,
- estimate_message = estimate_message,
+ live = _live_turn_groups(messages)
+ ratio = clamp_compaction_headroom_ratio(headroom_ratio)
+ hold_target = prompt_target - int(
+ prompt_target * (_COMPACTION_HEADROOM_RATIO if ratio is None else ratio)
)
- if reset_dropped:
+ for released in range(len(live) + 1):
+ held = {id(message) for group in live[released:] for message in group}
+ candidate, reset_dropped = truncate_oldest_messages(
+ messages,
+ 0.0,
+ protected_message_ids = (protected_message_ids or set()) | held,
+ estimate_message = estimate_message,
+ )
+ if not reset_dropped:
+ continue
fitted = candidate
dropped = reset_dropped
is_new_epoch = True
projected, block = _project(fitted)
current_tokens = count_tokens(projected)
measured = projected
+ if current_tokens <= hold_target:
+ break
if dropped == 0 and current_tokens <= prompt_target:
return messages, None
diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py
index 65175bd4f2e..d17b9dca3b0 100644
--- a/studio/backend/models/inference.py
+++ b/studio/backend/models/inference.py
@@ -2929,8 +2929,9 @@ class ChatCompletionRequest(BaseModel):
description = (
"[x-unsloth] Extra share of the prompt budget to drop when a rolling "
"compaction fires, so the boundary can stay put for a stretch of turns. "
- "0.25 is the process default (ROLLING_COMPACTION_HEADROOM_RATIO). Ignored "
- "for checkpoint compaction. Unset keeps the process default."
+ "A checkpoint reset leaves the same share free when it keeps the tool calls "
+ "of the turn in progress. 0.25 is the process default "
+ "(ROLLING_COMPACTION_HEADROOM_RATIO). Unset keeps the process default."
),
)
compact_context: Optional[bool] = Field(
diff --git a/studio/backend/tests/test_checkpoint_compaction.py b/studio/backend/tests/test_checkpoint_compaction.py
index 511cf937ded..2f4910aa24a 100644
--- a/studio/backend/tests/test_checkpoint_compaction.py
+++ b/studio/backend/tests/test_checkpoint_compaction.py
@@ -196,6 +196,81 @@ def test_an_irreducible_request_returns_the_original_messages():
assert truncation["irreducible_tokens"] > truncation["prompt_target"]
+def _exchange(index, result):
+ return [
+ {
+ "role": "assistant",
+ "content": "",
+ "tool_calls": [
+ {
+ "id": f"call-{index}",
+ "type": "function",
+ "function": {"name": "terminal", "arguments": "{}"},
+ }
+ ],
+ },
+ {"role": "tool", "tool_call_id": f"call-{index}", "content": result},
+ ]
+
+
+def _mid_turn(*results):
+ """A thread that overflows inside a tool loop: older history, the newest user message,
+ then one terminal exchange per result."""
+ messages = _thread() + [{"role": "user", "content": "Read main.py and apply your changes."}]
+ for index, result in enumerate(results):
+ messages += _exchange(index, result)
+ return messages
+
+
+ ("results", "kept"),
+ [
+ # Everything the turn has done fits beside the reset, so all of it stays.
+ (["MAIN-PY " * 100, "ls output"], [0, 1]),
+ # Every exchange before the newest is held, not just the one before it.
+ (["MAIN-PY " * 50, "ls output", "git status"], [0, 1, 2]),
+ # Only the oldest exchange has to go for the rest to fit.
+ (["A" * 2400, "B" * 1600, "ls output"], [1, 2]),
+ # Nothing fits but the newest, as on main.
+ (["A" * 2400, "B" * 2400, "C" * 1600], [2]),
+ ],
+)
+def test_a_reset_inside_a_tool_loop_keeps_the_turn_in_progress(results, kept):
+ """A reset mid-turn evicted every tool exchange of the turn in progress except the last,
+ so the file the model had just read with `cat` was gone and it read it again."""
+ messages = _mid_turn(*results)
+
+ fitted, truncation = _fit(messages)
+
+ assert truncation["fits"] is True
+ assert truncation["checkpoint_started"] is True
+ assert count(fitted) <= 1000
+ # Still a reset, not a rolling trim.
+ assert [m["content"] for m in fitted if m["role"] == "user"] == [
+ "Read main.py and apply your changes."
+ ]
+ assert [m["content"] for m in fitted if m["role"] == "tool"] == [results[i] for i in kept]
+ # Each kept result still follows its own call, which strict templates require.
+ calls = [m["tool_calls"][0]["id"] for m in fitted if m.get("tool_calls")]
+ assert calls == [f"call-{i}" for i in kept]
+
+
+def test_a_reset_inside_a_tool_loop_leaves_room_for_the_next_tool_call():
+ """Held to the brim, the turn in progress reset again on every later tool call and was
+ prefilled again each time."""
+ messages = _mid_turn("A" * 1200, "B" * 1200, "C" * 400)
+
+ brim, _ = _fit(messages, headroom_ratio = 0.0)
+ half, _ = _fit(messages, headroom_ratio = 0.5)
+ fitted, truncation = _fit(messages)
+
+ assert [m["content"][0] for m in brim if m["role"] == "tool"] == ["A", "B", "C"]
+ assert [m["content"][0] for m in half if m["role"] == "tool"] == ["C"]
+ assert truncation["checkpoint_started"] is True
+ assert [m["content"][0] for m in fitted if m["role"] == "tool"] == ["B", "C"]
+ assert _fit(fitted + _exchange(3, "D" * 400))[1] is None
+
+
def test_the_carried_forward_block_is_capped_and_excludes_the_giant_instruction():
"""A single enormous instruction is the thing that could starve the window, so it is
excluded whole. Half an instruction is worse than none: it reads as complete."""
@@ -2272,7 +2347,8 @@ def test_changing_the_extra_trim_discards_the_old_boundary(monkeypatch):
del stored[1]["metadata"]["custom"]["contextTruncation"]["boundary_headroom_ratio"]
assert llama_cpp._sticky_compaction_boundary("t1", compaction_headroom_ratio = 0.0) == 12
- # A checkpoint reset ignores headroom, so its depth is never refused over the ratio.
+ # A checkpoint reset spends headroom only on the turn in progress, which its depth never
+ # counts, so that depth is never refused over the ratio.
stored[1]["metadata"]["custom"]["contextTruncation"] = {
"fits": True,
"boundary_messages": 12,
diff --git a/studio/frontend/src/features/chat/types/api.ts b/studio/frontend/src/features/chat/types/api.ts
index 29c7a79359c..4b02f03da9a 100644
--- a/studio/frontend/src/features/chat/types/api.ts
+++ b/studio/frontend/src/features/chat/types/api.ts
@@ -753,7 +753,7 @@ export interface OpenAIChatCompletionsRequest {
context_overflow?: "error" | "truncate_middle" | "truncate_oldest";
/** Override UNSLOTH_CONTEXT_POLICY for this local GGUF request. */
context_policy?: "checkpoint" | "rolling";
- /** Extra share of the prompt budget to drop when a rolling compaction fires. */
+ /** Extra share of the prompt budget a rolling compaction drops, or a checkpoint reset keeping the turn in progress leaves free. */
compaction_headroom_ratio?: number;
max_tool_calls_per_message?: number;
tool_call_timeout?: number;