Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: keep conversation recall working across repeated compactions (#12174)

Daniel Han6beb057590af
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index 815cf0bb1e9..ce67819f73f 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -9,6 +9,7 @@ OpenAI-compatible /v1/chat/completions endpoint.
import ast
import atexit
+import bisect
import contextlib
import ctypes
import errno
@@ -278,6 +279,8 @@ def _fit_context(messages, **kwargs):
# Whether `sticky_dropped` is the depth of a checkpoint RESET. Defaults to True, which
# is what a caller with no thread state to consult has always assumed.
sticky_is_checkpoint = bool(kwargs.pop("sticky_is_checkpoint", True))
+ # Naming a tool the request lacks makes the model print the call as text.
+ recall_offered = bool(kwargs.pop("recall_offered", True))
requested_policy = kwargs.pop("context_policy", None)
if requested_policy not in ("checkpoint", "rolling"):
requested_policy = None
@@ -340,7 +343,7 @@ def _fit_context(messages, **kwargs):
fitted, truncation = checkpoint.fit_checkpoint_context(
messages,
can_reset = lambda: not _is_degraded(),
- searchable = lambda: not _is_degraded(),
+ searchable = lambda: recall_offered and not _is_degraded(),
**kwargs,
)
# `can_reset = False` has no phase two, so it refuses once the replayed
@@ -1565,6 +1568,92 @@ class _CompactionBranchState(NamedTuple):
recorded: int
+def _reply_text(content) -> str:
+ """A stored reply's visible text, which a Max Tokens continuation re-sends verbatim."""
+ if isinstance(content, str):
+ return content
+ if not isinstance(content, list):
+ return ""
+ return "".join(
+ str(part.get("text") or "")
+ for part in content
+ if isinstance(part, dict) and part.get("type") == "text"
+ )
+
+
+def _row_truncation(message: dict) -> Optional[dict]:
+ metadata = message.get("metadata")
+ metadata = metadata if isinstance(metadata, dict) else {}
+ custom = metadata.get("custom")
+ custom = custom if isinstance(custom, dict) else {}
+ truncation = metadata.get("contextTruncation") or custom.get("contextTruncation")
+ return truncation if isinstance(truncation, dict) else None
+
+
+# The auto-continue limit is 3 rounds; a manual Continue chain longer than this keeps its refusal.
+_RESUME_HOPS = 8
+_RESUME_SIBLINGS = 16
+
+
+def _max_tokens_cut(message: dict) -> bool:
+ metadata = message.get("metadata")
+ metadata = metadata if isinstance(metadata, dict) else {}
+ custom = metadata.get("custom")
+ custom = custom if isinstance(custom, dict) else {}
+ incomplete = metadata.get("incomplete") or custom.get("incomplete")
+ reason = incomplete.get("reason") if isinstance(incomplete, dict) else incomplete
+ return reason == "length"
+
+
+def _row_key(row: dict):
+ return row.get("id") or id(row)
+
+
+def _resumable_cuts(stored: list[dict]) -> dict:
+ """Max Tokens cuts per parent: ([positions], [rows]) in creation order."""
+ cuts: dict = {}
+ for position, row in enumerate(stored):
+ if row.get("role") == "assistant" and _max_tokens_cut(row):
+ at, rows = cuts.setdefault(row.get("parentId", row.get("parent_id")), ([], []))
+ at.append(position)
+ rows.append(row)
+ return cuts
+
+
+def _resumed_reply(message: dict, cuts: dict, positions: dict) -> dict:
+ """The reply an unfitted Max Tokens continuation resumed: its refusal records no epoch,
+ but the epoch in force is still the resumed reply's."""
+ for _ in range(_RESUME_HOPS):
+ truncation = _row_truncation(message)
+ if (
+ truncation is None
+ or truncation.get("fits")
+ or truncation.get("latest_turn_role") != "assistant"
+ ):
+ return message
+ position = positions.get(_row_key(message))
+ if position is None:
+ return message
+ text = _reply_text(message.get("content"))
+ at, rows = cuts.get(message.get("parentId", message.get("parent_id")), ((), ()))
+ end = bisect.bisect_left(at, position)
+ earlier = rows[max(0, end - _RESUME_SIBLINGS) : end]
+ # Newest earlier cut wins; equal text counts, since a refusal adds nothing.
+ source = next(
+ (
+ row
+ for row in reversed(earlier)
+ if (resumed := _reply_text(row.get("content")).rstrip())
+ and text.startswith(resumed)
+ ),
+ None,
+ )
+ if source is None:
+ return message
+ message = source
+ return message
+
+
def _compaction_branch_states(
stored: list[dict], branch_messages: Optional[list[dict]] = None
) -> list[_CompactionBranchState]:
@@ -1600,9 +1689,11 @@ def _compaction_branch_states(
candidates = exact
# A COMPLETED row with no truncation ends the epoch; only active/aborted are placeholders.
+ cuts = _resumable_cuts(stored)
+ positions = {_row_key(row): position for position, row in enumerate(stored)}
states = []
for message in candidates:
- metadata = message.get("metadata")
+ metadata = _resumed_reply(message, cuts, positions).get("metadata")
metadata = metadata if isinstance(metadata, dict) else {}
custom = metadata.get("custom")
custom = custom if isinstance(custom, dict) else {}
@@ -34847,6 +34938,7 @@ class LlamaCppBackend:
sticky_is_checkpoint = _sticky_is_checkpoint,
keeps_boundary = _keeps_compaction_boundary(thread_id),
can_reset = _can_reset,
+ recall_offered = False,
**_compaction_fit_kwargs(context_policy, compaction_headroom_ratio),
)
if truncation:
@@ -35795,6 +35887,7 @@ class LlamaCppBackend:
anchor_ids = _rolling_anchor_ids,
keeps_boundary = _keeps_compaction_boundary(thread_id),
can_reset = _iteration_can_reset,
+ recall_offered = "search_conversation" in (_enabled_tool_names or ()),
reserve_tokens = _conversation_recall_reserve(thread_id),
sticky_dropped = _iteration_sticky,
sticky_is_checkpoint = _iteration_sticky_is_checkpoint,
@@ -35972,6 +36065,7 @@ class LlamaCppBackend:
_backend_supports_tools(self),
tools_withheld = _memory_tool_withheld(thread_id, tools),
),
+ recall_offered = "search_conversation" in (_enabled_tool_names or ()),
**_compaction_fit_kwargs(context_policy, compaction_headroom_ratio),
)
# Recorded here, not left to the forwarding below. That list is
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 365640be0ca..b45faad4ad7 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -5824,10 +5824,14 @@ def _thread_has_checkpoint(thread_id, branch_messages = None) -> bool:
list(studio_db.list_chat_messages(str(thread_id)) or []),
_as_plain_messages(branch_messages),
)
+ # A rescue (fits false, turns dropped) did archive; a refusal drops nothing.
return bool(states) and all(
state.truncation is not None
- and state.truncation.get("fits")
and state.truncation.get("checkpoint")
+ and (
+ state.truncation.get("fits")
+ or int(state.truncation.get("dropped_messages") or 0) > 0
+ )
for state in states
)
except Exception:
diff --git a/studio/backend/tests/test_checkpoint_compaction.py b/studio/backend/tests/test_checkpoint_compaction.py
index 04528b70d28..003c63ffa49 100644
--- a/studio/backend/tests/test_checkpoint_compaction.py
+++ b/studio/backend/tests/test_checkpoint_compaction.py
@@ -2363,9 +2363,12 @@ def test_the_tool_loop_reopens_only_where_an_epoch_actually_happened(monkeypatch
assert inference_routes._thread_has_checkpoint("t1") is True
# A refused fit records diagnostics, not an epoch that can be searched.
- _thread({"fits": False, "dropped_messages": 12, "checkpoint": True})
+ _thread({"fits": False, "dropped_messages": 0, "checkpoint": True})
assert inference_routes._thread_has_checkpoint("t1") is False
+ _thread({"fits": False, "dropped_messages": 12, "checkpoint": True})
+ assert inference_routes._thread_has_checkpoint("t1") is True
+
_thread({"fits": True, "dropped_messages": 12, "checkpoint": True})
# ...but only for the branch the request is on. A Retry that forked BEFORE the
@@ -3900,3 +3903,185 @@ def test_a_nudge_sent_with_an_image_is_not_quoted_as_an_instruction():
def test_an_image_turn_is_judged_on_its_words_not_its_attachment():
assert carried_forward_items([_image_turn("continue")], max_tokens = 1024) == []
assert carried_forward_items([_image_turn(INSTRUCTION)], max_tokens = 1024) == [INSTRUCTION]
+
+
+def _refused_continuation_metadata(boundary):
+ return {
+ "contextTruncation": {
+ "fits": False,
+ "dropped_messages": boundary,
+ "boundary_messages": boundary,
+ "latest_turn_role": "assistant",
+ }
+ }
+
+
+def _cut_metadata(boundary):
+ return {**_checkpoint_metadata(boundary), "incomplete": {"reason": "length"}}
+
+
+ ("case", "resolved"),
+ [
+ ("extends", True),
+ ("adds_nothing", True),
+ ("no_source", False),
+ ("source_not_cut", False),
+ ("source_created_later", False),
+ ],
+)
+def test_a_continuation_that_could_not_fit_keeps_the_epoch_it_resumed(monkeypatch, case, resolved):
+ """Only an earlier Max Tokens cut this row extends (or repeats) was resumed."""
+ from core.inference import checkpoint, llama_cpp
+ from routes import inference as inference_routes
+
+ partial = "Here is the Rust version:\n\nfn main() {"
+ refused_text = partial if case == "adds_nothing" else partial + "\n run();\n}"
+ cut = _turn(
+ id = "cut",
+ parentId = "user-1",
+ content = partial,
+ metadata = _checkpoint_metadata(4) if case == "source_not_cut" else _cut_metadata(4),
+ )
+ refused = _turn(
+ id = "resumed",
+ parentId = "user-1",
+ content = [{"type": "text", "text": refused_text}],
+ metadata = _refused_continuation_metadata(4),
+ )
+ replies = {"no_source": [refused], "source_created_later": [refused, cut]}.get(
+ case, [cut, refused]
+ )
+ rows = [
+ _row(id = "user-1", content = "Write it in Rust."),
+ *replies,
+ _row(id = "user-2", parentId = "resumed", content = "Now fix the bugs."),
+ ]
+ branch = [
+ {"role": "user", "content": "Write it in Rust."},
+ {"role": "assistant", "content": refused_text},
+ {"role": "user", "content": "Now fix the bugs."},
+ ]
+ _stub_studio_db(monkeypatch, rows)
+ monkeypatch.setattr(checkpoint, "CONTEXT_POLICY", "checkpoint")
+
+ assert inference_routes._thread_has_checkpoint("t1", branch) is resolved
+ assert llama_cpp._sticky_compaction_state("t1", branch) == (
+ (4, True) if resolved else (0, False)
+ )
+
+
+def test_resolving_refused_continuations_stays_linear(monkeypatch):
+ import time
+
+ from core.inference import checkpoint, llama_cpp
+
+ rows = [_row(id = "user-1", content = "q")]
+ rows += [
+ _turn(
+ id = f"r{index}",
+ parentId = "user-1",
+ content = "same text",
+ metadata = {**_refused_continuation_metadata(4), "incomplete": {"reason": "length"}},
+ )
+ for index in range(5000)
+ ]
+ _stub_studio_db(monkeypatch, rows)
+ monkeypatch.setattr(checkpoint, "CONTEXT_POLICY", "checkpoint")
+
+ started = time.perf_counter()
+ llama_cpp._compaction_branch_states(
+ rows, [{"role": "user", "content": "q"}, {"role": "assistant", "content": "same text"}]
+ )
+ assert time.perf_counter() - started < 2.0
+
+
+def test_a_retry_sibling_is_not_mistaken_for_the_reply_a_refusal_resumed(monkeypatch):
+ from core.inference import checkpoint
+ from routes import inference as inference_routes
+
+ rows = [
+ _row(id = "user-1", content = "Write it in Rust."),
+ _turn(
+ id = "retry",
+ parentId = "user-1",
+ content = "Something else.",
+ metadata = _checkpoint_metadata(4),
+ ),
+ _turn(
+ id = "refused",
+ parentId = "user-1",
+ content = "A different reply.",
+ metadata = _refused_continuation_metadata(4),
+ ),
+ _row(id = "user-2", parentId = "refused", content = "Now fix the bugs."),
+ ]
+ branch = [
+ {"role": "user", "content": "Write it in Rust."},
+ {"role": "assistant", "content": "A different reply."},
+ {"role": "user", "content": "Now fix the bugs."},
+ ]
+ _stub_studio_db(monkeypatch, rows)
+ monkeypatch.setattr(checkpoint, "CONTEXT_POLICY", "checkpoint")
+
+ assert inference_routes._thread_has_checkpoint("t1", branch) is False
+
+
+def test_a_reset_on_a_request_without_the_recall_tool_does_not_name_it(monkeypatch):
+ """The first reset never carries the tool: the archive is written during it."""
+ from core.inference import llama_cpp
+
+ messages = _thread() + [{"role": "user", "content": "continue"}]
+ monkeypatch.setattr(llama_cpp, "_archive_is_degraded", lambda: False)
+
+ def _header(**kwargs):
+ fitted, truncation = llama_cpp._fit_context(
+ messages,
+ context_length = 1200,
+ max_tokens = 200,
+ count_tokens = count,
+ can_reset = True,
+ sticky_dropped = 0,
+ **kwargs,
+ )
+ assert truncation["checkpoint_started"] is True
+ return fitted[0]["content"]
+
+ withheld = _header(recall_offered = False)
+ assert checkpoint._NOT_SEARCHABLE in withheld
+ assert checkpoint._SEARCHABLE not in withheld
+ assert checkpoint._SEARCHABLE in _header(recall_offered = True)
+ assert checkpoint._SEARCHABLE in _header()
+
+
[email protected](("dropped", "admitted"), [(4, True), (0, False)])
+def test_a_rescued_reset_still_offers_recall_but_is_never_replayed(monkeypatch, dropped, admitted):
+ """A rescue archived what it dropped; a refusal dropped nothing."""
+ from core.inference import checkpoint, llama_cpp
+ from routes import inference as inference_routes
+
+ rows = [
+ _row(id = "user-1", content = "Write it in Rust."),
+ _turn(
+ id = "rust",
+ parentId = "user-1",
+ content = "fn main() {}",
+ metadata = {
+ "contextTruncation": {
+ "fits": False,
+ "checkpoint": True,
+ "checkpoint_started": True,
+ "dropped_messages": dropped,
+ "boundary_messages": dropped,
+ "latest_turn_role": "assistant",
+ }
+ },
+ ),
+ _row(id = "user-2", parentId = "rust", content = "Now fix the bugs."),
+ ]
+ branch = [{"role": row["role"], "content": row["content"]} for row in rows]
+ _stub_studio_db(monkeypatch, rows)
+ monkeypatch.setattr(checkpoint, "CONTEXT_POLICY", "checkpoint")
+
+ assert inference_routes._thread_has_checkpoint("t1", branch) is admitted
+ assert llama_cpp._sticky_compaction_state("t1", branch) == (0, False)