Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: keep conversation recall working across repeated compactions (#12174)
Daniel Han6beb057590af
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.pyindex 815cf0bb1e9..ce67819f73f 100644--- a/studio/backend/core/inference/llama_cpp.py+++ b/studio/backend/core/inference/llama_cpp.py@@ -9,6 +9,7 @@ OpenAI-compatible /v1/chat/completions endpoint.import astimport atexit+import bisectimport contextlibimport ctypesimport errno@@ -278,6 +279,8 @@ def _fit_context(messages, **kwargs):# Whether `sticky_dropped` is the depth of a checkpoint RESET. Defaults to True, which# is what a caller with no thread state to consult has always assumed.sticky_is_checkpoint = bool(kwargs.pop("sticky_is_checkpoint", True))+ # Naming a tool the request lacks makes the model print the call as text.+ recall_offered = bool(kwargs.pop("recall_offered", True))requested_policy = kwargs.pop("context_policy", None)if requested_policy not in ("checkpoint", "rolling"):requested_policy = None@@ -340,7 +343,7 @@ def _fit_context(messages, **kwargs):fitted, truncation = checkpoint.fit_checkpoint_context(messages,can_reset = lambda: not _is_degraded(),- searchable = lambda: not _is_degraded(),+ searchable = lambda: recall_offered and not _is_degraded(),**kwargs,)# `can_reset = False` has no phase two, so it refuses once the replayed@@ -1565,6 +1568,92 @@ class _CompactionBranchState(NamedTuple):recorded: int+def _reply_text(content) -> str:+ """A stored reply's visible text, which a Max Tokens continuation re-sends verbatim."""+ if isinstance(content, str):+ return content+ if not isinstance(content, list):+ return ""+ return "".join(+ str(part.get("text") or "")+ for part in content+ if isinstance(part, dict) and part.get("type") == "text"+ )+++def _row_truncation(message: dict) -> Optional[dict]:+ metadata = message.get("metadata")+ metadata = metadata if isinstance(metadata, dict) else {}+ custom = metadata.get("custom")+ custom = custom if isinstance(custom, dict) else {}+ truncation = metadata.get("contextTruncation") or custom.get("contextTruncation")+ return truncation if isinstance(truncation, dict) else None+++# The auto-continue limit is 3 rounds; a manual Continue chain longer than this keeps its refusal.+_RESUME_HOPS = 8+_RESUME_SIBLINGS = 16+++def _max_tokens_cut(message: dict) -> bool:+ metadata = message.get("metadata")+ metadata = metadata if isinstance(metadata, dict) else {}+ custom = metadata.get("custom")+ custom = custom if isinstance(custom, dict) else {}+ incomplete = metadata.get("incomplete") or custom.get("incomplete")+ reason = incomplete.get("reason") if isinstance(incomplete, dict) else incomplete+ return reason == "length"+++def _row_key(row: dict):+ return row.get("id") or id(row)+++def _resumable_cuts(stored: list[dict]) -> dict:+ """Max Tokens cuts per parent: ([positions], [rows]) in creation order."""+ cuts: dict = {}+ for position, row in enumerate(stored):+ if row.get("role") == "assistant" and _max_tokens_cut(row):+ at, rows = cuts.setdefault(row.get("parentId", row.get("parent_id")), ([], []))+ at.append(position)+ rows.append(row)+ return cuts+++def _resumed_reply(message: dict, cuts: dict, positions: dict) -> dict:+ """The reply an unfitted Max Tokens continuation resumed: its refusal records no epoch,+ but the epoch in force is still the resumed reply's."""+ for _ in range(_RESUME_HOPS):+ truncation = _row_truncation(message)+ if (+ truncation is None+ or truncation.get("fits")+ or truncation.get("latest_turn_role") != "assistant"+ ):+ return message+ position = positions.get(_row_key(message))+ if position is None:+ return message+ text = _reply_text(message.get("content"))+ at, rows = cuts.get(message.get("parentId", message.get("parent_id")), ((), ()))+ end = bisect.bisect_left(at, position)+ earlier = rows[max(0, end - _RESUME_SIBLINGS) : end]+ # Newest earlier cut wins; equal text counts, since a refusal adds nothing.+ source = next(+ (+ row+ for row in reversed(earlier)+ if (resumed := _reply_text(row.get("content")).rstrip())+ and text.startswith(resumed)+ ),+ None,+ )+ if source is None:+ return message+ message = source+ return message++def _compaction_branch_states(stored: list[dict], branch_messages: Optional[list[dict]] = None) -> list[_CompactionBranchState]:@@ -1600,9 +1689,11 @@ def _compaction_branch_states(candidates = exact# A COMPLETED row with no truncation ends the epoch; only active/aborted are placeholders.+ cuts = _resumable_cuts(stored)+ positions = {_row_key(row): position for position, row in enumerate(stored)}states = []for message in candidates:- metadata = message.get("metadata")+ metadata = _resumed_reply(message, cuts, positions).get("metadata")metadata = metadata if isinstance(metadata, dict) else {}custom = metadata.get("custom")custom = custom if isinstance(custom, dict) else {}@@ -34847,6 +34938,7 @@ class LlamaCppBackend:sticky_is_checkpoint = _sticky_is_checkpoint,keeps_boundary = _keeps_compaction_boundary(thread_id),can_reset = _can_reset,+ recall_offered = False,**_compaction_fit_kwargs(context_policy, compaction_headroom_ratio),)if truncation:@@ -35795,6 +35887,7 @@ class LlamaCppBackend:anchor_ids = _rolling_anchor_ids,keeps_boundary = _keeps_compaction_boundary(thread_id),can_reset = _iteration_can_reset,+ recall_offered = "search_conversation" in (_enabled_tool_names or ()),reserve_tokens = _conversation_recall_reserve(thread_id),sticky_dropped = _iteration_sticky,sticky_is_checkpoint = _iteration_sticky_is_checkpoint,@@ -35972,6 +36065,7 @@ class LlamaCppBackend:_backend_supports_tools(self),tools_withheld = _memory_tool_withheld(thread_id, tools),),+ recall_offered = "search_conversation" in (_enabled_tool_names or ()),**_compaction_fit_kwargs(context_policy, compaction_headroom_ratio),)# Recorded here, not left to the forwarding below. That list isdiff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex 365640be0ca..b45faad4ad7 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -5824,10 +5824,14 @@ def _thread_has_checkpoint(thread_id, branch_messages = None) -> bool:list(studio_db.list_chat_messages(str(thread_id)) or []),_as_plain_messages(branch_messages),)+ # A rescue (fits false, turns dropped) did archive; a refusal drops nothing.return bool(states) and all(state.truncation is not None- and state.truncation.get("fits")and state.truncation.get("checkpoint")+ and (+ state.truncation.get("fits")+ or int(state.truncation.get("dropped_messages") or 0) > 0+ )for state in states)except Exception:diff --git a/studio/backend/tests/test_checkpoint_compaction.py b/studio/backend/tests/test_checkpoint_compaction.pyindex 04528b70d28..003c63ffa49 100644--- a/studio/backend/tests/test_checkpoint_compaction.py+++ b/studio/backend/tests/test_checkpoint_compaction.py@@ -2363,9 +2363,12 @@ def test_the_tool_loop_reopens_only_where_an_epoch_actually_happened(monkeypatchassert inference_routes._thread_has_checkpoint("t1") is True# A refused fit records diagnostics, not an epoch that can be searched.- _thread({"fits": False, "dropped_messages": 12, "checkpoint": True})+ _thread({"fits": False, "dropped_messages": 0, "checkpoint": True})assert inference_routes._thread_has_checkpoint("t1") is False+ _thread({"fits": False, "dropped_messages": 12, "checkpoint": True})+ assert inference_routes._thread_has_checkpoint("t1") is True+_thread({"fits": True, "dropped_messages": 12, "checkpoint": True})# ...but only for the branch the request is on. A Retry that forked BEFORE the@@ -3900,3 +3903,185 @@ def test_a_nudge_sent_with_an_image_is_not_quoted_as_an_instruction():def test_an_image_turn_is_judged_on_its_words_not_its_attachment():assert carried_forward_items([_image_turn("continue")], max_tokens = 1024) == []assert carried_forward_items([_image_turn(INSTRUCTION)], max_tokens = 1024) == [INSTRUCTION]+++def _refused_continuation_metadata(boundary):+ return {+ "contextTruncation": {+ "fits": False,+ "dropped_messages": boundary,+ "boundary_messages": boundary,+ "latest_turn_role": "assistant",+ }+ }+++def _cut_metadata(boundary):+ return {**_checkpoint_metadata(boundary), "incomplete": {"reason": "length"}}+++ ("case", "resolved"),+ [+ ("extends", True),+ ("adds_nothing", True),+ ("no_source", False),+ ("source_not_cut", False),+ ("source_created_later", False),+ ],+)+def test_a_continuation_that_could_not_fit_keeps_the_epoch_it_resumed(monkeypatch, case, resolved):+ """Only an earlier Max Tokens cut this row extends (or repeats) was resumed."""+ from core.inference import checkpoint, llama_cpp+ from routes import inference as inference_routes++ partial = "Here is the Rust version:\n\nfn main() {"+ refused_text = partial if case == "adds_nothing" else partial + "\n run();\n}"+ cut = _turn(+ id = "cut",+ parentId = "user-1",+ content = partial,+ metadata = _checkpoint_metadata(4) if case == "source_not_cut" else _cut_metadata(4),+ )+ refused = _turn(+ id = "resumed",+ parentId = "user-1",+ content = [{"type": "text", "text": refused_text}],+ metadata = _refused_continuation_metadata(4),+ )+ replies = {"no_source": [refused], "source_created_later": [refused, cut]}.get(+ case, [cut, refused]+ )+ rows = [+ _row(id = "user-1", content = "Write it in Rust."),+ *replies,+ _row(id = "user-2", parentId = "resumed", content = "Now fix the bugs."),+ ]+ branch = [+ {"role": "user", "content": "Write it in Rust."},+ {"role": "assistant", "content": refused_text},+ {"role": "user", "content": "Now fix the bugs."},+ ]+ _stub_studio_db(monkeypatch, rows)+ monkeypatch.setattr(checkpoint, "CONTEXT_POLICY", "checkpoint")++ assert inference_routes._thread_has_checkpoint("t1", branch) is resolved+ assert llama_cpp._sticky_compaction_state("t1", branch) == (+ (4, True) if resolved else (0, False)+ )+++def test_resolving_refused_continuations_stays_linear(monkeypatch):+ import time++ from core.inference import checkpoint, llama_cpp++ rows = [_row(id = "user-1", content = "q")]+ rows += [+ _turn(+ id = f"r{index}",+ parentId = "user-1",+ content = "same text",+ metadata = {**_refused_continuation_metadata(4), "incomplete": {"reason": "length"}},+ )+ for index in range(5000)+ ]+ _stub_studio_db(monkeypatch, rows)+ monkeypatch.setattr(checkpoint, "CONTEXT_POLICY", "checkpoint")++ started = time.perf_counter()+ llama_cpp._compaction_branch_states(+ rows, [{"role": "user", "content": "q"}, {"role": "assistant", "content": "same text"}]+ )+ assert time.perf_counter() - started < 2.0+++def test_a_retry_sibling_is_not_mistaken_for_the_reply_a_refusal_resumed(monkeypatch):+ from core.inference import checkpoint+ from routes import inference as inference_routes++ rows = [+ _row(id = "user-1", content = "Write it in Rust."),+ _turn(+ id = "retry",+ parentId = "user-1",+ content = "Something else.",+ metadata = _checkpoint_metadata(4),+ ),+ _turn(+ id = "refused",+ parentId = "user-1",+ content = "A different reply.",+ metadata = _refused_continuation_metadata(4),+ ),+ _row(id = "user-2", parentId = "refused", content = "Now fix the bugs."),+ ]+ branch = [+ {"role": "user", "content": "Write it in Rust."},+ {"role": "assistant", "content": "A different reply."},+ {"role": "user", "content": "Now fix the bugs."},+ ]+ _stub_studio_db(monkeypatch, rows)+ monkeypatch.setattr(checkpoint, "CONTEXT_POLICY", "checkpoint")++ assert inference_routes._thread_has_checkpoint("t1", branch) is False+++def test_a_reset_on_a_request_without_the_recall_tool_does_not_name_it(monkeypatch):+ """The first reset never carries the tool: the archive is written during it."""+ from core.inference import llama_cpp++ messages = _thread() + [{"role": "user", "content": "continue"}]+ monkeypatch.setattr(llama_cpp, "_archive_is_degraded", lambda: False)++ def _header(**kwargs):+ fitted, truncation = llama_cpp._fit_context(+ messages,+ context_length = 1200,+ max_tokens = 200,+ count_tokens = count,+ can_reset = True,+ sticky_dropped = 0,+ **kwargs,+ )+ assert truncation["checkpoint_started"] is True+ return fitted[0]["content"]++ withheld = _header(recall_offered = False)+ assert checkpoint._NOT_SEARCHABLE in withheld+ assert checkpoint._SEARCHABLE not in withheld+ assert checkpoint._SEARCHABLE in _header(recall_offered = True)+ assert checkpoint._SEARCHABLE in _header()++[email protected](("dropped", "admitted"), [(4, True), (0, False)])+def test_a_rescued_reset_still_offers_recall_but_is_never_replayed(monkeypatch, dropped, admitted):+ """A rescue archived what it dropped; a refusal dropped nothing."""+ from core.inference import checkpoint, llama_cpp+ from routes import inference as inference_routes++ rows = [+ _row(id = "user-1", content = "Write it in Rust."),+ _turn(+ id = "rust",+ parentId = "user-1",+ content = "fn main() {}",+ metadata = {+ "contextTruncation": {+ "fits": False,+ "checkpoint": True,+ "checkpoint_started": True,+ "dropped_messages": dropped,+ "boundary_messages": dropped,+ "latest_turn_role": "assistant",+ }+ },+ ),+ _row(id = "user-2", parentId = "rust", content = "Now fix the bugs."),+ ]+ branch = [{"role": row["role"], "content": row["content"]} for row in rows]+ _stub_studio_db(monkeypatch, rows)+ monkeypatch.setattr(checkpoint, "CONTEXT_POLICY", "checkpoint")++ assert inference_routes._thread_has_checkpoint("t1", branch) is admitted+ assert llama_cpp._sticky_compaction_state("t1", branch) == (0, False)