Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: keep a threadless conversation's cut between requests so the prompt cache is reused (#13743)
Daniel Han06859b380d9b
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.pyindex 221efcaefdf..67b98a6586f 100644--- a/studio/backend/core/inference/llama_cpp.py+++ b/studio/backend/core/inference/llama_cpp.py@@ -373,7 +373,27 @@ def _fit_context(messages, **kwargs):return fitted, truncationexcept Exception: # noqa: BLE001 -- a policy failure must never break a chatlogger.warning("Checkpoint fit failed; falling back to the rolling window", exc_info = True)- return fit_rolling_context(messages, **kwargs)+ return _fit_rolling_remembered(messages, kwargs)+++def _fit_rolling_remembered(messages, kwargs):+ """`fit_rolling_context`, remembering the cut in memory when the caller has no thread to save it on.++ A one-off request cuts as before; a returning one replays its cut and takes the usual headroom bite.+ """+ if kwargs.get("keeps_boundary") or int(kwargs.get("sticky_dropped") or 0) > 0:+ return fit_rolling_context(messages, **kwargs)+ from core.inference import prefix_boundary++ scope = ("rolling", kwargs.get("context_length"))+ remembered = prefix_boundary.recall(messages, scope)+ fitted, truncation = fit_rolling_context(+ messages,+ **{**kwargs, "keeps_boundary": remembered > 0, "sticky_dropped": remembered},+ )+ if truncation and truncation.get("fits") and int(truncation.get("dropped_messages") or 0) > 0:+ prefix_boundary.remember(messages, fitted, int(truncation["dropped_messages"]), scope)+ return fitted, truncationdef _fit_with_instruction_pins(diff --git a/studio/backend/core/inference/prefix_boundary.py b/studio/backend/core/inference/prefix_boundary.pynew file mode 100644index 00000000000..338401493ad--- /dev/null+++ b/studio/backend/core/inference/prefix_boundary.py@@ -0,0 +1,91 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Where a conversation with no saved thread was last cut, so later requests cut in the same place and reuse+the prompt cache. Process-local and in memory only: a restart costs one recut.+"""++from __future__ import annotations++import hashlib+import json+import threading+import time+from collections import OrderedDict+from typing import Optional++_MAX_ENTRIES = 256+_IDLE_EXPIRY_S = 6 * 3600+# Turns that identify a conversation: they never change as it grows.+_KEY_TURNS = 2++_lock = threading.Lock()+_entries: "OrderedDict[str, tuple[int, int, str, float]]" = OrderedDict()+++def _digest(*parts) -> str:+ payload = json.dumps(parts, sort_keys = True, default = str, ensure_ascii = False)+ return hashlib.sha256(payload.encode("utf-8", "surrogatepass")).hexdigest()+++def _split(messages: list[dict]) -> tuple[list, list[dict]]:+ system = [message.get("content") for message in messages if message.get("role") == "system"]+ return system, [message for message in messages if message.get("role") != "system"]+++def _key(messages: list[dict], scope) -> Optional[str]:+ system, turns = _split(messages)+ if len(turns) <= _KEY_TURNS:+ return None+ return _digest(scope, system, turns[:_KEY_TURNS])+++def recall(messages: list[dict], scope = None) -> int:+ """The cut this conversation last took, or 0. Only replayed onto the same history."""+ key = _key(messages, scope)+ if key is None:+ return 0+ now = time.monotonic()+ with _lock:+ entry = _entries.get(key)+ if entry is None:+ return 0+ dropped, span, anchor, seen = entry+ if now - seen > _IDLE_EXPIRY_S:+ del _entries[key]+ return 0+ _entries.move_to_end(key)+ _entries[key] = (dropped, span, anchor, now)+ _, turns = _split(messages)+ # Up to the last removed turn: a conversation sharing only the opening (another user's, or edited) differs here.+ if span >= len(turns) or _digest(turns[:span]) != anchor:+ return 0+ return dropped+++def remember(+ messages: list[dict],+ kept: list[dict],+ dropped: int,+ scope = None,+) -> None:+ """Record that ``messages`` (the uncut request) became ``kept`` by losing ``dropped`` oldest turns."""+ key = _key(messages, scope)+ _, turns = _split(messages)+ if key is None or not 0 < dropped < len(turns):+ return+ kept_ids = {id(message) for message in kept}+ # By identity, as the evictor works: kept protected turns can sit between removed ones.+ removed = [index for index, message in enumerate(turns) if id(message) not in kept_ids]+ span = removed[dropped - 1] + 1 if len(removed) >= dropped else dropped+ entry = (int(dropped), span, _digest(turns[:span]), time.monotonic())+ with _lock:+ _entries[key] = entry+ _entries.move_to_end(key)+ while len(_entries) > _MAX_ENTRIES:+ _entries.popitem(last = False)+++def clear() -> None:+ with _lock:+ _entries.clear()diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex a53ca33cddd..f576d0d1138 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -1252,6 +1252,53 @@ def _apply_measured_overflow_truncation(body: dict, err_text: str, policy: str)return max(0, before - len(body.get("messages") or []))+def _passthrough_cut_scope(llama_backend, body: dict) -> tuple:+ # A cut measured under another model, window or tool catalogue does not carry over.+ tools = sorted(+ str(((tool or {}).get("function") or {}).get("name")) for tool in body.get("tools") or []+ )+ return (+ "passthrough",+ llama_backend.model_identifier,+ llama_backend.context_length,+ tuple(tools),+ )+++def _replay_passthrough_cut(+ body: dict, policy: Optional[str], context_length: Optional[int], scope+) -> int:+ """Apply this conversation's last cut before sending, while the uncut request would still overflow."""+ if policy != "truncate_oldest" or not context_length:+ return 0+ messages = body.get("messages") or []+ # Tool schemas render into the prompt too, and an agent's can be most of it. Images are media, not prompt text.+ estimate = (+ _estimate_messages_tokens(messages_without_unpriced_media(messages))+ + len(json.dumps(body.get("tools") or [], default = str)) // 4+ )+ if estimate <= _OVERFLOW_PROMPT_TARGET_FRACTION * context_length:+ return 0+ from core.inference import prefix_boundary++ remembered = prefix_boundary.recall(messages, scope)+ if not remembered:+ return 0+ cut, dropped = _truncate_oldest_messages(messages, 1.0, min_dropped = remembered)+ if dropped:+ body["messages"] = cut+ return dropped+++def _remember_passthrough_cut(+ original: list, body: dict, policy: Optional[str], dropped: int, scope+) -> None:+ if policy != "truncate_oldest" or dropped <= 0:+ return+ from core.inference import prefix_boundary+ prefix_boundary.remember(original, body.get("messages") or [], dropped, scope)++def _anthropic_stream_error_event(exc, *, force: bool = False):"""Return an Anthropic in-band stream error event when one is useful."""# Called only from stream except-handlers after the 200 headers, so flag failed up front@@ -45436,8 +45483,17 @@ async def _openai_passthrough_stream_admitted()_truncate_policy = _overflow_truncation_policy(payload)_truncate_budget = _OVERFLOW_TRUNCATE_MAX_RETRIES if _truncate_policy else 0- _truncated_messages = 0- _context_was_truncated = False+ _uncut_messages = list(body.get("messages") or [])+ # Off the event loop: the history can hold megabytes of images.+ _cut_scope = _passthrough_cut_scope(llama_backend, body)+ _truncated_messages = await asyncio.to_thread(+ _replay_passthrough_cut,+ body,+ _truncate_policy,+ llama_backend.context_length,+ _cut_scope,+ )+ _context_was_truncated = _truncated_messages > 0# One respawn per request: a second unreachable upstream after a successful# relaunch is a real failure, not a stale port._respawn_retried = False@@ -45449,6 +45505,9 @@ async def _openai_passthrough_stream_admitted(return False_context_was_truncated = True_truncated_messages += dropped+ _remember_passthrough_cut(+ _uncut_messages, body, _truncate_policy, _truncated_messages, _cut_scope+ )return Truewhile True:@@ -46354,8 +46413,13 @@ async def _openai_passthrough_non_streaming_upstream(_truncate_policy = _overflow_truncation_policy(payload)_truncate_budget = _OVERFLOW_TRUNCATE_MAX_RETRIES if _truncate_policy else 0- _truncated_messages = 0- _context_was_truncated = False+ _uncut_messages = list(body.get("messages") or [])+ # Off the event loop: the history can hold megabytes of images.+ _cut_scope = _passthrough_cut_scope(llama_backend, body)+ _truncated_messages = await asyncio.to_thread(+ _replay_passthrough_cut, body, _truncate_policy, llama_backend.context_length, _cut_scope+ )+ _context_was_truncated = _truncated_messages > 0# One respawn per request, as on the streaming twin._respawn_retried = False@@ -46366,6 +46430,9 @@ async def _openai_passthrough_non_streaming_upstream(return False_context_was_truncated = True_truncated_messages += dropped+ _remember_passthrough_cut(+ _uncut_messages, body, _truncate_policy, _truncated_messages, _cut_scope+ )return Trueasync def _post(body_to_send):diff --git a/studio/backend/tests/test_prefix_boundary.py b/studio/backend/tests/test_prefix_boundary.pynew file mode 100644index 00000000000..017efaf0e3f--- /dev/null+++ b/studio/backend/tests/test_prefix_boundary.py@@ -0,0 +1,244 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""A conversation with no saved thread keeps its cut between requests, so the prompt prefix is reused."""++from __future__ import annotations++import pytest++from core.inference import checkpoint, llama_cpp, prefix_boundary+from routes import inference as routes++[email protected](autouse = True)+def _fresh():+ prefix_boundary.clear()+ yield+ prefix_boundary.clear()+++def count(messages):+ return sum(max(1, len(str(m.get("content", ""))) // 4) for m in messages)+++def _transcript(turns):+ messages = [{"role": "system", "content": "you are a coding agent"}]+ for index in range(turns):+ messages += [+ {"role": "user", "content": f"Step {index}: " + "do the next thing " * 20},+ {"role": "assistant", "content": f"Done with step {index}. " + "details " * 30},+ ]+ return messages + [{"role": "user", "content": "continue"}]+++def _remember(+ messages,+ dropped,+ scope = None,+):+ kept = [messages[0], *messages[1 + dropped :]]+ prefix_boundary.remember(messages, kept, dropped, scope)+++SCOPE = ("passthrough", "org/model", 4096)+++def _cut_body(messages, dropped):+ return {"messages": [messages[0], *messages[1 + dropped :]]}+++def _fit(messages):+ return llama_cpp._fit_context(messages, context_length = 4096, max_tokens = 512, count_tokens = count)+++def _first_kept(fitted):+ return next(m["content"] for m in fitted if m["role"] != "system")+++def test_an_api_conversation_keeps_its_cut_while_it_grows(monkeypatch):+ monkeypatch.setattr(checkpoint, "CONTEXT_POLICY", "rolling")+ starts = []+ for turns in range(30, 46):+ fitted, truncation = _fit(_transcript(turns))+ assert truncation and truncation["fits"]+ starts.append(_first_kept(fitted))++ # Without the memory every request moved the start (16 requests, 16 starts).+ assert len(set(starts)) <= 4+ assert [starts.index(s) for s in starts] == sorted(starts.index(s) for s in starts)+++def test_without_the_memory_the_cut_moves_every_request(monkeypatch):+ monkeypatch.setattr(checkpoint, "CONTEXT_POLICY", "rolling")+ monkeypatch.setattr(prefix_boundary, "recall", lambda messages, scope = None: 0)+ starts = {_first_kept(_fit(_transcript(turns))[0]) for turns in range(30, 46)}++ assert len(starts) > 8+++def test_a_conversation_sharing_only_the_opening_does_not_borrow_the_cut():+ messages = _transcript(40)+ _remember(messages, 20)+ other = _transcript(40)+ other[20] = {**other[20], "content": "a different turn"}++ assert prefix_boundary.recall(messages) == 20+ assert prefix_boundary.recall(other) == 0+++def test_an_edited_history_forgets_the_cut():+ messages = _transcript(40)+ _remember(messages, 20)++ assert prefix_boundary.recall(messages[:15] + messages[16:]) == 0+++def test_a_thread_with_its_own_boundary_is_not_touched(monkeypatch):+ monkeypatch.setattr(checkpoint, "CONTEXT_POLICY", "rolling")+ calls = []+ monkeypatch.setattr(prefix_boundary, "recall", lambda *a, **k: calls.append(a) or 0)++ llama_cpp._fit_context(+ _transcript(40),+ context_length = 4096,+ max_tokens = 512,+ count_tokens = count,+ keeps_boundary = True,+ )++ assert calls == []+++def test_the_memory_is_bounded():+ for index in range(prefix_boundary._MAX_ENTRIES + 20):+ messages = _transcript(5)+ messages[1] = {**messages[1], "content": f"conversation {index}"}+ _remember(messages, 4)++ assert len(prefix_boundary._entries) == prefix_boundary._MAX_ENTRIES+++def test_the_passthrough_replays_the_cut_before_sending():+ messages = _transcript(40)+ routes._remember_passthrough_cut(+ messages, _cut_body(messages, 30), "truncate_oldest", 30, SCOPE+ )++ later = {"model": "m", "messages": _transcript(41)}+ dropped = routes._replay_passthrough_cut(later, "truncate_oldest", 4096, SCOPE)++ assert dropped >= 30+ assert later["messages"][0]["role"] == "system"+ assert len(later["messages"]) == len(_transcript(41)) - dropped+++def test_the_passthrough_leaves_other_policies_alone():+ messages = _transcript(40)+ routes._remember_passthrough_cut(+ messages, _cut_body(messages, 30), "truncate_oldest", 30, SCOPE+ )++ assert (+ routes._replay_passthrough_cut(+ {"model": "m", "messages": _transcript(41)}, "truncate_middle", 4096, SCOPE+ )+ == 0+ )+ assert (+ routes._replay_passthrough_cut(+ {"model": "m", "messages": _transcript(41)}, None, 4096, SCOPE+ )+ == 0+ )+++def test_the_passthrough_sends_a_conversation_that_now_fits_whole():+ messages = _transcript(40)+ routes._remember_passthrough_cut(+ messages, _cut_body(messages, 30), "truncate_oldest", 30, SCOPE+ )+ later = {"model": "m", "messages": _transcript(41)}++ assert routes._replay_passthrough_cut(later, "truncate_oldest", 131072, SCOPE) == 0+ assert len(later["messages"]) == len(_transcript(41))+++def test_a_conversation_differing_anywhere_in_the_dropped_part_does_not_borrow_the_cut():+ messages = _transcript(40)+ _remember(messages, 20)+ other = _transcript(40)+ # Same opening and the same turn right before the cut; one turn in between differs.+ other[10] = {**other[10], "content": "another user's turn"}++ assert prefix_boundary.recall(other) == 0+++def test_a_kept_protected_turn_inside_the_cut_is_covered_by_the_check():+ messages = _transcript(40)+ # Turn 3 was pinned and kept; the 20 removed turns run past it.+ kept = [messages[0], messages[3], *messages[22:]]+ prefix_boundary.remember(messages, kept, 20)+ other = _transcript(40)+ other[21] = {**other[21], "content": "a turn the cut removed, different here"}++ assert prefix_boundary.recall(messages) == 20+ assert prefix_boundary.recall(other) == 0+++def test_tool_schemas_count_toward_the_overflow_check():+ messages = _transcript(12)+ routes._remember_passthrough_cut(messages, _cut_body(messages, 6), "truncate_oldest", 6, SCOPE)+ body = {"messages": _transcript(13)}+ assert routes._replay_passthrough_cut(dict(body), "truncate_oldest", 4096, SCOPE) == 0++ tools = [+ {"type": "function", "function": {"name": f"t{i}", "description": "d" * 400}}+ for i in range(20)+ ]+ assert (+ routes._replay_passthrough_cut({**body, "tools": tools}, "truncate_oldest", 4096, SCOPE)+ >= 6+ )+++def test_a_cut_measured_on_another_model_is_not_replayed():+ messages = _transcript(40)+ routes._remember_passthrough_cut(+ messages, _cut_body(messages, 30), "truncate_oldest", 30, SCOPE+ )+ later = {"messages": _transcript(41)}++ assert (+ routes._replay_passthrough_cut(+ later, "truncate_oldest", 4096, ("passthrough", "other/model", 4096)+ )+ == 0+ )+++def test_a_changed_tool_catalogue_gets_its_own_cut():+ class Backend:+ model_identifier = "org/model"+ context_length = 4096++ tools = [{"type": "function", "function": {"name": "read_file"}}]+ with_tools = routes._passthrough_cut_scope(Backend, {"tools": tools})++ assert with_tools != routes._passthrough_cut_scope(Backend, {"tools": []})+ assert with_tools == routes._passthrough_cut_scope(Backend, {"tools": list(reversed(tools))})+++def test_an_image_does_not_count_as_prompt_text_in_the_overflow_check():+ messages = _transcript(12)+ routes._remember_passthrough_cut(messages, _cut_body(messages, 6), "truncate_oldest", 6, SCOPE)+ later = _transcript(13)+ later[-1] = {+ "role": "user",+ "content": [+ {"type": "text", "text": "what is in this picture?"},+ {"type": "image_url", "image_url": {"url": "data:image/png;base64," + "A" * 400_000}},+ ],+ }++ assert routes._replay_passthrough_cut({"messages": later}, "truncate_oldest", 4096, SCOPE) == 0