Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: keep a threadless conversation's cut between requests so the prompt cache is reused (#13743)

Daniel Han06859b380d9b
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index 221efcaefdf..67b98a6586f 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -373,7 +373,27 @@ def _fit_context(messages, **kwargs):
return fitted, truncation
except Exception: # noqa: BLE001 -- a policy failure must never break a chat
logger.warning("Checkpoint fit failed; falling back to the rolling window", exc_info = True)
- return fit_rolling_context(messages, **kwargs)
+ return _fit_rolling_remembered(messages, kwargs)
+
+
+def _fit_rolling_remembered(messages, kwargs):
+ """`fit_rolling_context`, remembering the cut in memory when the caller has no thread to save it on.
+
+ A one-off request cuts as before; a returning one replays its cut and takes the usual headroom bite.
+ """
+ if kwargs.get("keeps_boundary") or int(kwargs.get("sticky_dropped") or 0) > 0:
+ return fit_rolling_context(messages, **kwargs)
+ from core.inference import prefix_boundary
+
+ scope = ("rolling", kwargs.get("context_length"))
+ remembered = prefix_boundary.recall(messages, scope)
+ fitted, truncation = fit_rolling_context(
+ messages,
+ **{**kwargs, "keeps_boundary": remembered > 0, "sticky_dropped": remembered},
+ )
+ if truncation and truncation.get("fits") and int(truncation.get("dropped_messages") or 0) > 0:
+ prefix_boundary.remember(messages, fitted, int(truncation["dropped_messages"]), scope)
+ return fitted, truncation
def _fit_with_instruction_pins(
diff --git a/studio/backend/core/inference/prefix_boundary.py b/studio/backend/core/inference/prefix_boundary.py
new file mode 100644
index 00000000000..338401493ad
--- /dev/null
+++ b/studio/backend/core/inference/prefix_boundary.py
@@ -0,0 +1,91 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Where a conversation with no saved thread was last cut, so later requests cut in the same place and reuse
+the prompt cache. Process-local and in memory only: a restart costs one recut.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import threading
+import time
+from collections import OrderedDict
+from typing import Optional
+
+_MAX_ENTRIES = 256
+_IDLE_EXPIRY_S = 6 * 3600
+# Turns that identify a conversation: they never change as it grows.
+_KEY_TURNS = 2
+
+_lock = threading.Lock()
+_entries: "OrderedDict[str, tuple[int, int, str, float]]" = OrderedDict()
+
+
+def _digest(*parts) -> str:
+ payload = json.dumps(parts, sort_keys = True, default = str, ensure_ascii = False)
+ return hashlib.sha256(payload.encode("utf-8", "surrogatepass")).hexdigest()
+
+
+def _split(messages: list[dict]) -> tuple[list, list[dict]]:
+ system = [message.get("content") for message in messages if message.get("role") == "system"]
+ return system, [message for message in messages if message.get("role") != "system"]
+
+
+def _key(messages: list[dict], scope) -> Optional[str]:
+ system, turns = _split(messages)
+ if len(turns) <= _KEY_TURNS:
+ return None
+ return _digest(scope, system, turns[:_KEY_TURNS])
+
+
+def recall(messages: list[dict], scope = None) -> int:
+ """The cut this conversation last took, or 0. Only replayed onto the same history."""
+ key = _key(messages, scope)
+ if key is None:
+ return 0
+ now = time.monotonic()
+ with _lock:
+ entry = _entries.get(key)
+ if entry is None:
+ return 0
+ dropped, span, anchor, seen = entry
+ if now - seen > _IDLE_EXPIRY_S:
+ del _entries[key]
+ return 0
+ _entries.move_to_end(key)
+ _entries[key] = (dropped, span, anchor, now)
+ _, turns = _split(messages)
+ # Up to the last removed turn: a conversation sharing only the opening (another user's, or edited) differs here.
+ if span >= len(turns) or _digest(turns[:span]) != anchor:
+ return 0
+ return dropped
+
+
+def remember(
+ messages: list[dict],
+ kept: list[dict],
+ dropped: int,
+ scope = None,
+) -> None:
+ """Record that ``messages`` (the uncut request) became ``kept`` by losing ``dropped`` oldest turns."""
+ key = _key(messages, scope)
+ _, turns = _split(messages)
+ if key is None or not 0 < dropped < len(turns):
+ return
+ kept_ids = {id(message) for message in kept}
+ # By identity, as the evictor works: kept protected turns can sit between removed ones.
+ removed = [index for index, message in enumerate(turns) if id(message) not in kept_ids]
+ span = removed[dropped - 1] + 1 if len(removed) >= dropped else dropped
+ entry = (int(dropped), span, _digest(turns[:span]), time.monotonic())
+ with _lock:
+ _entries[key] = entry
+ _entries.move_to_end(key)
+ while len(_entries) > _MAX_ENTRIES:
+ _entries.popitem(last = False)
+
+
+def clear() -> None:
+ with _lock:
+ _entries.clear()
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index a53ca33cddd..f576d0d1138 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -1252,6 +1252,53 @@ def _apply_measured_overflow_truncation(body: dict, err_text: str, policy: str)
return max(0, before - len(body.get("messages") or []))
+def _passthrough_cut_scope(llama_backend, body: dict) -> tuple:
+ # A cut measured under another model, window or tool catalogue does not carry over.
+ tools = sorted(
+ str(((tool or {}).get("function") or {}).get("name")) for tool in body.get("tools") or []
+ )
+ return (
+ "passthrough",
+ llama_backend.model_identifier,
+ llama_backend.context_length,
+ tuple(tools),
+ )
+
+
+def _replay_passthrough_cut(
+ body: dict, policy: Optional[str], context_length: Optional[int], scope
+) -> int:
+ """Apply this conversation's last cut before sending, while the uncut request would still overflow."""
+ if policy != "truncate_oldest" or not context_length:
+ return 0
+ messages = body.get("messages") or []
+ # Tool schemas render into the prompt too, and an agent's can be most of it. Images are media, not prompt text.
+ estimate = (
+ _estimate_messages_tokens(messages_without_unpriced_media(messages))
+ + len(json.dumps(body.get("tools") or [], default = str)) // 4
+ )
+ if estimate <= _OVERFLOW_PROMPT_TARGET_FRACTION * context_length:
+ return 0
+ from core.inference import prefix_boundary
+
+ remembered = prefix_boundary.recall(messages, scope)
+ if not remembered:
+ return 0
+ cut, dropped = _truncate_oldest_messages(messages, 1.0, min_dropped = remembered)
+ if dropped:
+ body["messages"] = cut
+ return dropped
+
+
+def _remember_passthrough_cut(
+ original: list, body: dict, policy: Optional[str], dropped: int, scope
+) -> None:
+ if policy != "truncate_oldest" or dropped <= 0:
+ return
+ from core.inference import prefix_boundary
+ prefix_boundary.remember(original, body.get("messages") or [], dropped, scope)
+
+
def _anthropic_stream_error_event(exc, *, force: bool = False):
"""Return an Anthropic in-band stream error event when one is useful."""
# Called only from stream except-handlers after the 200 headers, so flag failed up front
@@ -45436,8 +45483,17 @@ async def _openai_passthrough_stream_admitted(
)
_truncate_policy = _overflow_truncation_policy(payload)
_truncate_budget = _OVERFLOW_TRUNCATE_MAX_RETRIES if _truncate_policy else 0
- _truncated_messages = 0
- _context_was_truncated = False
+ _uncut_messages = list(body.get("messages") or [])
+ # Off the event loop: the history can hold megabytes of images.
+ _cut_scope = _passthrough_cut_scope(llama_backend, body)
+ _truncated_messages = await asyncio.to_thread(
+ _replay_passthrough_cut,
+ body,
+ _truncate_policy,
+ llama_backend.context_length,
+ _cut_scope,
+ )
+ _context_was_truncated = _truncated_messages > 0
# One respawn per request: a second unreachable upstream after a successful
# relaunch is a real failure, not a stale port.
_respawn_retried = False
@@ -45449,6 +45505,9 @@ async def _openai_passthrough_stream_admitted(
return False
_context_was_truncated = True
_truncated_messages += dropped
+ _remember_passthrough_cut(
+ _uncut_messages, body, _truncate_policy, _truncated_messages, _cut_scope
+ )
return True
while True:
@@ -46354,8 +46413,13 @@ async def _openai_passthrough_non_streaming_upstream(
_truncate_policy = _overflow_truncation_policy(payload)
_truncate_budget = _OVERFLOW_TRUNCATE_MAX_RETRIES if _truncate_policy else 0
- _truncated_messages = 0
- _context_was_truncated = False
+ _uncut_messages = list(body.get("messages") or [])
+ # Off the event loop: the history can hold megabytes of images.
+ _cut_scope = _passthrough_cut_scope(llama_backend, body)
+ _truncated_messages = await asyncio.to_thread(
+ _replay_passthrough_cut, body, _truncate_policy, llama_backend.context_length, _cut_scope
+ )
+ _context_was_truncated = _truncated_messages > 0
# One respawn per request, as on the streaming twin.
_respawn_retried = False
@@ -46366,6 +46430,9 @@ async def _openai_passthrough_non_streaming_upstream(
return False
_context_was_truncated = True
_truncated_messages += dropped
+ _remember_passthrough_cut(
+ _uncut_messages, body, _truncate_policy, _truncated_messages, _cut_scope
+ )
return True
async def _post(body_to_send):
diff --git a/studio/backend/tests/test_prefix_boundary.py b/studio/backend/tests/test_prefix_boundary.py
new file mode 100644
index 00000000000..017efaf0e3f
--- /dev/null
+++ b/studio/backend/tests/test_prefix_boundary.py
@@ -0,0 +1,244 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""A conversation with no saved thread keeps its cut between requests, so the prompt prefix is reused."""
+
+from __future__ import annotations
+
+import pytest
+
+from core.inference import checkpoint, llama_cpp, prefix_boundary
+from routes import inference as routes
+
+
[email protected](autouse = True)
+def _fresh():
+ prefix_boundary.clear()
+ yield
+ prefix_boundary.clear()
+
+
+def count(messages):
+ return sum(max(1, len(str(m.get("content", ""))) // 4) for m in messages)
+
+
+def _transcript(turns):
+ messages = [{"role": "system", "content": "you are a coding agent"}]
+ for index in range(turns):
+ messages += [
+ {"role": "user", "content": f"Step {index}: " + "do the next thing " * 20},
+ {"role": "assistant", "content": f"Done with step {index}. " + "details " * 30},
+ ]
+ return messages + [{"role": "user", "content": "continue"}]
+
+
+def _remember(
+ messages,
+ dropped,
+ scope = None,
+):
+ kept = [messages[0], *messages[1 + dropped :]]
+ prefix_boundary.remember(messages, kept, dropped, scope)
+
+
+SCOPE = ("passthrough", "org/model", 4096)
+
+
+def _cut_body(messages, dropped):
+ return {"messages": [messages[0], *messages[1 + dropped :]]}
+
+
+def _fit(messages):
+ return llama_cpp._fit_context(messages, context_length = 4096, max_tokens = 512, count_tokens = count)
+
+
+def _first_kept(fitted):
+ return next(m["content"] for m in fitted if m["role"] != "system")
+
+
+def test_an_api_conversation_keeps_its_cut_while_it_grows(monkeypatch):
+ monkeypatch.setattr(checkpoint, "CONTEXT_POLICY", "rolling")
+ starts = []
+ for turns in range(30, 46):
+ fitted, truncation = _fit(_transcript(turns))
+ assert truncation and truncation["fits"]
+ starts.append(_first_kept(fitted))
+
+ # Without the memory every request moved the start (16 requests, 16 starts).
+ assert len(set(starts)) <= 4
+ assert [starts.index(s) for s in starts] == sorted(starts.index(s) for s in starts)
+
+
+def test_without_the_memory_the_cut_moves_every_request(monkeypatch):
+ monkeypatch.setattr(checkpoint, "CONTEXT_POLICY", "rolling")
+ monkeypatch.setattr(prefix_boundary, "recall", lambda messages, scope = None: 0)
+ starts = {_first_kept(_fit(_transcript(turns))[0]) for turns in range(30, 46)}
+
+ assert len(starts) > 8
+
+
+def test_a_conversation_sharing_only_the_opening_does_not_borrow_the_cut():
+ messages = _transcript(40)
+ _remember(messages, 20)
+ other = _transcript(40)
+ other[20] = {**other[20], "content": "a different turn"}
+
+ assert prefix_boundary.recall(messages) == 20
+ assert prefix_boundary.recall(other) == 0
+
+
+def test_an_edited_history_forgets_the_cut():
+ messages = _transcript(40)
+ _remember(messages, 20)
+
+ assert prefix_boundary.recall(messages[:15] + messages[16:]) == 0
+
+
+def test_a_thread_with_its_own_boundary_is_not_touched(monkeypatch):
+ monkeypatch.setattr(checkpoint, "CONTEXT_POLICY", "rolling")
+ calls = []
+ monkeypatch.setattr(prefix_boundary, "recall", lambda *a, **k: calls.append(a) or 0)
+
+ llama_cpp._fit_context(
+ _transcript(40),
+ context_length = 4096,
+ max_tokens = 512,
+ count_tokens = count,
+ keeps_boundary = True,
+ )
+
+ assert calls == []
+
+
+def test_the_memory_is_bounded():
+ for index in range(prefix_boundary._MAX_ENTRIES + 20):
+ messages = _transcript(5)
+ messages[1] = {**messages[1], "content": f"conversation {index}"}
+ _remember(messages, 4)
+
+ assert len(prefix_boundary._entries) == prefix_boundary._MAX_ENTRIES
+
+
+def test_the_passthrough_replays_the_cut_before_sending():
+ messages = _transcript(40)
+ routes._remember_passthrough_cut(
+ messages, _cut_body(messages, 30), "truncate_oldest", 30, SCOPE
+ )
+
+ later = {"model": "m", "messages": _transcript(41)}
+ dropped = routes._replay_passthrough_cut(later, "truncate_oldest", 4096, SCOPE)
+
+ assert dropped >= 30
+ assert later["messages"][0]["role"] == "system"
+ assert len(later["messages"]) == len(_transcript(41)) - dropped
+
+
+def test_the_passthrough_leaves_other_policies_alone():
+ messages = _transcript(40)
+ routes._remember_passthrough_cut(
+ messages, _cut_body(messages, 30), "truncate_oldest", 30, SCOPE
+ )
+
+ assert (
+ routes._replay_passthrough_cut(
+ {"model": "m", "messages": _transcript(41)}, "truncate_middle", 4096, SCOPE
+ )
+ == 0
+ )
+ assert (
+ routes._replay_passthrough_cut(
+ {"model": "m", "messages": _transcript(41)}, None, 4096, SCOPE
+ )
+ == 0
+ )
+
+
+def test_the_passthrough_sends_a_conversation_that_now_fits_whole():
+ messages = _transcript(40)
+ routes._remember_passthrough_cut(
+ messages, _cut_body(messages, 30), "truncate_oldest", 30, SCOPE
+ )
+ later = {"model": "m", "messages": _transcript(41)}
+
+ assert routes._replay_passthrough_cut(later, "truncate_oldest", 131072, SCOPE) == 0
+ assert len(later["messages"]) == len(_transcript(41))
+
+
+def test_a_conversation_differing_anywhere_in_the_dropped_part_does_not_borrow_the_cut():
+ messages = _transcript(40)
+ _remember(messages, 20)
+ other = _transcript(40)
+ # Same opening and the same turn right before the cut; one turn in between differs.
+ other[10] = {**other[10], "content": "another user's turn"}
+
+ assert prefix_boundary.recall(other) == 0
+
+
+def test_a_kept_protected_turn_inside_the_cut_is_covered_by_the_check():
+ messages = _transcript(40)
+ # Turn 3 was pinned and kept; the 20 removed turns run past it.
+ kept = [messages[0], messages[3], *messages[22:]]
+ prefix_boundary.remember(messages, kept, 20)
+ other = _transcript(40)
+ other[21] = {**other[21], "content": "a turn the cut removed, different here"}
+
+ assert prefix_boundary.recall(messages) == 20
+ assert prefix_boundary.recall(other) == 0
+
+
+def test_tool_schemas_count_toward_the_overflow_check():
+ messages = _transcript(12)
+ routes._remember_passthrough_cut(messages, _cut_body(messages, 6), "truncate_oldest", 6, SCOPE)
+ body = {"messages": _transcript(13)}
+ assert routes._replay_passthrough_cut(dict(body), "truncate_oldest", 4096, SCOPE) == 0
+
+ tools = [
+ {"type": "function", "function": {"name": f"t{i}", "description": "d" * 400}}
+ for i in range(20)
+ ]
+ assert (
+ routes._replay_passthrough_cut({**body, "tools": tools}, "truncate_oldest", 4096, SCOPE)
+ >= 6
+ )
+
+
+def test_a_cut_measured_on_another_model_is_not_replayed():
+ messages = _transcript(40)
+ routes._remember_passthrough_cut(
+ messages, _cut_body(messages, 30), "truncate_oldest", 30, SCOPE
+ )
+ later = {"messages": _transcript(41)}
+
+ assert (
+ routes._replay_passthrough_cut(
+ later, "truncate_oldest", 4096, ("passthrough", "other/model", 4096)
+ )
+ == 0
+ )
+
+
+def test_a_changed_tool_catalogue_gets_its_own_cut():
+ class Backend:
+ model_identifier = "org/model"
+ context_length = 4096
+
+ tools = [{"type": "function", "function": {"name": "read_file"}}]
+ with_tools = routes._passthrough_cut_scope(Backend, {"tools": tools})
+
+ assert with_tools != routes._passthrough_cut_scope(Backend, {"tools": []})
+ assert with_tools == routes._passthrough_cut_scope(Backend, {"tools": list(reversed(tools))})
+
+
+def test_an_image_does_not_count_as_prompt_text_in_the_overflow_check():
+ messages = _transcript(12)
+ routes._remember_passthrough_cut(messages, _cut_body(messages, 6), "truncate_oldest", 6, SCOPE)
+ later = _transcript(13)
+ later[-1] = {
+ "role": "user",
+ "content": [
+ {"type": "text", "text": "what is in this picture?"},
+ {"type": "image_url", "image_url": {"url": "data:image/png;base64," + "A" * 400_000}},
+ ],
+ }
+
+ assert routes._replay_passthrough_cut({"messages": later}, "truncate_oldest", 4096, SCOPE) == 0