Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: keep prompt cache on seeded tool-loop continuations (#10707)

Indrajit swain243af4aa65af
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index 8904e2e8954..ba110cf7d9f 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -777,11 +777,21 @@ _SPEC_KIND_CAPABILITY: dict[str, str] = {
_LLAMA_RANDOM_SEED = 0xFFFFFFFF
-def _apply_seeded_llama_request(payload: dict, seed: Optional[int]) -> None:
- """Disable prompt caching for fixed seeds so repeated requests stay reproducible."""
+def _apply_seeded_llama_request(
+ payload: dict,
+ seed: Optional[int],
+ *,
+ reuse_prompt_cache: bool = False,
+) -> None:
+ """Disable prompt caching for fixed seeds so repeated requests stay reproducible.
+
+ ``reuse_prompt_cache``: a tool-loop continuation of this run; cold, it re-prefills (#10698).
+ """
if seed is None:
return
payload["seed"] = seed
+ if reuse_prompt_cache:
+ return
# Compared as uint32: the schemas also accept 4294967295, the same "pick at random".
if (seed & 0xFFFFFFFF) != _LLAMA_RANDOM_SEED:
payload["cache_prompt"] = False
@@ -38336,6 +38346,8 @@ class LlamaCppBackend:
_continuation_max_tokens: Optional[int] = None
_continuation_credits = 0
_MAX_CONTINUATION_CREDITS = _MAX_LENGTH_CONTINUATIONS * max(1, max_tool_iterations)
+ # A fixed seed keeps the prompt cache once this run has sent a request (#10698).
+ _prompt_sent = False
iteration = -1
# Outside the loop: the cap is across the whole run, and a per-iteration list
# would only ever see the current batch, leaving every earlier iteration's
@@ -38608,7 +38620,8 @@ class LlamaCppBackend:
_continuation_max_tokens = None
if stop:
payload["stop"] = stop
- _apply_seeded_llama_request(payload, seed)
+ _apply_seeded_llama_request(payload, seed, reuse_prompt_cache = _prompt_sent)
+ _prompt_sent = True
_respawn_truncations: list[dict] = []
@@ -41350,7 +41363,7 @@ class LlamaCppBackend:
stream_payload["max_tokens"] = _final_max_tokens
if stop:
stream_payload["stop"] = stop
- _apply_seeded_llama_request(stream_payload, seed)
+ _apply_seeded_llama_request(stream_payload, seed, reuse_prompt_cache = _prompt_sent)
stream_payload["stream_options"] = {"include_usage": True}
# Progress events feed the first-token deadline; timings stay opt-in.
diff --git a/studio/backend/tests/test_llama_cpp_tool_loop.py b/studio/backend/tests/test_llama_cpp_tool_loop.py
index 507409266c3..56b196776a1 100644
--- a/studio/backend/tests/test_llama_cpp_tool_loop.py
+++ b/studio/backend/tests/test_llama_cpp_tool_loop.py
@@ -342,6 +342,80 @@ def test_plain_random_seed_sentinel_keeps_slot_prompt_cache_reuse(monkeypatch):
assert "cache_prompt" not in payloads[0]
+def test_reuse_prompt_cache_keeps_fixed_seed_without_disabling_slot_reuse():
+ """Tool-loop continuations opt out of #9979's cold-cache pin (#10698)."""
+ from core.inference.llama_cpp import _apply_seeded_llama_request
+
+ cold: dict = {}
+ _apply_seeded_llama_request(cold, 3407)
+ assert cold == {"seed": 3407, "cache_prompt": False}
+
+ warm: dict = {}
+ _apply_seeded_llama_request(warm, 3407, reuse_prompt_cache = True)
+ assert warm == {"seed": 3407}
+ assert "cache_prompt" not in warm
+
+
+def test_seeded_tool_loop_keeps_prompt_cache_after_the_first_round(monkeypatch):
+ """#10698: round 0 stays cold (#9979); later rounds keep the seed and the cache."""
+ streams = [
+ _structured_tool_call("web_search", {"query": "kernel"}, "call_search"),
+ [_sse({"content": "Linux 6.10."}), _done()],
+ ]
+ backend, payloads = _backend_and_payloads(monkeypatch, streams)
+ monkeypatch.setattr(
+ "core.inference.tools.execute_tool",
+ lambda name, arguments, **_kwargs: "Linux kernel 6.10",
+ )
+
+ list(
+ backend.generate_chat_completion_with_tools(
+ messages = [{"role": "user", "content": "search then answer"}],
+ tools = [
+ {
+ "type": "function",
+ "function": {
+ "name": "web_search",
+ "parameters": {"type": "object", "properties": {}},
+ },
+ }
+ ],
+ seed = 3407,
+ max_tool_iterations = 5,
+ permission_mode = "off",
+ )
+ )
+
+ assert len(payloads) >= 2
+ assert payloads[0]["seed"] == 3407
+ assert payloads[0]["cache_prompt"] is False
+ for later in payloads[1:]:
+ assert later["seed"] == 3407
+ assert "cache_prompt" not in later, later
+
+
+def test_seeded_zero_tool_iterations_keeps_cold_cache_on_the_final_pass(monkeypatch):
+ """max_tool_iterations=0 never sends an in-loop request; the final pass is round 0."""
+ backend, payloads = _backend_and_payloads(
+ monkeypatch,
+ [[_sse({"content": "no tools this turn"}), _done()]],
+ )
+
+ list(
+ backend.generate_chat_completion_with_tools(
+ messages = [{"role": "user", "content": "just answer"}],
+ tools = [],
+ seed = 3407,
+ max_tool_iterations = 0,
+ permission_mode = "off",
+ )
+ )
+
+ assert len(payloads) == 1
+ assert payloads[0]["seed"] == 3407
+ assert payloads[0]["cache_prompt"] is False
+
+
def test_tool_stream_reports_progress_without_leaking_a_content_event(monkeypatch):
stream = [
_progress(processed = 512, cached = 0, time_ms = 64),
diff --git a/studio/backend/tests/test_openai_tool_passthrough.py b/studio/backend/tests/test_openai_tool_passthrough.py
index 4f88bcb723b..31d2f8f1fe7 100644
--- a/studio/backend/tests/test_openai_tool_passthrough.py
+++ b/studio/backend/tests/test_openai_tool_passthrough.py
@@ -11040,6 +11040,9 @@ def test_the_two_seed_helpers_agree_on_which_seeds_are_random():
payload = {}
_apply_seeded_llama_request(payload, value)
assert payload["cache_prompt"] is False, (seed, value)
+ warm = {}
+ _apply_seeded_llama_request(warm, value, reuse_prompt_cache = True)
+ assert warm == {"seed": value}, (seed, value, warm)
def test_a_lenient_schema_reaches_llama_server_where_it_reads_one():