Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: keep prompt cache on seeded tool-loop continuations (#10707)
Indrajit swain243af4aa65af
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.pyindex 8904e2e8954..ba110cf7d9f 100644--- a/studio/backend/core/inference/llama_cpp.py+++ b/studio/backend/core/inference/llama_cpp.py@@ -777,11 +777,21 @@ _SPEC_KIND_CAPABILITY: dict[str, str] = {_LLAMA_RANDOM_SEED = 0xFFFFFFFF-def _apply_seeded_llama_request(payload: dict, seed: Optional[int]) -> None:- """Disable prompt caching for fixed seeds so repeated requests stay reproducible."""+def _apply_seeded_llama_request(+ payload: dict,+ seed: Optional[int],+ *,+ reuse_prompt_cache: bool = False,+) -> None:+ """Disable prompt caching for fixed seeds so repeated requests stay reproducible.++ ``reuse_prompt_cache``: a tool-loop continuation of this run; cold, it re-prefills (#10698).+ """if seed is None:returnpayload["seed"] = seed+ if reuse_prompt_cache:+ return# Compared as uint32: the schemas also accept 4294967295, the same "pick at random".if (seed & 0xFFFFFFFF) != _LLAMA_RANDOM_SEED:payload["cache_prompt"] = False@@ -38336,6 +38346,8 @@ class LlamaCppBackend:_continuation_max_tokens: Optional[int] = None_continuation_credits = 0_MAX_CONTINUATION_CREDITS = _MAX_LENGTH_CONTINUATIONS * max(1, max_tool_iterations)+ # A fixed seed keeps the prompt cache once this run has sent a request (#10698).+ _prompt_sent = Falseiteration = -1# Outside the loop: the cap is across the whole run, and a per-iteration list# would only ever see the current batch, leaving every earlier iteration's@@ -38608,7 +38620,8 @@ class LlamaCppBackend:_continuation_max_tokens = Noneif stop:payload["stop"] = stop- _apply_seeded_llama_request(payload, seed)+ _apply_seeded_llama_request(payload, seed, reuse_prompt_cache = _prompt_sent)+ _prompt_sent = True_respawn_truncations: list[dict] = []@@ -41350,7 +41363,7 @@ class LlamaCppBackend:stream_payload["max_tokens"] = _final_max_tokensif stop:stream_payload["stop"] = stop- _apply_seeded_llama_request(stream_payload, seed)+ _apply_seeded_llama_request(stream_payload, seed, reuse_prompt_cache = _prompt_sent)stream_payload["stream_options"] = {"include_usage": True}# Progress events feed the first-token deadline; timings stay opt-in.diff --git a/studio/backend/tests/test_llama_cpp_tool_loop.py b/studio/backend/tests/test_llama_cpp_tool_loop.pyindex 507409266c3..56b196776a1 100644--- a/studio/backend/tests/test_llama_cpp_tool_loop.py+++ b/studio/backend/tests/test_llama_cpp_tool_loop.py@@ -342,6 +342,80 @@ def test_plain_random_seed_sentinel_keeps_slot_prompt_cache_reuse(monkeypatch):assert "cache_prompt" not in payloads[0]+def test_reuse_prompt_cache_keeps_fixed_seed_without_disabling_slot_reuse():+ """Tool-loop continuations opt out of #9979's cold-cache pin (#10698)."""+ from core.inference.llama_cpp import _apply_seeded_llama_request++ cold: dict = {}+ _apply_seeded_llama_request(cold, 3407)+ assert cold == {"seed": 3407, "cache_prompt": False}++ warm: dict = {}+ _apply_seeded_llama_request(warm, 3407, reuse_prompt_cache = True)+ assert warm == {"seed": 3407}+ assert "cache_prompt" not in warm+++def test_seeded_tool_loop_keeps_prompt_cache_after_the_first_round(monkeypatch):+ """#10698: round 0 stays cold (#9979); later rounds keep the seed and the cache."""+ streams = [+ _structured_tool_call("web_search", {"query": "kernel"}, "call_search"),+ [_sse({"content": "Linux 6.10."}), _done()],+ ]+ backend, payloads = _backend_and_payloads(monkeypatch, streams)+ monkeypatch.setattr(+ "core.inference.tools.execute_tool",+ lambda name, arguments, **_kwargs: "Linux kernel 6.10",+ )++ list(+ backend.generate_chat_completion_with_tools(+ messages = [{"role": "user", "content": "search then answer"}],+ tools = [+ {+ "type": "function",+ "function": {+ "name": "web_search",+ "parameters": {"type": "object", "properties": {}},+ },+ }+ ],+ seed = 3407,+ max_tool_iterations = 5,+ permission_mode = "off",+ )+ )++ assert len(payloads) >= 2+ assert payloads[0]["seed"] == 3407+ assert payloads[0]["cache_prompt"] is False+ for later in payloads[1:]:+ assert later["seed"] == 3407+ assert "cache_prompt" not in later, later+++def test_seeded_zero_tool_iterations_keeps_cold_cache_on_the_final_pass(monkeypatch):+ """max_tool_iterations=0 never sends an in-loop request; the final pass is round 0."""+ backend, payloads = _backend_and_payloads(+ monkeypatch,+ [[_sse({"content": "no tools this turn"}), _done()]],+ )++ list(+ backend.generate_chat_completion_with_tools(+ messages = [{"role": "user", "content": "just answer"}],+ tools = [],+ seed = 3407,+ max_tool_iterations = 0,+ permission_mode = "off",+ )+ )++ assert len(payloads) == 1+ assert payloads[0]["seed"] == 3407+ assert payloads[0]["cache_prompt"] is False++def test_tool_stream_reports_progress_without_leaking_a_content_event(monkeypatch):stream = [_progress(processed = 512, cached = 0, time_ms = 64),diff --git a/studio/backend/tests/test_openai_tool_passthrough.py b/studio/backend/tests/test_openai_tool_passthrough.pyindex 4f88bcb723b..31d2f8f1fe7 100644--- a/studio/backend/tests/test_openai_tool_passthrough.py+++ b/studio/backend/tests/test_openai_tool_passthrough.py@@ -11040,6 +11040,9 @@ def test_the_two_seed_helpers_agree_on_which_seeds_are_random():payload = {}_apply_seeded_llama_request(payload, value)assert payload["cache_prompt"] is False, (seed, value)+ warm = {}+ _apply_seeded_llama_request(warm, value, reuse_prompt_cache = True)+ assert warm == {"seed": value}, (seed, value, warm)def test_a_lenient_schema_reaches_llama_server_where_it_reads_one():