Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: stop sizing a saved connection's Deep Research run to the local model (#10220)
Nilayf24334a3b3b0
diff --git a/studio/backend/core/research_runs.py b/studio/backend/core/research_runs.pyindex 05ba64b36a6..d5ac9e41eb9 100644--- a/studio/backend/core/research_runs.py+++ b/studio/backend/core/research_runs.py@@ -322,13 +322,25 @@ def _peek_inference_backend() -> Any:)-def _loaded_context_length() -> int | None:+def _external_provider_run(inference: dict[str, Any] | None) -> bool:+ return bool((inference or {}).get("providerType"))+++def _run_inference_request(run: dict) -> dict[str, Any]:+ return (run.get("config") or {}).get("inferenceRequest") or {}+++def _loaded_context_length(inference: dict[str, Any] | None = None) -> int | None:"""Best-effort read of the active model's context window in tokens, or None if unknown.Mirrors routes.inference._monitor_context_length (llama.cpp backend, else the inferenceorchestrator) so grounding sizes evidence to the same context the API layer serves. The MLbackends live in a worker subprocess, so the core.inference.inference singleton is unpopulated- here and importing it pulls in the ML stack; read the orchestrator the routes use instead."""+ here and importing it pulls in the ML stack; read the orchestrator the routes use instead.++ A run carrying a providerType runs on that connection, not on either local backend."""+ if _external_provider_run(inference):+ return Nonetry:from routes.inference import get_llama_cpp_backendllama = get_llama_cpp_backend()@@ -382,8 +394,9 @@ def _clamp_max_tokens_for_context(messages: list[dict],*,context_length: int | None = None,+ inference: dict[str, Any] | None = None,) -> int:- ctx = context_length if context_length is not None else _loaded_context_length()+ ctx = context_length if context_length is not None else _loaded_context_length(inference)if not ctx:return requestedavailable = max(1, ctx - _estimate_prompt_tokens(messages))@@ -396,7 +409,7 @@ def _resolve_max_tokens(requested = int(max_tokens or inference.get("maxTokens") or 4096)ceiling = 16384 if max_tokens is not None else 8192capped = min(requested, ceiling)- return _clamp_max_tokens_for_context(capped, messages)+ return _clamp_max_tokens_for_context(capped, messages, inference = inference)def _normalize_completion_usage(raw: Any) -> dict[str, int] | None:@@ -415,22 +428,33 @@ def _completion_hit_context_wall(*,requested_max_tokens: int,context_length: int | None = None,+ inference: dict[str, Any] | None = None,) -> bool:if not usage:return Falseprompt_tokens = int(usage.get("prompt_tokens") or 0)completion_tokens = int(usage.get("completion_tokens") or 0)total_tokens = int(usage.get("total_tokens") or prompt_tokens + completion_tokens)- ctx = context_length if context_length is not None else _loaded_context_length()+ ctx = context_length if context_length is not None else _loaded_context_length(inference)if ctx is not None and total_tokens >= ctx:return Truereturn completion_tokens < requested_max_tokensdef _synthesis_length_limit_error(- usage: dict[str, int] | None, *, requested_max_tokens: int+ usage: dict[str, int] | None,+ *,+ requested_max_tokens: int,+ inference: dict[str, Any] | None = None,) -> str:- if _completion_hit_context_wall(usage, requested_max_tokens = requested_max_tokens):+ # A saved connection never touched the loaded context, so neither half of the local+ # wording holds: the cap it reached is the provider's own output limit, and Context+ # Length in chat settings does not move it.+ if _external_provider_run(inference):+ return "Connected model report reached its output limit before completion"+ if _completion_hit_context_wall(+ usage, requested_max_tokens = requested_max_tokens, inference = inference+ ):return ("Local model report hit the loaded context window before completion. ""Increase Context Length in chat settings or reduce the research evidence size."@@ -671,14 +695,14 @@ async def _wall_clock_timeout(seconds: float | None) -> AsyncIterator[None]:handle.cancel()-def _prompt_char_budget(reserve_tokens: int) -> int | None:+def _prompt_char_budget(reserve_tokens: int, inference: dict[str, Any] | None = None) -> int | None:"""Chars the whole prompt may occupy on the loaded context, or None when it is unknown.The output reserve is capped at half the window: a flat reserve at or above the context(4096 on the 4096-token GGUF floor) would leave a budget of 0 and empty the prompt, and atruncated completion is far better than one that never saw the question."""- ctx = _loaded_context_length()+ ctx = _loaded_context_length(inference)if not ctx:return Nonereserve = min(reserve_tokens, max(1, ctx // 2))@@ -697,10 +721,12 @@ def _trimmable_budget(total: int | None, fixed_chars: int, hard_cap: int) -> intreturn max(0, min(hard_cap, total - fixed_chars))-def _synthesis_evidence_budget(fixed_chars: int = 0) -> int:+def _synthesis_evidence_budget(+ fixed_chars: int = 0, inference: dict[str, Any] | None = None+) -> int:"""Char budget for synthesis evidence (full cap when the context is unknown)."""return _trimmable_budget(- _prompt_char_budget(_SYNTHESIS_CONTEXT_RESERVE_TOKENS),+ _prompt_char_budget(_SYNTHESIS_CONTEXT_RESERVE_TOKENS, inference),fixed_chars,_MAX_SYNTHESIS_EVIDENCE_CHARS,)@@ -735,6 +761,7 @@ def _fit_synthesis_context(notes: list[str],prioritized_payloads: list[dict[str, Any]],fixed_chars: int = 0,+ inference: dict[str, Any] | None = None,) -> tuple[str, list[str]]:"""Share the adaptive synthesis budget between evidence and JSON prompt blocks.@@ -743,7 +770,7 @@ def _fit_synthesis_context(preventing model-derived state or an audit near its output cap from overflowing a small modelcontext."""- total_budget = _synthesis_evidence_budget(fixed_chars)+ total_budget = _synthesis_evidence_budget(fixed_chars, inference)placeholder = "{}"minimum_evidence = min(_MIN_SYNTHESIS_EVIDENCE_CHARS, total_budget)remaining_payload_budget = max(@@ -1854,7 +1881,9 @@ class ResearchSupervisor:)# The question is budgeted before the history but is unbounded on its own (a pasted document arrives# verbatim) and would overflow before planning.- planning_total = _prompt_char_budget(_SYNTHESIS_CONTEXT_RESERVE_TOKENS)+ planning_total = _prompt_char_budget(+ _SYNTHESIS_CONTEXT_RESERVE_TOKENS, _run_inference_request(run)+ )planning_question = question[: max(_MIN_QUESTION_CHARS,@@ -1925,7 +1954,7 @@ class ResearchSupervisor:max_auto_scrape = int(budgets.get("maxAutoScrape", 0))# On a tiny context the prompt overhead alone fills the window, so fall back to snippet-only.if max_auto_scrape > 0:- loaded_ctx = _loaded_context_length()+ loaded_ctx = _loaded_context_length(_run_inference_request(run))if loaded_ctx is not None and loaded_ctx < _AUTO_SCRAPE_MIN_CONTEXT_TOKENS:logger.info("research.auto_scrape_disabled_small_context run_id=%s context=%s",@@ -2050,7 +2079,9 @@ class ResearchSupervisor:)# A fixed 60k evidence tail is many times a small context and this runs every step, so an overflow# here kills the run before it can synthesize.- decision_total = _prompt_char_budget(_SYNTHESIS_CONTEXT_RESERVE_TOKENS)+ decision_total = _prompt_char_budget(+ _SYNTHESIS_CONTEXT_RESERVE_TOKENS, _run_inference_request(run)+ )decision_question, decision_plan_json = _fit_decision_inputs(question,run["plan"],@@ -2386,7 +2417,9 @@ class ResearchSupervisor:)# Model-derived JSON shares the evidence budget, and conversation history receives only what the# fixed scaffold leaves.- total_budget = _prompt_char_budget(_SYNTHESIS_CONTEXT_RESERVE_TOKENS)+ total_budget = _prompt_char_budget(+ _SYNTHESIS_CONTEXT_RESERVE_TOKENS, _run_inference_request(run)+ )plan_json = json.dumps(run["plan"], ensure_ascii = False)audit_system = _system_prompt_with_instructions(_SYNTHESIS_AUDIT_SYSTEM_PROMPT,@@ -2403,6 +2436,7 @@ class ResearchSupervisor:notes,[research_state],audit_scaffold_chars,+ _run_inference_request(run),)audit_conversation_context = conversation_context[: _trimmable_budget(@@ -2480,6 +2514,7 @@ class ResearchSupervisor:notes,[synthesis_audit, research_state],report_scaffold_chars,+ _run_inference_request(run),)synthesis_conversation_context = conversation_context[: _trimmable_budget(@@ -2568,7 +2603,7 @@ class ResearchSupervisor:]recovery_max_tokens = _resolve_max_tokens(16384,- run["config"].get("inferenceRequest") or {},+ _run_inference_request(run),recovery_messages,)(@@ -2606,7 +2641,7 @@ class ResearchSupervisor:else:requested_max_tokens = _resolve_max_tokens(16384,- run["config"].get("inferenceRequest") or {},+ _run_inference_request(run),synthesis_messages,)await self._check_active(run["id"])@@ -2614,6 +2649,7 @@ class ResearchSupervisor:truncation_notice = _synthesis_length_limit_error(synthesis_usage,requested_max_tokens = requested_max_tokens,+ inference = _run_inference_request(run),).rstrip(".")report = _validate_report_sources(report, sources)report = _validate_report_document_sources(report, document_sources)diff --git a/studio/backend/tests/test_research_runs_hardening.py b/studio/backend/tests/test_research_runs_hardening.pyindex d603b794267..625233e8bfd 100644--- a/studio/backend/tests/test_research_runs_hardening.py+++ b/studio/backend/tests/test_research_runs_hardening.py@@ -245,11 +245,11 @@ def test_prompt_budget_counts_the_whole_prompt(monkeypatch):# Budgeting only the evidence cannot prevent an overflow: at a small context the# untrimmable scaffolding (system prompt, plan, source catalogs) is already several times# the window, and the old floor added 1500 chars on top of that.- monkeypatch.setattr(research_runs, "_loaded_context_length", lambda: None)+ monkeypatch.setattr(research_runs, "_loaded_context_length", lambda _inf = None: None)assert research_runs._prompt_char_budget(4096) is Noneassert research_runs._trimmable_budget(None, 99_999, 500) == 500- monkeypatch.setattr(research_runs, "_loaded_context_length", lambda: 16384)+ monkeypatch.setattr(research_runs, "_loaded_context_length", lambda _inf = None: 16384)total = research_runs._prompt_char_budget(4096)assert total == int((16384 - 4096) * research_runs._SYNTHESIS_EVIDENCE_CHARS_PER_TOKEN)# A trimmable section never exceeds what is left, and never goes negative.@@ -259,7 +259,7 @@ def test_prompt_budget_counts_the_whole_prompt(monkeypatch):def test_resolve_max_tokens_clamps_to_loaded_context(monkeypatch):- monkeypatch.setattr(research_runs, "_loaded_context_length", lambda: 12_288)+ monkeypatch.setattr(research_runs, "_loaded_context_length", lambda _inf = None: 12_288)messages = [{"role": "user", "content": "x" * 33_000}]prompt_tokens = _estimate_prompt_tokens(messages)resolved = _resolve_max_tokens(16_384, {}, messages)@@ -267,6 +267,65 @@ def test_resolve_max_tokens_clamps_to_loaded_context(monkeypatch):assert resolved < 16_384+def _pin_local_context(monkeypatch, tokens: int) -> None:+ import routes.inference as inference_routes+ monkeypatch.setattr(+ inference_routes,+ "get_llama_cpp_backend",+ lambda: SimpleNamespace(is_loaded = True, context_length = tokens),+ )+++_EXTERNAL_INFERENCE = {+ "model": "gpt-5.6-sol",+ "providerId": "provider-1",+ "providerType": "openai_codex",+ "externalModel": "gpt-5.6-sol",+}+++def test_a_run_on_a_saved_connection_is_not_sized_to_the_local_model(monkeypatch):+ _pin_local_context(monkeypatch, 4096)+ messages = [{"role": "user", "content": "x" * 4_000}]+ reserve = research_runs._SYNTHESIS_CONTEXT_RESERVE_TOKENS++ assert _resolve_max_tokens(16_384, {}, messages) == 4_096 - _estimate_prompt_tokens(messages)+ assert _resolve_max_tokens(16_384, _EXTERNAL_INFERENCE, messages) == 16_384++ assert research_runs._prompt_char_budget(reserve, {}) == 6_144+ assert research_runs._prompt_char_budget(reserve, _EXTERNAL_INFERENCE) is None++ assert research_runs._synthesis_evidence_budget(0, {}) == 6_144+ assert (+ research_runs._synthesis_evidence_budget(0, _EXTERNAL_INFERENCE)+ == research_runs._MAX_SYNTHESIS_EVIDENCE_CHARS+ )+++def test_a_saved_connection_run_is_not_blamed_on_the_loaded_context(monkeypatch):+ _pin_local_context(monkeypatch, 4096)+ usage = {"prompt_tokens": 3_000, "completion_tokens": 1_096, "total_tokens": 4_096}++ assert _completion_hit_context_wall(usage, requested_max_tokens = 16_384)+ assert not _completion_hit_context_wall(+ usage, requested_max_tokens = 1_096, inference = _EXTERNAL_INFERENCE+ )+ assert "Increase Context Length" not in _synthesis_length_limit_error(+ usage, requested_max_tokens = 1_096, inference = _EXTERNAL_INFERENCE+ )++ # Synthesis asks for 16_384 and a provider stops at its own output cap, so+ # completion_tokens < requested is what an ordinary cloud run reports. That is the+ # shape the notice has to get right, and the case above cannot see it.+ capped = {"prompt_tokens": 40_000, "completion_tokens": 8_192, "total_tokens": 48_192}+ message = _synthesis_length_limit_error(+ capped, requested_max_tokens = 16_384, inference = _EXTERNAL_INFERENCE+ )+ assert "Increase Context Length" not in message+ assert "Local model" not in message+ assert "output limit" in message++def test_completion_hit_context_wall_matches_live_probe():usage = {"prompt_tokens": 11_032,@@ -306,7 +365,10 @@ def test_every_research_prompt_path_is_budgeted():# sections against the loaded context, else the run dies before or after doing the work.src = Path(research_runs.__file__).read_text(encoding = "utf-8")for budget in ("planning_total = ", "decision_total = ", "total_budget = "):- assert f"{budget}_prompt_char_budget(_SYNTHESIS_CONTEXT_RESERVE_TOKENS)" in src+ assert f"{budget}_prompt_char_budget(" in src+ call = src.split(f"{budget}_prompt_char_budget(", 1)[1].split(")", 1)[0]+ assert "_SYNTHESIS_CONTEXT_RESERVE_TOKENS" in call+ assert "_run_inference_request(run" in callassert "evidence[-60000:]" not in src# The question reaches the planner verbatim, so it is budgeted too, but never to nothing.assert "planning_question = question[" in src@@ -324,7 +386,7 @@ def test_prompt_budget_never_empties_the_question_or_evidence(monkeypatch):# A flat 4096-token reserve on the 4096-token GGUF floor made the budget 0, which sliced the# question to "" so the planner never saw the request. Reserve at most half the window.for ctx in (1024, 2048, 4096):- monkeypatch.setattr(research_runs, "_loaded_context_length", lambda c = ctx: c)+ monkeypatch.setattr(research_runs, "_loaded_context_length", lambda _inf = None, c = ctx: c)total = research_runs._prompt_char_budget(research_runs._SYNTHESIS_CONTEXT_RESERVE_TOKENS)assert total is not None and total > 0assert total < int(ctx * research_runs._SYNTHESIS_EVIDENCE_CHARS_PER_TOKEN)diff --git a/studio/backend/tests/test_research_runs_storage.py b/studio/backend/tests/test_research_runs_storage.pyindex 8e90a9558dd..c283c07ff29 100644--- a/studio/backend/tests/test_research_runs_storage.py+++ b/studio/backend/tests/test_research_runs_storage.py@@ -210,31 +210,31 @@ def test_synthesis_evidence_budget_tracks_loaded_context(monkeypatch):from core import research_runs as worker# Unknown context keeps the full cap (backwards compatible).- monkeypatch.setattr(worker, "_loaded_context_length", lambda: None)+ monkeypatch.setattr(worker, "_loaded_context_length", lambda _inf = None: None)assert worker._synthesis_evidence_budget() == worker._MAX_SYNTHESIS_EVIDENCE_CHARS# A small context shrinks the budget so evidence fits, and the rest of the prompt eats into# it, but the output reserve is capped at half the window so the budget never collapses to 0# and empties the prompt (which is worse than a truncated one).- monkeypatch.setattr(worker, "_loaded_context_length", lambda: 2048)+ monkeypatch.setattr(worker, "_loaded_context_length", lambda _inf = None: 2048)small = worker._synthesis_evidence_budget()assert 0 < small < worker._MAX_SYNTHESIS_EVIDENCE_CHARSassert worker._synthesis_evidence_budget(small) == 0# The rest of the prompt counts against the same budget, not just the evidence.- monkeypatch.setattr(worker, "_loaded_context_length", lambda: 16384)+ monkeypatch.setattr(worker, "_loaded_context_length", lambda _inf = None: 16384)roomy = worker._synthesis_evidence_budget()assert 0 < worker._synthesis_evidence_budget(8_000) < roomy# A large context uses (and clamps to) the full cap.- monkeypatch.setattr(worker, "_loaded_context_length", lambda: 32768)+ monkeypatch.setattr(worker, "_loaded_context_length", lambda _inf = None: 32768)assert worker._synthesis_evidence_budget() == worker._MAX_SYNTHESIS_EVIDENCE_CHARSdef test_synthesis_context_budgets_model_derived_json_with_evidence(monkeypatch):from core import research_runs as worker- monkeypatch.setattr(worker, "_loaded_context_length", lambda: 8192)+ monkeypatch.setattr(worker, "_loaded_context_length", lambda _inf = None: 8192)notes = [f"### Step {index}\n" + "evidence " * 2_000 for index in range(6)]audit = {"thesis": "a" * 3_000}research_state = {"summary": "s" * 3_000}@@ -2155,7 +2155,7 @@ def test_auto_scrape_skipped_on_small_context(research_home, monkeypatch):# grounding is skipped (snippet-only) even when maxAutoScrape is set.from core import research_runs as worker- monkeypatch.setattr(worker, "_loaded_context_length", lambda: 2048)+ monkeypatch.setattr(worker, "_loaded_context_length", lambda _inf = None: 2048)_create(budgets = _SCRAPE_BUDGETS)def fake_tool(name, arguments, *args, **kwargs):