Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: stop sizing a saved connection's Deep Research run to the local model (#10220)

Nilayf24334a3b3b0
diff --git a/studio/backend/core/research_runs.py b/studio/backend/core/research_runs.py
index 05ba64b36a6..d5ac9e41eb9 100644
--- a/studio/backend/core/research_runs.py
+++ b/studio/backend/core/research_runs.py
@@ -322,13 +322,25 @@ def _peek_inference_backend() -> Any:
)
-def _loaded_context_length() -> int | None:
+def _external_provider_run(inference: dict[str, Any] | None) -> bool:
+ return bool((inference or {}).get("providerType"))
+
+
+def _run_inference_request(run: dict) -> dict[str, Any]:
+ return (run.get("config") or {}).get("inferenceRequest") or {}
+
+
+def _loaded_context_length(inference: dict[str, Any] | None = None) -> int | None:
"""Best-effort read of the active model's context window in tokens, or None if unknown.
Mirrors routes.inference._monitor_context_length (llama.cpp backend, else the inference
orchestrator) so grounding sizes evidence to the same context the API layer serves. The ML
backends live in a worker subprocess, so the core.inference.inference singleton is unpopulated
- here and importing it pulls in the ML stack; read the orchestrator the routes use instead."""
+ here and importing it pulls in the ML stack; read the orchestrator the routes use instead.
+
+ A run carrying a providerType runs on that connection, not on either local backend."""
+ if _external_provider_run(inference):
+ return None
try:
from routes.inference import get_llama_cpp_backend
llama = get_llama_cpp_backend()
@@ -382,8 +394,9 @@ def _clamp_max_tokens_for_context(
messages: list[dict],
*,
context_length: int | None = None,
+ inference: dict[str, Any] | None = None,
) -> int:
- ctx = context_length if context_length is not None else _loaded_context_length()
+ ctx = context_length if context_length is not None else _loaded_context_length(inference)
if not ctx:
return requested
available = max(1, ctx - _estimate_prompt_tokens(messages))
@@ -396,7 +409,7 @@ def _resolve_max_tokens(
requested = int(max_tokens or inference.get("maxTokens") or 4096)
ceiling = 16384 if max_tokens is not None else 8192
capped = min(requested, ceiling)
- return _clamp_max_tokens_for_context(capped, messages)
+ return _clamp_max_tokens_for_context(capped, messages, inference = inference)
def _normalize_completion_usage(raw: Any) -> dict[str, int] | None:
@@ -415,22 +428,33 @@ def _completion_hit_context_wall(
*,
requested_max_tokens: int,
context_length: int | None = None,
+ inference: dict[str, Any] | None = None,
) -> bool:
if not usage:
return False
prompt_tokens = int(usage.get("prompt_tokens") or 0)
completion_tokens = int(usage.get("completion_tokens") or 0)
total_tokens = int(usage.get("total_tokens") or prompt_tokens + completion_tokens)
- ctx = context_length if context_length is not None else _loaded_context_length()
+ ctx = context_length if context_length is not None else _loaded_context_length(inference)
if ctx is not None and total_tokens >= ctx:
return True
return completion_tokens < requested_max_tokens
def _synthesis_length_limit_error(
- usage: dict[str, int] | None, *, requested_max_tokens: int
+ usage: dict[str, int] | None,
+ *,
+ requested_max_tokens: int,
+ inference: dict[str, Any] | None = None,
) -> str:
- if _completion_hit_context_wall(usage, requested_max_tokens = requested_max_tokens):
+ # A saved connection never touched the loaded context, so neither half of the local
+ # wording holds: the cap it reached is the provider's own output limit, and Context
+ # Length in chat settings does not move it.
+ if _external_provider_run(inference):
+ return "Connected model report reached its output limit before completion"
+ if _completion_hit_context_wall(
+ usage, requested_max_tokens = requested_max_tokens, inference = inference
+ ):
return (
"Local model report hit the loaded context window before completion. "
"Increase Context Length in chat settings or reduce the research evidence size."
@@ -671,14 +695,14 @@ async def _wall_clock_timeout(seconds: float | None) -> AsyncIterator[None]:
handle.cancel()
-def _prompt_char_budget(reserve_tokens: int) -> int | None:
+def _prompt_char_budget(reserve_tokens: int, inference: dict[str, Any] | None = None) -> int | None:
"""Chars the whole prompt may occupy on the loaded context, or None when it is unknown.
The output reserve is capped at half the window: a flat reserve at or above the context
(4096 on the 4096-token GGUF floor) would leave a budget of 0 and empty the prompt, and a
truncated completion is far better than one that never saw the question.
"""
- ctx = _loaded_context_length()
+ ctx = _loaded_context_length(inference)
if not ctx:
return None
reserve = min(reserve_tokens, max(1, ctx // 2))
@@ -697,10 +721,12 @@ def _trimmable_budget(total: int | None, fixed_chars: int, hard_cap: int) -> int
return max(0, min(hard_cap, total - fixed_chars))
-def _synthesis_evidence_budget(fixed_chars: int = 0) -> int:
+def _synthesis_evidence_budget(
+ fixed_chars: int = 0, inference: dict[str, Any] | None = None
+) -> int:
"""Char budget for synthesis evidence (full cap when the context is unknown)."""
return _trimmable_budget(
- _prompt_char_budget(_SYNTHESIS_CONTEXT_RESERVE_TOKENS),
+ _prompt_char_budget(_SYNTHESIS_CONTEXT_RESERVE_TOKENS, inference),
fixed_chars,
_MAX_SYNTHESIS_EVIDENCE_CHARS,
)
@@ -735,6 +761,7 @@ def _fit_synthesis_context(
notes: list[str],
prioritized_payloads: list[dict[str, Any]],
fixed_chars: int = 0,
+ inference: dict[str, Any] | None = None,
) -> tuple[str, list[str]]:
"""Share the adaptive synthesis budget between evidence and JSON prompt blocks.
@@ -743,7 +770,7 @@ def _fit_synthesis_context(
preventing model-derived state or an audit near its output cap from overflowing a small model
context.
"""
- total_budget = _synthesis_evidence_budget(fixed_chars)
+ total_budget = _synthesis_evidence_budget(fixed_chars, inference)
placeholder = "{}"
minimum_evidence = min(_MIN_SYNTHESIS_EVIDENCE_CHARS, total_budget)
remaining_payload_budget = max(
@@ -1854,7 +1881,9 @@ class ResearchSupervisor:
)
# The question is budgeted before the history but is unbounded on its own (a pasted document arrives
# verbatim) and would overflow before planning.
- planning_total = _prompt_char_budget(_SYNTHESIS_CONTEXT_RESERVE_TOKENS)
+ planning_total = _prompt_char_budget(
+ _SYNTHESIS_CONTEXT_RESERVE_TOKENS, _run_inference_request(run)
+ )
planning_question = question[
: max(
_MIN_QUESTION_CHARS,
@@ -1925,7 +1954,7 @@ class ResearchSupervisor:
max_auto_scrape = int(budgets.get("maxAutoScrape", 0))
# On a tiny context the prompt overhead alone fills the window, so fall back to snippet-only.
if max_auto_scrape > 0:
- loaded_ctx = _loaded_context_length()
+ loaded_ctx = _loaded_context_length(_run_inference_request(run))
if loaded_ctx is not None and loaded_ctx < _AUTO_SCRAPE_MIN_CONTEXT_TOKENS:
logger.info(
"research.auto_scrape_disabled_small_context run_id=%s context=%s",
@@ -2050,7 +2079,9 @@ class ResearchSupervisor:
)
# A fixed 60k evidence tail is many times a small context and this runs every step, so an overflow
# here kills the run before it can synthesize.
- decision_total = _prompt_char_budget(_SYNTHESIS_CONTEXT_RESERVE_TOKENS)
+ decision_total = _prompt_char_budget(
+ _SYNTHESIS_CONTEXT_RESERVE_TOKENS, _run_inference_request(run)
+ )
decision_question, decision_plan_json = _fit_decision_inputs(
question,
run["plan"],
@@ -2386,7 +2417,9 @@ class ResearchSupervisor:
)
# Model-derived JSON shares the evidence budget, and conversation history receives only what the
# fixed scaffold leaves.
- total_budget = _prompt_char_budget(_SYNTHESIS_CONTEXT_RESERVE_TOKENS)
+ total_budget = _prompt_char_budget(
+ _SYNTHESIS_CONTEXT_RESERVE_TOKENS, _run_inference_request(run)
+ )
plan_json = json.dumps(run["plan"], ensure_ascii = False)
audit_system = _system_prompt_with_instructions(
_SYNTHESIS_AUDIT_SYSTEM_PROMPT,
@@ -2403,6 +2436,7 @@ class ResearchSupervisor:
notes,
[research_state],
audit_scaffold_chars,
+ _run_inference_request(run),
)
audit_conversation_context = conversation_context[
: _trimmable_budget(
@@ -2480,6 +2514,7 @@ class ResearchSupervisor:
notes,
[synthesis_audit, research_state],
report_scaffold_chars,
+ _run_inference_request(run),
)
synthesis_conversation_context = conversation_context[
: _trimmable_budget(
@@ -2568,7 +2603,7 @@ class ResearchSupervisor:
]
recovery_max_tokens = _resolve_max_tokens(
16384,
- run["config"].get("inferenceRequest") or {},
+ _run_inference_request(run),
recovery_messages,
)
(
@@ -2606,7 +2641,7 @@ class ResearchSupervisor:
else:
requested_max_tokens = _resolve_max_tokens(
16384,
- run["config"].get("inferenceRequest") or {},
+ _run_inference_request(run),
synthesis_messages,
)
await self._check_active(run["id"])
@@ -2614,6 +2649,7 @@ class ResearchSupervisor:
truncation_notice = _synthesis_length_limit_error(
synthesis_usage,
requested_max_tokens = requested_max_tokens,
+ inference = _run_inference_request(run),
).rstrip(".")
report = _validate_report_sources(report, sources)
report = _validate_report_document_sources(report, document_sources)
diff --git a/studio/backend/tests/test_research_runs_hardening.py b/studio/backend/tests/test_research_runs_hardening.py
index d603b794267..625233e8bfd 100644
--- a/studio/backend/tests/test_research_runs_hardening.py
+++ b/studio/backend/tests/test_research_runs_hardening.py
@@ -245,11 +245,11 @@ def test_prompt_budget_counts_the_whole_prompt(monkeypatch):
# Budgeting only the evidence cannot prevent an overflow: at a small context the
# untrimmable scaffolding (system prompt, plan, source catalogs) is already several times
# the window, and the old floor added 1500 chars on top of that.
- monkeypatch.setattr(research_runs, "_loaded_context_length", lambda: None)
+ monkeypatch.setattr(research_runs, "_loaded_context_length", lambda _inf = None: None)
assert research_runs._prompt_char_budget(4096) is None
assert research_runs._trimmable_budget(None, 99_999, 500) == 500
- monkeypatch.setattr(research_runs, "_loaded_context_length", lambda: 16384)
+ monkeypatch.setattr(research_runs, "_loaded_context_length", lambda _inf = None: 16384)
total = research_runs._prompt_char_budget(4096)
assert total == int((16384 - 4096) * research_runs._SYNTHESIS_EVIDENCE_CHARS_PER_TOKEN)
# A trimmable section never exceeds what is left, and never goes negative.
@@ -259,7 +259,7 @@ def test_prompt_budget_counts_the_whole_prompt(monkeypatch):
def test_resolve_max_tokens_clamps_to_loaded_context(monkeypatch):
- monkeypatch.setattr(research_runs, "_loaded_context_length", lambda: 12_288)
+ monkeypatch.setattr(research_runs, "_loaded_context_length", lambda _inf = None: 12_288)
messages = [{"role": "user", "content": "x" * 33_000}]
prompt_tokens = _estimate_prompt_tokens(messages)
resolved = _resolve_max_tokens(16_384, {}, messages)
@@ -267,6 +267,65 @@ def test_resolve_max_tokens_clamps_to_loaded_context(monkeypatch):
assert resolved < 16_384
+def _pin_local_context(monkeypatch, tokens: int) -> None:
+ import routes.inference as inference_routes
+ monkeypatch.setattr(
+ inference_routes,
+ "get_llama_cpp_backend",
+ lambda: SimpleNamespace(is_loaded = True, context_length = tokens),
+ )
+
+
+_EXTERNAL_INFERENCE = {
+ "model": "gpt-5.6-sol",
+ "providerId": "provider-1",
+ "providerType": "openai_codex",
+ "externalModel": "gpt-5.6-sol",
+}
+
+
+def test_a_run_on_a_saved_connection_is_not_sized_to_the_local_model(monkeypatch):
+ _pin_local_context(monkeypatch, 4096)
+ messages = [{"role": "user", "content": "x" * 4_000}]
+ reserve = research_runs._SYNTHESIS_CONTEXT_RESERVE_TOKENS
+
+ assert _resolve_max_tokens(16_384, {}, messages) == 4_096 - _estimate_prompt_tokens(messages)
+ assert _resolve_max_tokens(16_384, _EXTERNAL_INFERENCE, messages) == 16_384
+
+ assert research_runs._prompt_char_budget(reserve, {}) == 6_144
+ assert research_runs._prompt_char_budget(reserve, _EXTERNAL_INFERENCE) is None
+
+ assert research_runs._synthesis_evidence_budget(0, {}) == 6_144
+ assert (
+ research_runs._synthesis_evidence_budget(0, _EXTERNAL_INFERENCE)
+ == research_runs._MAX_SYNTHESIS_EVIDENCE_CHARS
+ )
+
+
+def test_a_saved_connection_run_is_not_blamed_on_the_loaded_context(monkeypatch):
+ _pin_local_context(monkeypatch, 4096)
+ usage = {"prompt_tokens": 3_000, "completion_tokens": 1_096, "total_tokens": 4_096}
+
+ assert _completion_hit_context_wall(usage, requested_max_tokens = 16_384)
+ assert not _completion_hit_context_wall(
+ usage, requested_max_tokens = 1_096, inference = _EXTERNAL_INFERENCE
+ )
+ assert "Increase Context Length" not in _synthesis_length_limit_error(
+ usage, requested_max_tokens = 1_096, inference = _EXTERNAL_INFERENCE
+ )
+
+ # Synthesis asks for 16_384 and a provider stops at its own output cap, so
+ # completion_tokens < requested is what an ordinary cloud run reports. That is the
+ # shape the notice has to get right, and the case above cannot see it.
+ capped = {"prompt_tokens": 40_000, "completion_tokens": 8_192, "total_tokens": 48_192}
+ message = _synthesis_length_limit_error(
+ capped, requested_max_tokens = 16_384, inference = _EXTERNAL_INFERENCE
+ )
+ assert "Increase Context Length" not in message
+ assert "Local model" not in message
+ assert "output limit" in message
+
+
def test_completion_hit_context_wall_matches_live_probe():
usage = {
"prompt_tokens": 11_032,
@@ -306,7 +365,10 @@ def test_every_research_prompt_path_is_budgeted():
# sections against the loaded context, else the run dies before or after doing the work.
src = Path(research_runs.__file__).read_text(encoding = "utf-8")
for budget in ("planning_total = ", "decision_total = ", "total_budget = "):
- assert f"{budget}_prompt_char_budget(_SYNTHESIS_CONTEXT_RESERVE_TOKENS)" in src
+ assert f"{budget}_prompt_char_budget(" in src
+ call = src.split(f"{budget}_prompt_char_budget(", 1)[1].split(")", 1)[0]
+ assert "_SYNTHESIS_CONTEXT_RESERVE_TOKENS" in call
+ assert "_run_inference_request(run" in call
assert "evidence[-60000:]" not in src
# The question reaches the planner verbatim, so it is budgeted too, but never to nothing.
assert "planning_question = question[" in src
@@ -324,7 +386,7 @@ def test_prompt_budget_never_empties_the_question_or_evidence(monkeypatch):
# A flat 4096-token reserve on the 4096-token GGUF floor made the budget 0, which sliced the
# question to "" so the planner never saw the request. Reserve at most half the window.
for ctx in (1024, 2048, 4096):
- monkeypatch.setattr(research_runs, "_loaded_context_length", lambda c = ctx: c)
+ monkeypatch.setattr(research_runs, "_loaded_context_length", lambda _inf = None, c = ctx: c)
total = research_runs._prompt_char_budget(research_runs._SYNTHESIS_CONTEXT_RESERVE_TOKENS)
assert total is not None and total > 0
assert total < int(ctx * research_runs._SYNTHESIS_EVIDENCE_CHARS_PER_TOKEN)
diff --git a/studio/backend/tests/test_research_runs_storage.py b/studio/backend/tests/test_research_runs_storage.py
index 8e90a9558dd..c283c07ff29 100644
--- a/studio/backend/tests/test_research_runs_storage.py
+++ b/studio/backend/tests/test_research_runs_storage.py
@@ -210,31 +210,31 @@ def test_synthesis_evidence_budget_tracks_loaded_context(monkeypatch):
from core import research_runs as worker
# Unknown context keeps the full cap (backwards compatible).
- monkeypatch.setattr(worker, "_loaded_context_length", lambda: None)
+ monkeypatch.setattr(worker, "_loaded_context_length", lambda _inf = None: None)
assert worker._synthesis_evidence_budget() == worker._MAX_SYNTHESIS_EVIDENCE_CHARS
# A small context shrinks the budget so evidence fits, and the rest of the prompt eats into
# it, but the output reserve is capped at half the window so the budget never collapses to 0
# and empties the prompt (which is worse than a truncated one).
- monkeypatch.setattr(worker, "_loaded_context_length", lambda: 2048)
+ monkeypatch.setattr(worker, "_loaded_context_length", lambda _inf = None: 2048)
small = worker._synthesis_evidence_budget()
assert 0 < small < worker._MAX_SYNTHESIS_EVIDENCE_CHARS
assert worker._synthesis_evidence_budget(small) == 0
# The rest of the prompt counts against the same budget, not just the evidence.
- monkeypatch.setattr(worker, "_loaded_context_length", lambda: 16384)
+ monkeypatch.setattr(worker, "_loaded_context_length", lambda _inf = None: 16384)
roomy = worker._synthesis_evidence_budget()
assert 0 < worker._synthesis_evidence_budget(8_000) < roomy
# A large context uses (and clamps to) the full cap.
- monkeypatch.setattr(worker, "_loaded_context_length", lambda: 32768)
+ monkeypatch.setattr(worker, "_loaded_context_length", lambda _inf = None: 32768)
assert worker._synthesis_evidence_budget() == worker._MAX_SYNTHESIS_EVIDENCE_CHARS
def test_synthesis_context_budgets_model_derived_json_with_evidence(monkeypatch):
from core import research_runs as worker
- monkeypatch.setattr(worker, "_loaded_context_length", lambda: 8192)
+ monkeypatch.setattr(worker, "_loaded_context_length", lambda _inf = None: 8192)
notes = [f"### Step {index}\n" + "evidence " * 2_000 for index in range(6)]
audit = {"thesis": "a" * 3_000}
research_state = {"summary": "s" * 3_000}
@@ -2155,7 +2155,7 @@ def test_auto_scrape_skipped_on_small_context(research_home, monkeypatch):
# grounding is skipped (snippet-only) even when maxAutoScrape is set.
from core import research_runs as worker
- monkeypatch.setattr(worker, "_loaded_context_length", lambda: 2048)
+ monkeypatch.setattr(worker, "_loaded_context_length", lambda _inf = None: 2048)
_create(budgets = _SCRAPE_BUDGETS)
def fake_tool(name, arguments, *args, **kwargs):