Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: Deep Research Retry continues with the current time budget (#13358)

Daniel Hane16ef2d0937c
diff --git a/studio/backend/core/research_runs.py b/studio/backend/core/research_runs.py
index 67eda3aa7d9..4be29cf6542 100644
--- a/studio/backend/core/research_runs.py
+++ b/studio/backend/core/research_runs.py
@@ -2393,7 +2393,10 @@ class ResearchSupervisor:
# The structured inline card renders the plan; no second markdown copy below it.
async def _research(self, run: dict) -> None:
- resuming = run.get("claimedFromStatus") == "running"
+ # Completed steps on a queued run are ones a Retry kept (a fresh plan's steps are all pending).
+ resuming = run.get("claimedFromStatus") == "running" or any(
+ step.get("status") == "completed" for step in run.get("steps") or ()
+ )
fresh = await asyncio.to_thread(db.get_run, run["id"])
if not fresh or not fresh.get("plan"):
raise ValueError("Approved plan is missing")
diff --git a/studio/backend/routes/research_runs.py b/studio/backend/routes/research_runs.py
index 8e6b6f18408..92531cb3196 100644
--- a/studio/backend/routes/research_runs.py
+++ b/studio/backend/routes/research_runs.py
@@ -122,6 +122,19 @@ class ApprovePlan(BaseModel):
planHash: str = Field(min_length = 64, max_length = 64)
+class RetryResearchRun(BaseModel):
+ # Extra fields are ignored, not refused: a newer client must never lose Retry on an older server.
+ budgets: dict[str, int] | None = None
+ websitePolicy: dict[str, list[str]] | None = None
+
+ @field_validator("budgets", mode = "before")
+ @classmethod
+ def _reject_boolean_budgets(cls, value: Any) -> Any:
+ if isinstance(value, dict) and isinstance(value.get("modelTimeoutSeconds"), bool):
+ raise ValueError("modelTimeoutSeconds must be an integer, not a boolean")
+ return value
+
+
def _require_run(run_id: str) -> dict:
run = db.get_run(run_id)
if run is None:
@@ -547,10 +560,31 @@ def retry_research_run(
run_id: str,
request: Request,
current_subject: str = Depends(get_current_subject),
+ payload: RetryResearchRun | None = None,
):
_require_run(run_id)
+ model_timeout = None
+ website_policy = None
+ if payload is not None:
+ if payload.budgets and "modelTimeoutSeconds" in payload.budgets:
+ model_timeout = int(payload.budgets["modelTimeoutSeconds"])
+ if model_timeout != 0 and not (
+ _MIN_FINITE_MODEL_TIMEOUT_SECONDS
+ <= model_timeout
+ <= _MAX_FINITE_MODEL_TIMEOUT_SECONDS
+ ):
+ raise HTTPException(
+ status_code = 400,
+ detail = "modelTimeoutSeconds must be 0 (unlimited) or between "
+ f"{_MIN_FINITE_MODEL_TIMEOUT_SECONDS} and {_MAX_FINITE_MODEL_TIMEOUT_SECONDS}",
+ )
+ if payload.websitePolicy is not None:
+ try:
+ website_policy = normalize_website_policy(payload.websitePolicy)
+ except ValueError as exc:
+ raise HTTPException(status_code = 400, detail = str(exc)) from exc
try:
- db.retry(run_id)
+ db.retry(run_id, model_timeout_seconds = model_timeout, website_policy = website_policy)
except (db.ResearchConflictError, KeyError) as exc:
raise HTTPException(status_code = 409, detail = str(exc)) from exc
supervisor = getattr(request.app.state, "research_supervisor", None)
diff --git a/studio/backend/storage/research_runs_db.py b/studio/backend/storage/research_runs_db.py
index 2ae2adaf261..6913733cc59 100644
--- a/studio/backend/storage/research_runs_db.py
+++ b/studio/backend/storage/research_runs_db.py
@@ -13,7 +13,7 @@ import threading
import time
from typing import Any
-from core.inference.web_access_policy import check_url_access
+from core.inference.web_access_policy import check_url_access, normalize_website_policy
from storage.studio_db import get_connection as _studio_connection
ACTIVE_STATUSES = frozenset(
@@ -787,12 +787,18 @@ def request_cancel(run_id: str) -> str:
conn.close()
-def retry(run_id: str, max_retries: int = 3) -> str:
+def retry(
+ run_id: str,
+ max_retries: int = 3,
+ *,
+ model_timeout_seconds: int | None = None,
+ website_policy: dict[str, list[str]] | None = None,
+) -> str:
conn = get_connection()
try:
conn.execute("BEGIN IMMEDIATE")
row = conn.execute(
- "SELECT status, retry_count, plan_json, owner_subject, thread_id "
+ "SELECT status, retry_count, plan_json, owner_subject, thread_id, config_json "
"FROM research_runs WHERE id = ?",
(run_id,),
).fetchone()
@@ -841,16 +847,29 @@ def retry(run_id: str, max_retries: int = 3) -> str:
if row["plan_json"]
else "planning"
)
+ config = _loads(row["config_json"], {})
+ policy_changed = website_policy is not None and website_policy != normalize_website_policy(
+ config.get("websitePolicy")
+ )
+ if model_timeout_seconds is not None:
+ config.setdefault("budgets", {})["modelTimeoutSeconds"] = int(model_timeout_seconds)
+ if website_policy is not None:
+ config["websitePolicy"] = website_policy
conn.execute(
"UPDATE research_runs SET status = ?, cancel_requested = 0, retry_count = retry_count + 1, "
"error_message = NULL, report_text = NULL, completed_at = NULL, lease_owner = NULL, "
- "lease_expires_at = NULL, updated_at = ? WHERE id = ?",
- (status, now_ms(), run_id),
+ "lease_expires_at = NULL, config_json = ?, updated_at = ? WHERE id = ?",
+ (status, json.dumps(config, ensure_ascii = False), now_ms(), run_id),
)
- if status != "awaiting_approval":
- conn.execute("DELETE FROM research_plan_steps WHERE run_id = ?", (run_id,))
- conn.execute("DELETE FROM research_sources WHERE run_id = ?", (run_id,))
- conn.execute("DELETE FROM research_document_sources WHERE run_id = ?", (run_id,))
+ # Only a failed run keeps evidence, and only completed steps, so a transiently failed search runs again.
+ keep_evidence = row["status"] == "failed" and status == "queued" and not policy_changed
+ if keep_evidence:
+ _drop_unfinished_steps_locked(conn, run_id, ("completed",))
+ else:
+ if status != "awaiting_approval":
+ conn.execute("DELETE FROM research_plan_steps WHERE run_id = ?", (run_id,))
+ conn.execute("DELETE FROM research_sources WHERE run_id = ?", (run_id,))
+ conn.execute("DELETE FROM research_document_sources WHERE run_id = ?", (run_id,))
_event_locked(conn, run_id, "run.retried", {"status": status})
_commit_event(conn)
return status
@@ -1101,24 +1120,7 @@ def prepare_execution_resume(run_id: str, worker_id: str) -> bool:
if not _worker_can_write_locked(conn, run_id, worker_id, {"running"}):
conn.commit()
return False
- interrupted = conn.execute(
- "SELECT position FROM research_plan_steps WHERE run_id = ? "
- "AND status NOT IN ('completed','failed')",
- (run_id,),
- ).fetchall()
- conn.executemany(
- "DELETE FROM research_sources WHERE run_id = ? AND step_position = ?",
- [(run_id, int(row["position"])) for row in interrupted],
- )
- conn.executemany(
- "DELETE FROM research_document_sources WHERE run_id = ? AND step_position = ?",
- [(run_id, int(row["position"])) for row in interrupted],
- )
- conn.execute(
- "DELETE FROM research_plan_steps WHERE run_id = ? "
- "AND status NOT IN ('completed','failed')",
- (run_id,),
- )
+ _drop_unfinished_steps_locked(conn, run_id)
conn.commit()
return True
except Exception:
@@ -1128,6 +1130,30 @@ def prepare_execution_resume(run_id: str, worker_id: str) -> bool:
conn.close()
+def _drop_unfinished_steps_locked(
+ conn: sqlite3.Connection,
+ run_id: str,
+ keep: tuple[str, ...] = ("completed", "failed"),
+) -> None:
+ marks = ",".join("?" for _ in keep)
+ interrupted = conn.execute(
+ f"SELECT position FROM research_plan_steps WHERE run_id = ? AND status NOT IN ({marks})",
+ (run_id, *keep),
+ ).fetchall()
+ conn.executemany(
+ "DELETE FROM research_sources WHERE run_id = ? AND step_position = ?",
+ [(run_id, int(row["position"])) for row in interrupted],
+ )
+ conn.executemany(
+ "DELETE FROM research_document_sources WHERE run_id = ? AND step_position = ?",
+ [(run_id, int(row["position"])) for row in interrupted],
+ )
+ conn.execute(
+ f"DELETE FROM research_plan_steps WHERE run_id = ? AND status NOT IN ({marks})",
+ (run_id, *keep),
+ )
+
+
def upsert_execution_step(
run_id: str,
position: int,
diff --git a/studio/backend/tests/test_research_runs_storage.py b/studio/backend/tests/test_research_runs_storage.py
index de1a60d0187..f71bf619382 100644
--- a/studio/backend/tests/test_research_runs_storage.py
+++ b/studio/backend/tests/test_research_runs_storage.py
@@ -1542,8 +1542,8 @@ def test_retry_is_bounded_and_resumes_from_saved_plan(research_home):
retried = research_db.get_run("run-1")
assert retried["retryCount"] == 1
assert retried["report"] is None
- assert retried["steps"] == []
- assert retried["sources"] == []
+ assert [step["title"] for step in retried["steps"]] == ["Old step"]
+ assert [source["url"] for source in retried["sources"]] == ["https://old.example"]
assert research_db.get_reasoning_text("run-1") == ""
assert research_db.list_events("run-1")[-1]["data"]["attempt"] == 1
research_db.claim_next("worker-2")
@@ -1552,6 +1552,180 @@ def test_retry_is_bounded_and_resumes_from_saved_plan(research_home):
research_db.retry("run-1", max_retries = 1)
+def _failed_run_with_saved_step():
+ _shared_setup_2()
+ research_db.upsert_execution_step("run-1", 0, "Saved", "saved", "completed")
+ research_db.upsert_source("run-1", 0, "https://saved.example", "Saved", "Saved evidence")
+ research_db.upsert_execution_step("run-1", 1, "Interrupted", "partial", "running")
+ research_db.upsert_source("run-1", 1, "https://partial.example", "Partial", "Partial")
+ research_db.finish(
+ "run-1", "worker-1", "failed", "Local model request exhausted its total time budget"
+ )
+
+
+def test_retry_takes_the_new_time_budget_and_keeps_finished_steps(research_home):
+ _failed_run_with_saved_step()
+ policy = research_db.get_run("run-1")["config"].get("websitePolicy")
+
+ assert research_db.retry("run-1", model_timeout_seconds = 0) == "queued"
+ retried = research_db.get_run("run-1")
+ assert retried["config"]["budgets"]["modelTimeoutSeconds"] == 0
+ assert retried["config"].get("websitePolicy") == policy
+ assert [step["title"] for step in retried["steps"]] == ["Saved"]
+ assert [source["url"] for source in retried["sources"]] == ["https://saved.example"]
+
+
+def test_retry_drops_failed_steps_so_failed_searches_run_again(research_home):
+ _shared_setup_2()
+ research_db.upsert_execution_step("run-1", 0, "Kept", "kept", "completed")
+ research_db.upsert_source("run-1", 0, "https://kept.example", "Kept", "Kept evidence")
+ research_db.upsert_execution_step("run-1", 1, "Search failed", "flaky", "failed")
+ research_db.finish("run-1", "worker-1", "failed", "No research evidence was gathered")
+
+ assert research_db.retry("run-1") == "queued"
+ assert [step["title"] for step in research_db.get_run("run-1")["steps"]] == ["Kept"]
+
+
+def test_retry_with_changed_website_access_starts_over(research_home):
+ _failed_run_with_saved_step()
+ policy = {"allowedDomains": [], "blockedDomains": ["saved.example"]}
+
+ assert research_db.retry("run-1", website_policy = policy) == "queued"
+ retried = research_db.get_run("run-1")
+ assert retried["config"]["websitePolicy"] == policy
+ assert retried["steps"] == []
+ assert retried["sources"] == []
+
+
+def test_retry_of_a_stopped_run_starts_over(research_home):
+ _shared_setup_2()
+ research_db.upsert_execution_step("run-1", 0, "Saved", "saved", "completed")
+ research_db.upsert_source("run-1", 0, "https://saved.example", "Saved", "Saved evidence")
+ assert research_db.request_cancel("run-1") == "cancelling"
+ research_db.finish("run-1", "worker-1", "cancelled")
+
+ assert research_db.retry("run-1", model_timeout_seconds = 0) == "queued"
+ retried = research_db.get_run("run-1")
+ assert retried["config"]["budgets"]["modelTimeoutSeconds"] == 0
+ assert retried["steps"] == []
+ assert retried["sources"] == []
+
+
+def _retry_client():
+ from fastapi import FastAPI
+ from fastapi.testclient import TestClient
+
+ from auth.authentication import get_current_subject
+ from routes import research_runs as research_route
+
+ app = FastAPI()
+ app.include_router(research_route.router, prefix = "/runs")
+ app.dependency_overrides[get_current_subject] = lambda: "alice"
+ return TestClient(app)
+
+
+def test_retry_route_without_a_body_keeps_the_saved_settings(research_home):
+ _failed_run_with_saved_step()
+ before = research_db.get_run("run-1")["config"]
+
+ with _retry_client() as client:
+ response = client.post("/runs/run-1/retry")
+ assert response.status_code == 200, response.text
+ after = research_db.get_run("run-1")["config"]
+ assert after["budgets"] == before["budgets"]
+ assert after.get("websitePolicy") == before.get("websitePolicy")
+
+
+def test_retry_route_validates_and_applies_the_sent_settings(research_home):
+ _failed_run_with_saved_step()
+
+ with _retry_client() as client:
+ refused = client.post("/runs/run-1/retry", json = {"budgets": {"modelTimeoutSeconds": 3}})
+ assert refused.status_code == 400
+ assert research_db.get_run("run-1")["status"] == "failed"
+ refused = client.post("/runs/run-1/retry", json = {"budgets": {"modelTimeoutSeconds": True}})
+ assert refused.status_code == 422
+ response = client.post(
+ "/runs/run-1/retry",
+ json = {
+ "budgets": {"modelTimeoutSeconds": 7200},
+ "websitePolicy": {"allowedDomains": [], "blockedDomains": ["Example.COM"]},
+ "futureSetting": True,
+ },
+ )
+ assert response.status_code == 200, response.text
+ config = research_db.get_run("run-1")["config"]
+ assert config["budgets"]["modelTimeoutSeconds"] == 7200
+ assert config["websitePolicy"] == {"allowedDomains": [], "blockedDomains": ["example.com"]}
+
+
+def test_retry_after_a_timed_out_report_continues_without_redoing_research(
+ research_home, monkeypatch
+):
+ """#11837: Retry with a bigger budget reuses the gathered evidence instead of searching again."""
+ from core import research_runs as worker
+
+ _patch_web_rank(monkeypatch)
+ supervisor = worker.ResearchSupervisor(SimpleNamespace(state = SimpleNamespace(server_port = 1)))
+ decisions = iter(
+ [
+ json.dumps({"action": "search", "title": "Find", "query": "grounding evidence"}),
+ json.dumps({"action": "finish", "title": "Enough"}),
+ json.dumps({"action": "finish", "title": "Still enough"}),
+ ]
+ )
+ synthesis_attempts = []
+
+ async def fake_stream_completion(run, messages, **kwargs):
+ system = messages[0]["content"]
+ if "rigorous web research plan" in system:
+ return json.dumps(_plan()), "", "stop", None
+ if "iterative research process" in system:
+ return next(decisions), "", "stop", None
+ if "evidence-to-claim audit" in system:
+ return json.dumps({"supportedClaims": []}), "", "stop", None
+ synthesis_attempts.append(run["config"]["budgets"]["modelTimeoutSeconds"])
+ if len(synthesis_attempts) == 1:
+ raise worker.ModelWallClockTimeout("wall clock")
+ assert "alpha snippet" in messages[1]["content"]
+ report = "# Report\n\nFinding [Alpha](https://a.example.com)."
+ research_db.set_report_progress(run["id"], report)
+ return report, "", "stop", None
+
+ searches = []
+
+ def fake_tool(name, arguments, *args, **kwargs):
+ searches.append(arguments)
+ return _two_source_search()
+
+ _create()
+ monkeypatch.setattr(supervisor, "_stream_completion", fake_stream_completion)
+ monkeypatch.setattr(worker, "execute_tool", fake_tool)
+ asyncio.run(supervisor._process(research_db.claim_next(supervisor.worker_id)))
+ planned = research_db.get_run("run-1")
+ research_db.approve("run-1", planned["planRevision"], planned["planHash"])
+ asyncio.run(supervisor._process(research_db.claim_next(supervisor.worker_id)))
+ failed = research_db.get_run("run-1")
+ assert failed["status"] == "failed"
+ assert failed["error"] == "Local model request exhausted its total time budget"
+ assert len(searches) == 1
+
+ with _retry_client() as client:
+ response = client.post("/runs/run-1/retry", json = {"budgets": {"modelTimeoutSeconds": 0}})
+ assert response.status_code == 200, response.text
+ asyncio.run(supervisor._process(research_db.claim_next(supervisor.worker_id)))
+
+ completed = research_db.get_run("run-1")
+ assert completed["status"] == "completed", completed["error"]
+ assert synthesis_attempts == [30, 0]
+ assert len(searches) == 1
+ assert {source["url"] for source in completed["sources"]} == {
+ "https://a.example.com",
+ "https://b.example.com",
+ }
+ assert completed["report"].startswith("# Report")
+
+
def test_retry_of_unapproved_plan_requires_approval_again(research_home):
_create()
plan = research_db.set_plan("run-1", _plan())
diff --git a/studio/frontend/src/features/chat/api/research-api.ts b/studio/frontend/src/features/chat/api/research-api.ts
index 05f7fdb82e5..74b07d77f81 100644
--- a/studio/frontend/src/features/chat/api/research-api.ts
+++ b/studio/frontend/src/features/chat/api/research-api.ts
@@ -7,6 +7,7 @@ import type {
ResearchEvent,
ResearchPlan,
ResearchRun,
+ ResearchWebsitePolicy,
} from "../types/research";
type StreamResearchEvent = Omit<ResearchEvent, "data" | "run"> & {
@@ -124,7 +125,26 @@ export const approveResearchRun = (
planHash: string,
) => mutate(id, "approve", { planRevision, planHash });
export const cancelResearchRun = (id: string) => mutate(id, "cancel");
-export const retryResearchRun = (id: string) => mutate(id, "retry");
+export const retryResearchRun = (
+ id: string,
+ settings?: {
+ modelTimeoutSeconds: number;
+ websitePolicy: ResearchWebsitePolicy;
+ },
+) =>
+ mutate(
+ id,
+ "retry",
+ settings
+ ? {
+ budgets: { modelTimeoutSeconds: settings.modelTimeoutSeconds },
+ websitePolicy: {
+ allowedDomains: [...settings.websitePolicy.allowedDomains],
+ blockedDomains: [...settings.websitePolicy.blockedDomains],
+ },
+ }
+ : undefined,
+ );
export async function updateResearchPlan(
id: string,
diff --git a/studio/frontend/src/features/chat/components/research-activity-panel.tsx b/studio/frontend/src/features/chat/components/research-activity-panel.tsx
index ec20d219a24..c93221de7b1 100644
--- a/studio/frontend/src/features/chat/components/research-activity-panel.tsx
+++ b/studio/frontend/src/features/chat/components/research-activity-panel.tsx
@@ -62,6 +62,7 @@ import {
retryResearchRun,
updateResearchPlan,
} from "../api/research-api";
+import { useChatRuntimeStore } from "../stores/chat-runtime-store";
import {
type ResearchActivity,
ensureResearchRunFollowed,
@@ -799,7 +800,11 @@ function ResearchActions({ runId }: { runId: string }): ReactElement | null {
const retry = async () => {
setPending(true);
try {
- const retried = await retryResearchRun(run.id);
+ const settings = useChatRuntimeStore.getState();
+ const retried = await retryResearchRun(run.id, {
+ modelTimeoutSeconds: settings.researchModelTimeoutSeconds,
+ websitePolicy: settings.researchWebsitePolicy,
+ });
ingestResearchUpdate(retried);
useResearchRunStore.getState().setConnectionError(retried.id, null);
ensureResearchRunFollowed(retried.id, retried);
diff --git a/studio/frontend/tests/research-retry-settings.test.ts b/studio/frontend/tests/research-retry-settings.test.ts
new file mode 100644
index 00000000000..d38aba92f8e
--- /dev/null
+++ b/studio/frontend/tests/research-retry-settings.test.ts
@@ -0,0 +1,62 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+// #11837: Retry sends the current time budget and website access.
+
+import assert from "node:assert/strict";
+import { register } from "node:module";
+import test from "node:test";
+
+import {
+ installLocalStorageFake,
+ registerBundlerResolver,
+} from "./helpers/kit.ts";
+
+register("./helpers/settings-api-resolver.mjs", import.meta.url);
+registerBundlerResolver();
+installLocalStorageFake();
+
+const { retryResearchRun } = await import(
+ "../src/features/chat/api/research-api.ts"
+);
+
+async function captureRetry(
+ call: () => Promise<unknown>,
+): Promise<{ url: string; init: RequestInit | undefined }> {
+ const original = globalThis.fetch;
+ let seen: { url: string; init: RequestInit | undefined } | null = null;
+ globalThis.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => {
+ seen = { url: String(input), init };
+ return new Response(JSON.stringify({ id: "run-1", status: "queued" }), {
+ status: 200,
+ headers: { "content-type": "application/json" },
+ });
+ }) as typeof globalThis.fetch;
+ try {
+ await call();
+ } finally {
+ globalThis.fetch = original;
+ }
+ assert.ok(seen);
+ return seen;
+}
+
+test("retry sends the current time budget and website access", async () => {
+ const { url, init } = await captureRetry(() =>
+ retryResearchRun("run-1", {
+ modelTimeoutSeconds: 0,
+ websitePolicy: { allowedDomains: [], blockedDomains: ["example.com"] },
+ }),
+ );
+ assert.match(url, /\/api\/chat\/research-runs\/run-1\/retry$/);
+ assert.equal(init?.method, "POST");
+ assert.deepEqual(JSON.parse(String(init?.body)), {
+ budgets: { modelTimeoutSeconds: 0 },
+ websitePolicy: { allowedDomains: [], blockedDomains: ["example.com"] },
+ });
+});
+
+test("retry without settings sends no body", async () => {
+ const { init } = await captureRetry(() => retryResearchRun("run-1"));
+ assert.equal(init?.body, undefined);
+});
diff --git a/tests/studio/test_deep_research_frontend_contract.py b/tests/studio/test_deep_research_frontend_contract.py
index 8048352fc53..0c04f16975c 100644
--- a/tests/studio/test_deep_research_frontend_contract.py
+++ b/tests/studio/test_deep_research_frontend_contract.py
@@ -37,8 +37,9 @@ def test_research_api_is_isolated_and_cursor_based() -> None:
# The store owns the stream, so nothing else can stall ingestion by not reading.
assert "ensureResearchRunFollowed(run.id, run);" in store
assert "export async function* watchResearchRun" in store
- for action in ("cancel", "retry"):
- assert f'mutate(id, "{action}")' in api
+ assert 'mutate(id, "cancel")' in api
+ # Retry carries the current time budget and website access (#11837).
+ assert re.search(r'mutate\(\s*id,\s*"retry",', api)
assert 'mutate(id, "approve", { planRevision, planHash })' in api
assert "JSON.stringify({ plan, expectedRevision })" in api