Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: Deep Research Retry continues with the current time budget (#13358)
Daniel Hane16ef2d0937c
diff --git a/studio/backend/core/research_runs.py b/studio/backend/core/research_runs.pyindex 67eda3aa7d9..4be29cf6542 100644--- a/studio/backend/core/research_runs.py+++ b/studio/backend/core/research_runs.py@@ -2393,7 +2393,10 @@ class ResearchSupervisor:# The structured inline card renders the plan; no second markdown copy below it.async def _research(self, run: dict) -> None:- resuming = run.get("claimedFromStatus") == "running"+ # Completed steps on a queued run are ones a Retry kept (a fresh plan's steps are all pending).+ resuming = run.get("claimedFromStatus") == "running" or any(+ step.get("status") == "completed" for step in run.get("steps") or ()+ )fresh = await asyncio.to_thread(db.get_run, run["id"])if not fresh or not fresh.get("plan"):raise ValueError("Approved plan is missing")diff --git a/studio/backend/routes/research_runs.py b/studio/backend/routes/research_runs.pyindex 8e6b6f18408..92531cb3196 100644--- a/studio/backend/routes/research_runs.py+++ b/studio/backend/routes/research_runs.py@@ -122,6 +122,19 @@ class ApprovePlan(BaseModel):planHash: str = Field(min_length = 64, max_length = 64)+class RetryResearchRun(BaseModel):+ # Extra fields are ignored, not refused: a newer client must never lose Retry on an older server.+ budgets: dict[str, int] | None = None+ websitePolicy: dict[str, list[str]] | None = None++ @field_validator("budgets", mode = "before")+ @classmethod+ def _reject_boolean_budgets(cls, value: Any) -> Any:+ if isinstance(value, dict) and isinstance(value.get("modelTimeoutSeconds"), bool):+ raise ValueError("modelTimeoutSeconds must be an integer, not a boolean")+ return value++def _require_run(run_id: str) -> dict:run = db.get_run(run_id)if run is None:@@ -547,10 +560,31 @@ def retry_research_run(run_id: str,request: Request,current_subject: str = Depends(get_current_subject),+ payload: RetryResearchRun | None = None,):_require_run(run_id)+ model_timeout = None+ website_policy = None+ if payload is not None:+ if payload.budgets and "modelTimeoutSeconds" in payload.budgets:+ model_timeout = int(payload.budgets["modelTimeoutSeconds"])+ if model_timeout != 0 and not (+ _MIN_FINITE_MODEL_TIMEOUT_SECONDS+ <= model_timeout+ <= _MAX_FINITE_MODEL_TIMEOUT_SECONDS+ ):+ raise HTTPException(+ status_code = 400,+ detail = "modelTimeoutSeconds must be 0 (unlimited) or between "+ f"{_MIN_FINITE_MODEL_TIMEOUT_SECONDS} and {_MAX_FINITE_MODEL_TIMEOUT_SECONDS}",+ )+ if payload.websitePolicy is not None:+ try:+ website_policy = normalize_website_policy(payload.websitePolicy)+ except ValueError as exc:+ raise HTTPException(status_code = 400, detail = str(exc)) from exctry:- db.retry(run_id)+ db.retry(run_id, model_timeout_seconds = model_timeout, website_policy = website_policy)except (db.ResearchConflictError, KeyError) as exc:raise HTTPException(status_code = 409, detail = str(exc)) from excsupervisor = getattr(request.app.state, "research_supervisor", None)diff --git a/studio/backend/storage/research_runs_db.py b/studio/backend/storage/research_runs_db.pyindex 2ae2adaf261..6913733cc59 100644--- a/studio/backend/storage/research_runs_db.py+++ b/studio/backend/storage/research_runs_db.py@@ -13,7 +13,7 @@ import threadingimport timefrom typing import Any-from core.inference.web_access_policy import check_url_access+from core.inference.web_access_policy import check_url_access, normalize_website_policyfrom storage.studio_db import get_connection as _studio_connectionACTIVE_STATUSES = frozenset(@@ -787,12 +787,18 @@ def request_cancel(run_id: str) -> str:conn.close()-def retry(run_id: str, max_retries: int = 3) -> str:+def retry(+ run_id: str,+ max_retries: int = 3,+ *,+ model_timeout_seconds: int | None = None,+ website_policy: dict[str, list[str]] | None = None,+) -> str:conn = get_connection()try:conn.execute("BEGIN IMMEDIATE")row = conn.execute(- "SELECT status, retry_count, plan_json, owner_subject, thread_id "+ "SELECT status, retry_count, plan_json, owner_subject, thread_id, config_json ""FROM research_runs WHERE id = ?",(run_id,),).fetchone()@@ -841,16 +847,29 @@ def retry(run_id: str, max_retries: int = 3) -> str:if row["plan_json"]else "planning")+ config = _loads(row["config_json"], {})+ policy_changed = website_policy is not None and website_policy != normalize_website_policy(+ config.get("websitePolicy")+ )+ if model_timeout_seconds is not None:+ config.setdefault("budgets", {})["modelTimeoutSeconds"] = int(model_timeout_seconds)+ if website_policy is not None:+ config["websitePolicy"] = website_policyconn.execute("UPDATE research_runs SET status = ?, cancel_requested = 0, retry_count = retry_count + 1, ""error_message = NULL, report_text = NULL, completed_at = NULL, lease_owner = NULL, "- "lease_expires_at = NULL, updated_at = ? WHERE id = ?",- (status, now_ms(), run_id),+ "lease_expires_at = NULL, config_json = ?, updated_at = ? WHERE id = ?",+ (status, json.dumps(config, ensure_ascii = False), now_ms(), run_id),)- if status != "awaiting_approval":- conn.execute("DELETE FROM research_plan_steps WHERE run_id = ?", (run_id,))- conn.execute("DELETE FROM research_sources WHERE run_id = ?", (run_id,))- conn.execute("DELETE FROM research_document_sources WHERE run_id = ?", (run_id,))+ # Only a failed run keeps evidence, and only completed steps, so a transiently failed search runs again.+ keep_evidence = row["status"] == "failed" and status == "queued" and not policy_changed+ if keep_evidence:+ _drop_unfinished_steps_locked(conn, run_id, ("completed",))+ else:+ if status != "awaiting_approval":+ conn.execute("DELETE FROM research_plan_steps WHERE run_id = ?", (run_id,))+ conn.execute("DELETE FROM research_sources WHERE run_id = ?", (run_id,))+ conn.execute("DELETE FROM research_document_sources WHERE run_id = ?", (run_id,))_event_locked(conn, run_id, "run.retried", {"status": status})_commit_event(conn)return status@@ -1101,24 +1120,7 @@ def prepare_execution_resume(run_id: str, worker_id: str) -> bool:if not _worker_can_write_locked(conn, run_id, worker_id, {"running"}):conn.commit()return False- interrupted = conn.execute(- "SELECT position FROM research_plan_steps WHERE run_id = ? "- "AND status NOT IN ('completed','failed')",- (run_id,),- ).fetchall()- conn.executemany(- "DELETE FROM research_sources WHERE run_id = ? AND step_position = ?",- [(run_id, int(row["position"])) for row in interrupted],- )- conn.executemany(- "DELETE FROM research_document_sources WHERE run_id = ? AND step_position = ?",- [(run_id, int(row["position"])) for row in interrupted],- )- conn.execute(- "DELETE FROM research_plan_steps WHERE run_id = ? "- "AND status NOT IN ('completed','failed')",- (run_id,),- )+ _drop_unfinished_steps_locked(conn, run_id)conn.commit()return Trueexcept Exception:@@ -1128,6 +1130,30 @@ def prepare_execution_resume(run_id: str, worker_id: str) -> bool:conn.close()+def _drop_unfinished_steps_locked(+ conn: sqlite3.Connection,+ run_id: str,+ keep: tuple[str, ...] = ("completed", "failed"),+) -> None:+ marks = ",".join("?" for _ in keep)+ interrupted = conn.execute(+ f"SELECT position FROM research_plan_steps WHERE run_id = ? AND status NOT IN ({marks})",+ (run_id, *keep),+ ).fetchall()+ conn.executemany(+ "DELETE FROM research_sources WHERE run_id = ? AND step_position = ?",+ [(run_id, int(row["position"])) for row in interrupted],+ )+ conn.executemany(+ "DELETE FROM research_document_sources WHERE run_id = ? AND step_position = ?",+ [(run_id, int(row["position"])) for row in interrupted],+ )+ conn.execute(+ f"DELETE FROM research_plan_steps WHERE run_id = ? AND status NOT IN ({marks})",+ (run_id, *keep),+ )++def upsert_execution_step(run_id: str,position: int,diff --git a/studio/backend/tests/test_research_runs_storage.py b/studio/backend/tests/test_research_runs_storage.pyindex de1a60d0187..f71bf619382 100644--- a/studio/backend/tests/test_research_runs_storage.py+++ b/studio/backend/tests/test_research_runs_storage.py@@ -1542,8 +1542,8 @@ def test_retry_is_bounded_and_resumes_from_saved_plan(research_home):retried = research_db.get_run("run-1")assert retried["retryCount"] == 1assert retried["report"] is None- assert retried["steps"] == []- assert retried["sources"] == []+ assert [step["title"] for step in retried["steps"]] == ["Old step"]+ assert [source["url"] for source in retried["sources"]] == ["https://old.example"]assert research_db.get_reasoning_text("run-1") == ""assert research_db.list_events("run-1")[-1]["data"]["attempt"] == 1research_db.claim_next("worker-2")@@ -1552,6 +1552,180 @@ def test_retry_is_bounded_and_resumes_from_saved_plan(research_home):research_db.retry("run-1", max_retries = 1)+def _failed_run_with_saved_step():+ _shared_setup_2()+ research_db.upsert_execution_step("run-1", 0, "Saved", "saved", "completed")+ research_db.upsert_source("run-1", 0, "https://saved.example", "Saved", "Saved evidence")+ research_db.upsert_execution_step("run-1", 1, "Interrupted", "partial", "running")+ research_db.upsert_source("run-1", 1, "https://partial.example", "Partial", "Partial")+ research_db.finish(+ "run-1", "worker-1", "failed", "Local model request exhausted its total time budget"+ )+++def test_retry_takes_the_new_time_budget_and_keeps_finished_steps(research_home):+ _failed_run_with_saved_step()+ policy = research_db.get_run("run-1")["config"].get("websitePolicy")++ assert research_db.retry("run-1", model_timeout_seconds = 0) == "queued"+ retried = research_db.get_run("run-1")+ assert retried["config"]["budgets"]["modelTimeoutSeconds"] == 0+ assert retried["config"].get("websitePolicy") == policy+ assert [step["title"] for step in retried["steps"]] == ["Saved"]+ assert [source["url"] for source in retried["sources"]] == ["https://saved.example"]+++def test_retry_drops_failed_steps_so_failed_searches_run_again(research_home):+ _shared_setup_2()+ research_db.upsert_execution_step("run-1", 0, "Kept", "kept", "completed")+ research_db.upsert_source("run-1", 0, "https://kept.example", "Kept", "Kept evidence")+ research_db.upsert_execution_step("run-1", 1, "Search failed", "flaky", "failed")+ research_db.finish("run-1", "worker-1", "failed", "No research evidence was gathered")++ assert research_db.retry("run-1") == "queued"+ assert [step["title"] for step in research_db.get_run("run-1")["steps"]] == ["Kept"]+++def test_retry_with_changed_website_access_starts_over(research_home):+ _failed_run_with_saved_step()+ policy = {"allowedDomains": [], "blockedDomains": ["saved.example"]}++ assert research_db.retry("run-1", website_policy = policy) == "queued"+ retried = research_db.get_run("run-1")+ assert retried["config"]["websitePolicy"] == policy+ assert retried["steps"] == []+ assert retried["sources"] == []+++def test_retry_of_a_stopped_run_starts_over(research_home):+ _shared_setup_2()+ research_db.upsert_execution_step("run-1", 0, "Saved", "saved", "completed")+ research_db.upsert_source("run-1", 0, "https://saved.example", "Saved", "Saved evidence")+ assert research_db.request_cancel("run-1") == "cancelling"+ research_db.finish("run-1", "worker-1", "cancelled")++ assert research_db.retry("run-1", model_timeout_seconds = 0) == "queued"+ retried = research_db.get_run("run-1")+ assert retried["config"]["budgets"]["modelTimeoutSeconds"] == 0+ assert retried["steps"] == []+ assert retried["sources"] == []+++def _retry_client():+ from fastapi import FastAPI+ from fastapi.testclient import TestClient++ from auth.authentication import get_current_subject+ from routes import research_runs as research_route++ app = FastAPI()+ app.include_router(research_route.router, prefix = "/runs")+ app.dependency_overrides[get_current_subject] = lambda: "alice"+ return TestClient(app)+++def test_retry_route_without_a_body_keeps_the_saved_settings(research_home):+ _failed_run_with_saved_step()+ before = research_db.get_run("run-1")["config"]++ with _retry_client() as client:+ response = client.post("/runs/run-1/retry")+ assert response.status_code == 200, response.text+ after = research_db.get_run("run-1")["config"]+ assert after["budgets"] == before["budgets"]+ assert after.get("websitePolicy") == before.get("websitePolicy")+++def test_retry_route_validates_and_applies_the_sent_settings(research_home):+ _failed_run_with_saved_step()++ with _retry_client() as client:+ refused = client.post("/runs/run-1/retry", json = {"budgets": {"modelTimeoutSeconds": 3}})+ assert refused.status_code == 400+ assert research_db.get_run("run-1")["status"] == "failed"+ refused = client.post("/runs/run-1/retry", json = {"budgets": {"modelTimeoutSeconds": True}})+ assert refused.status_code == 422+ response = client.post(+ "/runs/run-1/retry",+ json = {+ "budgets": {"modelTimeoutSeconds": 7200},+ "websitePolicy": {"allowedDomains": [], "blockedDomains": ["Example.COM"]},+ "futureSetting": True,+ },+ )+ assert response.status_code == 200, response.text+ config = research_db.get_run("run-1")["config"]+ assert config["budgets"]["modelTimeoutSeconds"] == 7200+ assert config["websitePolicy"] == {"allowedDomains": [], "blockedDomains": ["example.com"]}+++def test_retry_after_a_timed_out_report_continues_without_redoing_research(+ research_home, monkeypatch+):+ """#11837: Retry with a bigger budget reuses the gathered evidence instead of searching again."""+ from core import research_runs as worker++ _patch_web_rank(monkeypatch)+ supervisor = worker.ResearchSupervisor(SimpleNamespace(state = SimpleNamespace(server_port = 1)))+ decisions = iter(+ [+ json.dumps({"action": "search", "title": "Find", "query": "grounding evidence"}),+ json.dumps({"action": "finish", "title": "Enough"}),+ json.dumps({"action": "finish", "title": "Still enough"}),+ ]+ )+ synthesis_attempts = []++ async def fake_stream_completion(run, messages, **kwargs):+ system = messages[0]["content"]+ if "rigorous web research plan" in system:+ return json.dumps(_plan()), "", "stop", None+ if "iterative research process" in system:+ return next(decisions), "", "stop", None+ if "evidence-to-claim audit" in system:+ return json.dumps({"supportedClaims": []}), "", "stop", None+ synthesis_attempts.append(run["config"]["budgets"]["modelTimeoutSeconds"])+ if len(synthesis_attempts) == 1:+ raise worker.ModelWallClockTimeout("wall clock")+ assert "alpha snippet" in messages[1]["content"]+ report = "# Report\n\nFinding [Alpha](https://a.example.com)."+ research_db.set_report_progress(run["id"], report)+ return report, "", "stop", None++ searches = []++ def fake_tool(name, arguments, *args, **kwargs):+ searches.append(arguments)+ return _two_source_search()++ _create()+ monkeypatch.setattr(supervisor, "_stream_completion", fake_stream_completion)+ monkeypatch.setattr(worker, "execute_tool", fake_tool)+ asyncio.run(supervisor._process(research_db.claim_next(supervisor.worker_id)))+ planned = research_db.get_run("run-1")+ research_db.approve("run-1", planned["planRevision"], planned["planHash"])+ asyncio.run(supervisor._process(research_db.claim_next(supervisor.worker_id)))+ failed = research_db.get_run("run-1")+ assert failed["status"] == "failed"+ assert failed["error"] == "Local model request exhausted its total time budget"+ assert len(searches) == 1++ with _retry_client() as client:+ response = client.post("/runs/run-1/retry", json = {"budgets": {"modelTimeoutSeconds": 0}})+ assert response.status_code == 200, response.text+ asyncio.run(supervisor._process(research_db.claim_next(supervisor.worker_id)))++ completed = research_db.get_run("run-1")+ assert completed["status"] == "completed", completed["error"]+ assert synthesis_attempts == [30, 0]+ assert len(searches) == 1+ assert {source["url"] for source in completed["sources"]} == {+ "https://a.example.com",+ "https://b.example.com",+ }+ assert completed["report"].startswith("# Report")++def test_retry_of_unapproved_plan_requires_approval_again(research_home):_create()plan = research_db.set_plan("run-1", _plan())diff --git a/studio/frontend/src/features/chat/api/research-api.ts b/studio/frontend/src/features/chat/api/research-api.tsindex 05f7fdb82e5..74b07d77f81 100644--- a/studio/frontend/src/features/chat/api/research-api.ts+++ b/studio/frontend/src/features/chat/api/research-api.ts@@ -7,6 +7,7 @@ import type {ResearchEvent,ResearchPlan,ResearchRun,+ ResearchWebsitePolicy,} from "../types/research";type StreamResearchEvent = Omit<ResearchEvent, "data" | "run"> & {@@ -124,7 +125,26 @@ export const approveResearchRun = (planHash: string,) => mutate(id, "approve", { planRevision, planHash });export const cancelResearchRun = (id: string) => mutate(id, "cancel");-export const retryResearchRun = (id: string) => mutate(id, "retry");+export const retryResearchRun = (+ id: string,+ settings?: {+ modelTimeoutSeconds: number;+ websitePolicy: ResearchWebsitePolicy;+ },+) =>+ mutate(+ id,+ "retry",+ settings+ ? {+ budgets: { modelTimeoutSeconds: settings.modelTimeoutSeconds },+ websitePolicy: {+ allowedDomains: [...settings.websitePolicy.allowedDomains],+ blockedDomains: [...settings.websitePolicy.blockedDomains],+ },+ }+ : undefined,+ );export async function updateResearchPlan(id: string,diff --git a/studio/frontend/src/features/chat/components/research-activity-panel.tsx b/studio/frontend/src/features/chat/components/research-activity-panel.tsxindex ec20d219a24..c93221de7b1 100644--- a/studio/frontend/src/features/chat/components/research-activity-panel.tsx+++ b/studio/frontend/src/features/chat/components/research-activity-panel.tsx@@ -62,6 +62,7 @@ import {retryResearchRun,updateResearchPlan,} from "../api/research-api";+import { useChatRuntimeStore } from "../stores/chat-runtime-store";import {type ResearchActivity,ensureResearchRunFollowed,@@ -799,7 +800,11 @@ function ResearchActions({ runId }: { runId: string }): ReactElement | null {const retry = async () => {setPending(true);try {- const retried = await retryResearchRun(run.id);+ const settings = useChatRuntimeStore.getState();+ const retried = await retryResearchRun(run.id, {+ modelTimeoutSeconds: settings.researchModelTimeoutSeconds,+ websitePolicy: settings.researchWebsitePolicy,+ });ingestResearchUpdate(retried);useResearchRunStore.getState().setConnectionError(retried.id, null);ensureResearchRunFollowed(retried.id, retried);diff --git a/studio/frontend/tests/research-retry-settings.test.ts b/studio/frontend/tests/research-retry-settings.test.tsnew file mode 100644index 00000000000..d38aba92f8e--- /dev/null+++ b/studio/frontend/tests/research-retry-settings.test.ts@@ -0,0 +1,62 @@+// SPDX-License-Identifier: AGPL-3.0-only+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++// #11837: Retry sends the current time budget and website access.++import assert from "node:assert/strict";+import { register } from "node:module";+import test from "node:test";++import {+ installLocalStorageFake,+ registerBundlerResolver,+} from "./helpers/kit.ts";++register("./helpers/settings-api-resolver.mjs", import.meta.url);+registerBundlerResolver();+installLocalStorageFake();++const { retryResearchRun } = await import(+ "../src/features/chat/api/research-api.ts"+);++async function captureRetry(+ call: () => Promise<unknown>,+): Promise<{ url: string; init: RequestInit | undefined }> {+ const original = globalThis.fetch;+ let seen: { url: string; init: RequestInit | undefined } | null = null;+ globalThis.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => {+ seen = { url: String(input), init };+ return new Response(JSON.stringify({ id: "run-1", status: "queued" }), {+ status: 200,+ headers: { "content-type": "application/json" },+ });+ }) as typeof globalThis.fetch;+ try {+ await call();+ } finally {+ globalThis.fetch = original;+ }+ assert.ok(seen);+ return seen;+}++test("retry sends the current time budget and website access", async () => {+ const { url, init } = await captureRetry(() =>+ retryResearchRun("run-1", {+ modelTimeoutSeconds: 0,+ websitePolicy: { allowedDomains: [], blockedDomains: ["example.com"] },+ }),+ );+ assert.match(url, /\/api\/chat\/research-runs\/run-1\/retry$/);+ assert.equal(init?.method, "POST");+ assert.deepEqual(JSON.parse(String(init?.body)), {+ budgets: { modelTimeoutSeconds: 0 },+ websitePolicy: { allowedDomains: [], blockedDomains: ["example.com"] },+ });+});++test("retry without settings sends no body", async () => {+ const { init } = await captureRetry(() => retryResearchRun("run-1"));+ assert.equal(init?.body, undefined);+});diff --git a/tests/studio/test_deep_research_frontend_contract.py b/tests/studio/test_deep_research_frontend_contract.pyindex 8048352fc53..0c04f16975c 100644--- a/tests/studio/test_deep_research_frontend_contract.py+++ b/tests/studio/test_deep_research_frontend_contract.py@@ -37,8 +37,9 @@ def test_research_api_is_isolated_and_cursor_based() -> None:# The store owns the stream, so nothing else can stall ingestion by not reading.assert "ensureResearchRunFollowed(run.id, run);" in storeassert "export async function* watchResearchRun" in store- for action in ("cancel", "retry"):- assert f'mutate(id, "{action}")' in api+ assert 'mutate(id, "cancel")' in api+ # Retry carries the current time budget and website access (#11837).+ assert re.search(r'mutate\(\s*id,\s*"retry",', api)assert 'mutate(id, "approve", { planRevision, planHash })' in apiassert "JSON.stringify({ plan, expectedRevision })" in api