Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: Keep unfinished research reports (#10759)
Nilaya97afe96de1f
diff --git a/studio/backend/core/research_runs.py b/studio/backend/core/research_runs.pyindex 30041bd2b8c..80753834ac9 100644--- a/studio/backend/core/research_runs.py+++ b/studio/backend/core/research_runs.py@@ -19,7 +19,7 @@ import uuidfrom contextlib import asynccontextmanagerfrom datetime import datetime, timedelta, timezonefrom email.utils import parsedate_to_datetime-from typing import Any, AsyncIterator, Callable+from typing import Any, AsyncIterator, Callable, NamedTupleimport httpx@@ -252,6 +252,10 @@ class ModelWallClockTimeout(httpx.ReadTimeout):pass+# Every separator, not just "\n": one survivor lets provider text open a block of its own.+_LINE_SEPARATOR = re.compile(r"[\r\n\v\f\x1c-\x1e\x85\u2028\u2029]+")++def _safe_error(exc: BaseException) -> str:if isinstance(exc, ModelFirstOutputTimeout):return "Local model never started producing output"@@ -267,7 +271,7 @@ def _safe_error(exc: BaseException) -> str:# reading it here dropped the Model settings hint from an oversize refusal.friendly = getattr(exc, "friendly", None)text = friendly if isinstance(friendly, str) and friendly else str(exc)- text = text.replace("\n", " ").strip()+ text = _LINE_SEPARATOR.sub(" ", text).strip()return (text or exc.__class__.__name__)[:_MAX_ERROR_CHARS]@@ -1014,6 +1018,40 @@ def _run_moved_on(fresh: dict | None, attempt: int) -> bool:return int(fresh.get("retryCount") or 0) != attempt+def _cited_sources(run: dict) -> list[dict]:+ """Sources trimmed to the budget: past it, the model never saw them to cite."""+ max_sources = int(((run.get("config") or {}).get("budgets") or {}).get("maxSources") or 0)+ return list(run.get("sources") or [])[:max_sources]+++# A WAL commit on a busy disk has held the writer lock for 37s; under the 120-second lease.+_TERMINAL_WRITE_DEADLINE_SECONDS = 60.0+_TERMINAL_WRITE_RETRY_SECONDS = 1.0+++def _as_literal_markdown(text: str) -> str:+ """Provider text for a Markdown surface, as the one inline context nothing reparses."""+ longest = max((len(run) for run in re.findall(r"`+", text)), default = 0)+ fence = "`" * (longest + 1)+ # A span whose content touches a backtick needs padding, which the renderer then strips.+ pad = " " if text.startswith("`") or text.endswith("`") else ""+ return f"{fence}{pad}{text}{pad}{fence}"+++def _report_under_notice(report: str, lead: str, notice: str) -> str:+ """A report under a notice; above it, since a cut report can end in an unterminated fence."""+ return f"> **{lead}** {notice}\n\n{report.lstrip()}"+++class _ReportDraft(NamedTuple):+ """``whole`` is whether the provider said the model stopped of its own accord, which an+ interrupted stream can still have done; ``reason`` is why it stops short, when known."""++ text: str+ whole: bool+ reason: str = ""++def _update_assistant(run: dict,text: str,@@ -1110,6 +1148,8 @@ class ResearchSupervisor:self._task: asyncio.Task | None = Noneself._cancel_events: dict[str, threading.Event] = {}self._lost_leases: set[str] = set()+ # In memory, not read back on failure: every write that would store a draft can itself fail.+ self._report_drafts: dict[str, _ReportDraft] = {}self._last_claim_account: str | None = Nonedef start(self) -> None:@@ -1249,6 +1289,39 @@ class ResearchSupervisor:await self._check_active(run_id)raise LeaseLost()+ def _salvage_report(self, run_id: str, notice: str) -> str:+ """The report a failed run had, under a notice, or "" if synthesis produced none."""+ draft = self._report_drafts.get(account_key(run_id))+ if draft is None or not draft.text:+ return ""+ if draft.whole:+ return _report_under_notice(draft.text, "Report complete.", notice)+ reason = f"{draft.reason}. {notice}" if draft.reason else notice+ return _report_under_notice(draft.text, "Incomplete report.", reason)++ async def _finish_terminal(+ self, run_id: str, status: str, error: str | None, report: str+ ) -> str | None:+ """Commit the terminal status, waiting a bounded while for a writer lock held elsewhere.+ Gives up by raising, not by standing in a lease-expired row that would discard the run's+ report and its real error; the run stays claimable once its lease runs out."""+ deadline = time.monotonic() + _TERMINAL_WRITE_DEADLINE_SECONDS+ while True:+ try:+ return await asyncio.to_thread(+ db.finish,+ run_id,+ self.worker_id,+ status,+ error,+ {"report": report} if report else None,+ )+ except sqlite3.OperationalError as exc:+ if not is_sqlite_busy_error(exc) or time.monotonic() >= deadline:+ raise+ logger.warning("research.terminal_write_busy run_id=%s: %s", run_id, exc)+ await asyncio.sleep(_TERMINAL_WRITE_RETRY_SECONDS)+async def _finish_after_lease_loss(self, run_id: str) -> str | None:while True:try:@@ -1537,6 +1610,7 @@ class ResearchSupervisor:max_tokens: int | None = None,enable_thinking: bool | None = None,preview_labels: bool = False,+ on_partial: Callable[[str, str, str | None], None] | None = None,) -> tuple[str, str, str | None, dict[str, int] | None]:call_id = uuid.uuid4().hexexpires = (@@ -1941,17 +2015,35 @@ class ResearchSupervisor:)await flush_progress()return report, reasoning, finish_reason, usage- except (ModelFirstOutputTimeout, ModelOutputIdleTimeout, ModelWallClockTimeout):+ # Not errors: the flush below would fail without writing.+ except (RunCancelled, LeaseLost):raise- except httpx.ReadTimeout as exc:+ except Exception as exc:+ # Before the flush, which only displays it; with the finish reason, which text cannot carry.+ if on_partial is not None:+ on_partial(report, reasoning, finish_reason)+ if report_progress:+ try:+ # Best effort: a progress update must not replace the error that stopped the run.+ await flush_progress()+ except Exception:+ logger.warning(+ "research.error_flush_failed run_id=%s", run["id"], exc_info = True+ )+ if isinstance(+ exc, (ModelFirstOutputTimeout, ModelOutputIdleTimeout, ModelWallClockTimeout)+ ):+ raise# Transport backstop: HTTPX raises this with no message, so name the stall instead.- if semantic_output_at is None:- raise ModelFirstOutputTimeout("Local model never produced output") from exc- raise ModelOutputIdleTimeout("Local model stopped producing output") from exc- except (TimeoutError, asyncio.TimeoutError) as exc:- raise ModelWallClockTimeout(- "Local model request exceeded its wall-clock timeout"- ) from exc+ if isinstance(exc, httpx.ReadTimeout):+ if semantic_output_at is None:+ raise ModelFirstOutputTimeout("Local model never produced output") from exc+ raise ModelOutputIdleTimeout("Local model stopped producing output") from exc+ if isinstance(exc, (TimeoutError, asyncio.TimeoutError)):+ raise ModelWallClockTimeout(+ "Local model request exceeded its wall-clock timeout"+ ) from exc+ raisefinally:# revoked before the phase event, so a cancel there cannot leak a live key.try:@@ -2055,12 +2147,9 @@ class ResearchSupervisor:except Exception as exc:error = _safe_error(exc)logger.warning("research.run_failed run_id=%s error=%s", run["id"], error)- try:- actual_status = await asyncio.to_thread(- db.finish, run["id"], self.worker_id, "failed", error- )- except sqlite3.OperationalError:- actual_status = await self._finish_after_lease_loss(run["id"])+ reported = f"Research failed: {_as_literal_markdown(error)}"+ partial_report = self._salvage_report(run["id"], reported)+ actual_status = await self._finish_terminal(run["id"], "failed", error, partial_report)if actual_status is None:actual_status = await self._finish_after_lease_loss(run["id"])fresh = await asyncio.to_thread(db.get_run, run["id"])@@ -2069,8 +2158,13 @@ class ResearchSupervisor:_update_assistant, fresh, "Research cancelled.", "cancelled")elif actual_status == "failed" and not _run_moved_on(fresh, attempt):+ saved = fresh.get("report")await asyncio.to_thread(- _update_assistant, fresh, f"Research failed: {error}", "failed"+ _update_assistant,+ fresh,+ saved or reported,+ "failed",+ _cited_sources(fresh) if saved else None,)finally:heartbeat.cancel()@@ -2080,6 +2174,7 @@ class ResearchSupervisor:passself._cancel_events.pop(account_key(run["id"]), None)self._lost_leases.discard(account_key(run["id"]))+ self._report_drafts.pop(account_key(run["id"]), None)async def _heartbeat(self, run_id: str) -> None:delay = 30.0@@ -2810,6 +2905,40 @@ class ResearchSupervisor:),},]++ def _delivered(draft: str) -> str:+ """What the reader would get: a model out of budget pads with citations validation drops."""+ return _validate_report(draft, sources, document_sources)++ def _keep_draft(+ report: str,+ finish_reason: str | None,+ reason: str = "",+ ) -> None:+ """Keep the best report reached, so a later failure can still publish it. Ranked as+ the success path ranks its drafts: finished over cut off, then longer, then newest."""+ if not reason and finish_reason == "length":+ # Without the token count, which only a call reaching its own end reports.+ reason = _synthesis_length_limit_error(+ None, requested_max_tokens = 0, inference = _run_inference_request(run)+ ).rstrip(".")+ text = _delivered(report).strip()+ draft = _ReportDraft(+ text, finish_reason in _NATURAL_FINISH_REASONS and bool(text), reason+ )+ kept = self._report_drafts.get(account_key(run["id"]))+ if kept is None or (draft.whole, len(draft.text)) >= (kept.whole, len(kept.text)):+ self._report_drafts[account_key(run["id"])] = draft++ def _keep_partial(content: str, reasoning: str, finish_reason: str | None) -> None:+ """Keep a cut-off stream's text past the final-report boundary; above it is analysis."""+ draft = (+ _report_after_boundary(content, _REPORT_BOUNDARY_MARKER)+ or _report_after_boundary(reasoning, _REPORT_BOUNDARY_MARKER)+ or ""+ )+ _keep_draft(draft, finish_reason)+synthesis_max_tokens = await asyncio.to_thread(_synthesis_max_tokens,run["config"].get("inferenceRequest") or {},@@ -2826,6 +2955,7 @@ class ResearchSupervisor:synthesis_messages,phase = "synthesis",max_tokens = synthesis_max_tokens,+ on_partial = _keep_partial,)except (RunCancelled, LeaseLost, httpx.ReadTimeout):raise@@ -2852,20 +2982,13 @@ class ResearchSupervisor:synthesis_messages,phase = "synthesis",max_tokens = synthesis_max_tokens,+ on_partial = _keep_partial,)- await self._check_active(run["id"])report = _select_synthesis_report(report, synthesis_reasoning)+ # Before the checks below, which can fail once the report already exists.+ _keep_draft(report, synthesis_finish_reason)+ await self._check_active(run["id"])truncation_notice = ""-- def _delivered(draft: str) -> str:- """What the reader would actually get from this draft.-- The validators below drop a model-authored source list and every citation the- catalogs do not back, and a model that ran out of budget is exactly the one- liable to pad with both, so raw length is not what the drafts should be judged- on. Used only to compare them; whichever wins is stored as the model wrote it."""- return _validate_report(draft, sources, document_sources)-if _synthesis_needs_recovery(report, synthesis_finish_reason):recovery_reason = ("exhausted its output budget"@@ -2902,6 +3025,7 @@ class ResearchSupervisor:phase = "synthesis_recovery",max_tokens = synthesis_max_tokens,enable_thinking = False,+ on_partial = _keep_partial,)except (RunCancelled, LeaseLost):raise@@ -2943,6 +3067,7 @@ class ResearchSupervisor:_run_inference_request(run),synthesis_messages,)+ _keep_draft(report, synthesis_finish_reason)await self._check_active(run["id"])if report and synthesis_finish_reason == "length":truncation_notice = _synthesis_length_limit_error(@@ -2961,8 +3086,10 @@ class ResearchSupervisor:# be -- inside a code fence, a list, a quote -- and anything appended under an# unterminated container is swallowed by it, whereas the first line of a document is# inside nothing. The reader also learns the report is cut short before reading it.+ # Recorded before the notice goes on: a delivery that fails writes its own.+ _keep_draft(report, synthesis_finish_reason, truncation_notice)if truncation_notice:- report = f"> **Incomplete report.** {truncation_notice}.\n\n{report.lstrip()}"+ report = _report_under_notice(report, "Incomplete report.", f"{truncation_notice}.")reasoning = await asyncio.to_thread(db.get_reasoning_text, run["id"])if synthesis_reasoning and synthesis_reasoning not in reasoning:reasoning += synthesis_reasoning@@ -2980,9 +3107,7 @@ class ResearchSupervisor:reasoning,self.worker_id,)- actual_status = await asyncio.to_thread(- db.finish, run["id"], self.worker_id, "completed", None, {"report": report}- )+ actual_status = await self._finish_terminal(run["id"], "completed", None, report)if actual_status is None:raise LeaseLost()run = await asyncio.to_thread(db.get_run, run["id"])diff --git a/studio/backend/storage/research_runs_db.py b/studio/backend/storage/research_runs_db.pyindex eca3f27bd7e..dcc889154da 100644--- a/studio/backend/storage/research_runs_db.py+++ b/studio/backend/storage/research_runs_db.py@@ -998,8 +998,9 @@ def finish(else status)actual_error = None if actual_status == "cancelled" else error+ # A cancelled run stores no report: a stop is the user asking for nothing back.report_text = None- if actual_status == "completed" and event_payload:+ if actual_status in {"completed", "failed"} and event_payload:candidate = event_payload.get("report")if isinstance(candidate, str):report_text = candidatediff --git a/studio/backend/tests/test_research_interrupted_report.py b/studio/backend/tests/test_research_interrupted_report.pynew file mode 100644index 00000000000..214f7439e7c--- /dev/null+++ b/studio/backend/tests/test_research_interrupted_report.py@@ -0,0 +1,141 @@+# SPDX-License-Identifier: AGPL-3.0-only++import asyncio+import json+from types import SimpleNamespace++import httpx+import pytest++from core import research_runs as worker+from storage import research_runs_db as research_db+from storage import studio_db+from .test_research_synthesis_recovery import _claimed_run, research_home+++REPORT = "## Findings\n\n" + "The evidence explains the result. " * 80+RAW = f"Private preamble.\n{worker._REPORT_BOUNDARY_MARKER}\n{REPORT}"+++def _interrupt(monkeypatch, text, error, channel, final_reason):+ class Stream(httpx.AsyncByteStream):+ async def __aiter__(self):+ for offset in range(0, len(text), 300):+ chunk = {"choices": [{"delta": {channel: text[offset : offset + 300]}}]}+ yield f"data: {json.dumps(chunk)}\n\n".encode()+ if final_reason is not None:+ done = {"choices": [{"delta": {}, "finish_reason": final_reason}]}+ yield f"data: {json.dumps(done)}\n\n".encode()+ raise error++ def respond(request):+ # Only the report call is interrupted; the steps before it have to land, and their+ # unparseable bodies make each one take its single seed action.+ if b"untrusted_synthesis_audit_json" in request.content:+ return httpx.Response(200, stream = Stream())+ return httpx.Response(200, content = b'data: {"choices":[{"delta":{"content":"x"}}]}\n\n')++ original = httpx.AsyncClient+ monkeypatch.setattr(+ worker.httpx,+ "AsyncClient",+ lambda **kwargs: original(transport = httpx.MockTransport(respond), **kwargs),+ )+ monkeypatch.setattr(+ worker.auth_storage, "create_api_key", lambda **kwargs: ("fixture", {"id": 1})+ )+ monkeypatch.setattr(worker.auth_storage, "revoke_internal_api_key", lambda key_id: None)++ supervisor = worker.ResearchSupervisor(SimpleNamespace(state = SimpleNamespace(server_port = 1)))+ run = _claimed_run(supervisor)+ asyncio.run(supervisor._process(run))+ return research_db.get_run(run["id"])+++ ("error", "channel", "final_reason", "lead"),+ [+ (httpx.ReadError("connection reset"), "content", None, "Incomplete report."),+ (worker.ModelWallClockTimeout("timed out"), "content", None, "Incomplete report."),+ # Some local models write the report only as reasoning, which synthesis accepts.+ (httpx.ReadError("connection reset"), "reasoning_content", None, "Incomplete report."),+ # Cut after the model said it was done: the report is whole, only delivery was not.+ (httpx.ReadError("connection reset"), "content", "stop", "Report complete."),+ ],+)+def test_interrupted_stream_retains_last_bytes_in_failed_report_and_chat(+ research_home, monkeypatch, error, channel, final_reason, lead+):+ run = _interrupt(monkeypatch, RAW, error, channel, final_reason)+ assert run["status"] == "failed"+ assert run["error"] == worker._safe_error(error)+ notice = f"> **{lead}** Research failed: `{run['error']}`"+ assert run["report"] == f"{notice}\n\n{REPORT.strip()}"+ message = studio_db.get_chat_message(run["threadId"], run["assistantMessageId"])+ text = "\n".join(part["text"] for part in message["content"] if part["type"] == "text")+ assert text == run["report"]+ assert "Private preamble" not in text+ terminal = research_db.list_events(run["id"], 0)[-1]+ assert terminal["type"] == "run.failed"+ assert terminal["data"]["report"] == run["report"]+++ "text", ["", REPORT, f"```\n{RAW}\n```", f"{worker._REPORT_BOUNDARY_MARKER}\n"]+)+def test_unidentified_report_is_not_exposed(research_home, monkeypatch, text):+ run = _interrupt(monkeypatch, text, httpx.ReadError("connection reset"), "content", None)+ assert run["status"] == "failed"+ assert run["report"] is None+++def test_a_returned_report_is_kept_before_the_checks_that_could_lose_it(research_home, monkeypatch):+ supervisor = worker.ResearchSupervisor(SimpleNamespace(state = SimpleNamespace(server_port = 1)))+ claimed = _claimed_run(supervisor)++ async def gone(run_id):+ raise RuntimeError("database is gone")++ async def streamed(run, messages, **kwargs):+ if kwargs.get("phase") != "synthesis":+ return "not json", "", "stop", None+ monkeypatch.setattr(supervisor, "_check_active", gone)+ return RAW, "", "stop", None++ monkeypatch.setattr(supervisor, "_stream_completion", streamed)+ asyncio.run(supervisor._process(claimed))++ # Work remains between a returned report and the row storing it; none of it can lose the report.+ assert research_db.get_run(claimed["id"])["report"] == (+ f"> **Report complete.** Research failed: `database is gone`\n\n{REPORT.strip()}"+ )+++def test_interrupted_report_validates_citations(research_home, monkeypatch):+ raw = RAW + "\n[Backed](https://backed.test) [invented](https://invented.test)"+ raw += "\n\n## Sources\n\nUntrusted source list"+ monkeypatch.setattr(+ worker,+ "execute_tool",+ lambda *args, **kwargs: "Title: Backed\nURL: https://backed.test\nSnippet: evidence",+ )+ run = _interrupt(monkeypatch, raw, httpx.ReadError("connection reset"), "content", None)+ assert "https://backed.test" in run["report"]+ assert "invented.test" not in run["report"]+ assert "Untrusted source list" not in run["report"]++[email protected]("error", [worker.RunCancelled(), worker.LeaseLost()])+def test_control_flow_interruptions_do_not_publish_partial_report(+ research_home, monkeypatch, error+):+ run = _interrupt(monkeypatch, RAW, error, "content", None)+ assert run["report"] is None+++def test_failed_finish_does_not_promote_raw_progress(research_home):+ supervisor = worker.ResearchSupervisor(SimpleNamespace(state = SimpleNamespace(server_port = 1)))+ run = _claimed_run(supervisor)+ research_db.set_report_progress(run["id"], RAW, worker_id = supervisor.worker_id)+ assert research_db.finish(run["id"], supervisor.worker_id, "failed", "failed") == "failed"+ assert research_db.get_run(run["id"])["report"] is Nonediff --git a/studio/backend/tests/test_research_runs_storage.py b/studio/backend/tests/test_research_runs_storage.pyindex fc288596e2e..200fdaa461e 100644--- a/studio/backend/tests/test_research_runs_storage.py+++ b/studio/backend/tests/test_research_runs_storage.py@@ -2873,7 +2873,8 @@ def test_terminal_sse_event_contains_report_and_complete_snapshot(research_home)("cancelled", "expected_status", "text"),[(True, "cancelled", "Research cancelled."),- (False, "failed", "Research failed: mocked model failure"),+ # Provider text reaches a Markdown surface here too, so it is quoted literally.+ (False, "failed", "Research failed: `mocked model failure`"),],)def test_worker_terminal_paths_create_one_fallback_without_frontend_message(diff --git a/studio/frontend/src/features/chat/components/research-message.tsx b/studio/frontend/src/features/chat/components/research-message.tsxindex 9e1492fbd5e..5c580917084 100644--- a/studio/frontend/src/features/chat/components/research-message.tsx+++ b/studio/frontend/src/features/chat/components/research-message.tsx@@ -79,7 +79,9 @@ export function ResearchMessage(): ReactElement | null {);}- if (run.status === "completed" && run.report) {+ // A failed run's report opens with its own notice, so nothing here repeats it.+ if ((run.status === "completed" || run.status === "failed") && run.report) {+ const failed = run.status === "failed";const sources: SourceData[] = run.sources.map((source) => ({id: String(source.id ?? source.url),url: source.url,@@ -109,9 +111,17 @@ export function ResearchMessage(): ReactElement | null {className="mb-3 flex items-center gap-2 rounded-full text-sm text-muted-foreground transition-colors hover:text-foreground focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring"><span className="flex size-5 items-center justify-center rounded-full bg-primary/10 text-primary">- <Check className="size-3" />+ {failed ? (+ <TriangleAlert className="size-3" />+ ) : (+ <Check className="size-3" />+ )}+ </span>+ <span>+ {failed+ ? `Deep research failed · ${sourceCount} sources`+ : `Deep research completed · ${sourceCount} sources`}</span>- <span>Deep research completed · {sourceCount} sources</span><span className="text-primary">View activity</span></button><MarkdownPreview