Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: Keep unfinished research reports (#10759)

Nilaya97afe96de1f
diff --git a/studio/backend/core/research_runs.py b/studio/backend/core/research_runs.py
index 30041bd2b8c..80753834ac9 100644
--- a/studio/backend/core/research_runs.py
+++ b/studio/backend/core/research_runs.py
@@ -19,7 +19,7 @@ import uuid
from contextlib import asynccontextmanager
from datetime import datetime, timedelta, timezone
from email.utils import parsedate_to_datetime
-from typing import Any, AsyncIterator, Callable
+from typing import Any, AsyncIterator, Callable, NamedTuple
import httpx
@@ -252,6 +252,10 @@ class ModelWallClockTimeout(httpx.ReadTimeout):
pass
+# Every separator, not just "\n": one survivor lets provider text open a block of its own.
+_LINE_SEPARATOR = re.compile(r"[\r\n\v\f\x1c-\x1e\x85\u2028\u2029]+")
+
+
def _safe_error(exc: BaseException) -> str:
if isinstance(exc, ModelFirstOutputTimeout):
return "Local model never started producing output"
@@ -267,7 +271,7 @@ def _safe_error(exc: BaseException) -> str:
# reading it here dropped the Model settings hint from an oversize refusal.
friendly = getattr(exc, "friendly", None)
text = friendly if isinstance(friendly, str) and friendly else str(exc)
- text = text.replace("\n", " ").strip()
+ text = _LINE_SEPARATOR.sub(" ", text).strip()
return (text or exc.__class__.__name__)[:_MAX_ERROR_CHARS]
@@ -1014,6 +1018,40 @@ def _run_moved_on(fresh: dict | None, attempt: int) -> bool:
return int(fresh.get("retryCount") or 0) != attempt
+def _cited_sources(run: dict) -> list[dict]:
+ """Sources trimmed to the budget: past it, the model never saw them to cite."""
+ max_sources = int(((run.get("config") or {}).get("budgets") or {}).get("maxSources") or 0)
+ return list(run.get("sources") or [])[:max_sources]
+
+
+# A WAL commit on a busy disk has held the writer lock for 37s; under the 120-second lease.
+_TERMINAL_WRITE_DEADLINE_SECONDS = 60.0
+_TERMINAL_WRITE_RETRY_SECONDS = 1.0
+
+
+def _as_literal_markdown(text: str) -> str:
+ """Provider text for a Markdown surface, as the one inline context nothing reparses."""
+ longest = max((len(run) for run in re.findall(r"`+", text)), default = 0)
+ fence = "`" * (longest + 1)
+ # A span whose content touches a backtick needs padding, which the renderer then strips.
+ pad = " " if text.startswith("`") or text.endswith("`") else ""
+ return f"{fence}{pad}{text}{pad}{fence}"
+
+
+def _report_under_notice(report: str, lead: str, notice: str) -> str:
+ """A report under a notice; above it, since a cut report can end in an unterminated fence."""
+ return f"> **{lead}** {notice}\n\n{report.lstrip()}"
+
+
+class _ReportDraft(NamedTuple):
+ """``whole`` is whether the provider said the model stopped of its own accord, which an
+ interrupted stream can still have done; ``reason`` is why it stops short, when known."""
+
+ text: str
+ whole: bool
+ reason: str = ""
+
+
def _update_assistant(
run: dict,
text: str,
@@ -1110,6 +1148,8 @@ class ResearchSupervisor:
self._task: asyncio.Task | None = None
self._cancel_events: dict[str, threading.Event] = {}
self._lost_leases: set[str] = set()
+ # In memory, not read back on failure: every write that would store a draft can itself fail.
+ self._report_drafts: dict[str, _ReportDraft] = {}
self._last_claim_account: str | None = None
def start(self) -> None:
@@ -1249,6 +1289,39 @@ class ResearchSupervisor:
await self._check_active(run_id)
raise LeaseLost()
+ def _salvage_report(self, run_id: str, notice: str) -> str:
+ """The report a failed run had, under a notice, or "" if synthesis produced none."""
+ draft = self._report_drafts.get(account_key(run_id))
+ if draft is None or not draft.text:
+ return ""
+ if draft.whole:
+ return _report_under_notice(draft.text, "Report complete.", notice)
+ reason = f"{draft.reason}. {notice}" if draft.reason else notice
+ return _report_under_notice(draft.text, "Incomplete report.", reason)
+
+ async def _finish_terminal(
+ self, run_id: str, status: str, error: str | None, report: str
+ ) -> str | None:
+ """Commit the terminal status, waiting a bounded while for a writer lock held elsewhere.
+ Gives up by raising, not by standing in a lease-expired row that would discard the run's
+ report and its real error; the run stays claimable once its lease runs out."""
+ deadline = time.monotonic() + _TERMINAL_WRITE_DEADLINE_SECONDS
+ while True:
+ try:
+ return await asyncio.to_thread(
+ db.finish,
+ run_id,
+ self.worker_id,
+ status,
+ error,
+ {"report": report} if report else None,
+ )
+ except sqlite3.OperationalError as exc:
+ if not is_sqlite_busy_error(exc) or time.monotonic() >= deadline:
+ raise
+ logger.warning("research.terminal_write_busy run_id=%s: %s", run_id, exc)
+ await asyncio.sleep(_TERMINAL_WRITE_RETRY_SECONDS)
+
async def _finish_after_lease_loss(self, run_id: str) -> str | None:
while True:
try:
@@ -1537,6 +1610,7 @@ class ResearchSupervisor:
max_tokens: int | None = None,
enable_thinking: bool | None = None,
preview_labels: bool = False,
+ on_partial: Callable[[str, str, str | None], None] | None = None,
) -> tuple[str, str, str | None, dict[str, int] | None]:
call_id = uuid.uuid4().hex
expires = (
@@ -1941,17 +2015,35 @@ class ResearchSupervisor:
)
await flush_progress()
return report, reasoning, finish_reason, usage
- except (ModelFirstOutputTimeout, ModelOutputIdleTimeout, ModelWallClockTimeout):
+ # Not errors: the flush below would fail without writing.
+ except (RunCancelled, LeaseLost):
raise
- except httpx.ReadTimeout as exc:
+ except Exception as exc:
+ # Before the flush, which only displays it; with the finish reason, which text cannot carry.
+ if on_partial is not None:
+ on_partial(report, reasoning, finish_reason)
+ if report_progress:
+ try:
+ # Best effort: a progress update must not replace the error that stopped the run.
+ await flush_progress()
+ except Exception:
+ logger.warning(
+ "research.error_flush_failed run_id=%s", run["id"], exc_info = True
+ )
+ if isinstance(
+ exc, (ModelFirstOutputTimeout, ModelOutputIdleTimeout, ModelWallClockTimeout)
+ ):
+ raise
# Transport backstop: HTTPX raises this with no message, so name the stall instead.
- if semantic_output_at is None:
- raise ModelFirstOutputTimeout("Local model never produced output") from exc
- raise ModelOutputIdleTimeout("Local model stopped producing output") from exc
- except (TimeoutError, asyncio.TimeoutError) as exc:
- raise ModelWallClockTimeout(
- "Local model request exceeded its wall-clock timeout"
- ) from exc
+ if isinstance(exc, httpx.ReadTimeout):
+ if semantic_output_at is None:
+ raise ModelFirstOutputTimeout("Local model never produced output") from exc
+ raise ModelOutputIdleTimeout("Local model stopped producing output") from exc
+ if isinstance(exc, (TimeoutError, asyncio.TimeoutError)):
+ raise ModelWallClockTimeout(
+ "Local model request exceeded its wall-clock timeout"
+ ) from exc
+ raise
finally:
# revoked before the phase event, so a cancel there cannot leak a live key.
try:
@@ -2055,12 +2147,9 @@ class ResearchSupervisor:
except Exception as exc:
error = _safe_error(exc)
logger.warning("research.run_failed run_id=%s error=%s", run["id"], error)
- try:
- actual_status = await asyncio.to_thread(
- db.finish, run["id"], self.worker_id, "failed", error
- )
- except sqlite3.OperationalError:
- actual_status = await self._finish_after_lease_loss(run["id"])
+ reported = f"Research failed: {_as_literal_markdown(error)}"
+ partial_report = self._salvage_report(run["id"], reported)
+ actual_status = await self._finish_terminal(run["id"], "failed", error, partial_report)
if actual_status is None:
actual_status = await self._finish_after_lease_loss(run["id"])
fresh = await asyncio.to_thread(db.get_run, run["id"])
@@ -2069,8 +2158,13 @@ class ResearchSupervisor:
_update_assistant, fresh, "Research cancelled.", "cancelled"
)
elif actual_status == "failed" and not _run_moved_on(fresh, attempt):
+ saved = fresh.get("report")
await asyncio.to_thread(
- _update_assistant, fresh, f"Research failed: {error}", "failed"
+ _update_assistant,
+ fresh,
+ saved or reported,
+ "failed",
+ _cited_sources(fresh) if saved else None,
)
finally:
heartbeat.cancel()
@@ -2080,6 +2174,7 @@ class ResearchSupervisor:
pass
self._cancel_events.pop(account_key(run["id"]), None)
self._lost_leases.discard(account_key(run["id"]))
+ self._report_drafts.pop(account_key(run["id"]), None)
async def _heartbeat(self, run_id: str) -> None:
delay = 30.0
@@ -2810,6 +2905,40 @@ class ResearchSupervisor:
),
},
]
+
+ def _delivered(draft: str) -> str:
+ """What the reader would get: a model out of budget pads with citations validation drops."""
+ return _validate_report(draft, sources, document_sources)
+
+ def _keep_draft(
+ report: str,
+ finish_reason: str | None,
+ reason: str = "",
+ ) -> None:
+ """Keep the best report reached, so a later failure can still publish it. Ranked as
+ the success path ranks its drafts: finished over cut off, then longer, then newest."""
+ if not reason and finish_reason == "length":
+ # Without the token count, which only a call reaching its own end reports.
+ reason = _synthesis_length_limit_error(
+ None, requested_max_tokens = 0, inference = _run_inference_request(run)
+ ).rstrip(".")
+ text = _delivered(report).strip()
+ draft = _ReportDraft(
+ text, finish_reason in _NATURAL_FINISH_REASONS and bool(text), reason
+ )
+ kept = self._report_drafts.get(account_key(run["id"]))
+ if kept is None or (draft.whole, len(draft.text)) >= (kept.whole, len(kept.text)):
+ self._report_drafts[account_key(run["id"])] = draft
+
+ def _keep_partial(content: str, reasoning: str, finish_reason: str | None) -> None:
+ """Keep a cut-off stream's text past the final-report boundary; above it is analysis."""
+ draft = (
+ _report_after_boundary(content, _REPORT_BOUNDARY_MARKER)
+ or _report_after_boundary(reasoning, _REPORT_BOUNDARY_MARKER)
+ or ""
+ )
+ _keep_draft(draft, finish_reason)
+
synthesis_max_tokens = await asyncio.to_thread(
_synthesis_max_tokens,
run["config"].get("inferenceRequest") or {},
@@ -2826,6 +2955,7 @@ class ResearchSupervisor:
synthesis_messages,
phase = "synthesis",
max_tokens = synthesis_max_tokens,
+ on_partial = _keep_partial,
)
except (RunCancelled, LeaseLost, httpx.ReadTimeout):
raise
@@ -2852,20 +2982,13 @@ class ResearchSupervisor:
synthesis_messages,
phase = "synthesis",
max_tokens = synthesis_max_tokens,
+ on_partial = _keep_partial,
)
- await self._check_active(run["id"])
report = _select_synthesis_report(report, synthesis_reasoning)
+ # Before the checks below, which can fail once the report already exists.
+ _keep_draft(report, synthesis_finish_reason)
+ await self._check_active(run["id"])
truncation_notice = ""
-
- def _delivered(draft: str) -> str:
- """What the reader would actually get from this draft.
-
- The validators below drop a model-authored source list and every citation the
- catalogs do not back, and a model that ran out of budget is exactly the one
- liable to pad with both, so raw length is not what the drafts should be judged
- on. Used only to compare them; whichever wins is stored as the model wrote it."""
- return _validate_report(draft, sources, document_sources)
-
if _synthesis_needs_recovery(report, synthesis_finish_reason):
recovery_reason = (
"exhausted its output budget"
@@ -2902,6 +3025,7 @@ class ResearchSupervisor:
phase = "synthesis_recovery",
max_tokens = synthesis_max_tokens,
enable_thinking = False,
+ on_partial = _keep_partial,
)
except (RunCancelled, LeaseLost):
raise
@@ -2943,6 +3067,7 @@ class ResearchSupervisor:
_run_inference_request(run),
synthesis_messages,
)
+ _keep_draft(report, synthesis_finish_reason)
await self._check_active(run["id"])
if report and synthesis_finish_reason == "length":
truncation_notice = _synthesis_length_limit_error(
@@ -2961,8 +3086,10 @@ class ResearchSupervisor:
# be -- inside a code fence, a list, a quote -- and anything appended under an
# unterminated container is swallowed by it, whereas the first line of a document is
# inside nothing. The reader also learns the report is cut short before reading it.
+ # Recorded before the notice goes on: a delivery that fails writes its own.
+ _keep_draft(report, synthesis_finish_reason, truncation_notice)
if truncation_notice:
- report = f"> **Incomplete report.** {truncation_notice}.\n\n{report.lstrip()}"
+ report = _report_under_notice(report, "Incomplete report.", f"{truncation_notice}.")
reasoning = await asyncio.to_thread(db.get_reasoning_text, run["id"])
if synthesis_reasoning and synthesis_reasoning not in reasoning:
reasoning += synthesis_reasoning
@@ -2980,9 +3107,7 @@ class ResearchSupervisor:
reasoning,
self.worker_id,
)
- actual_status = await asyncio.to_thread(
- db.finish, run["id"], self.worker_id, "completed", None, {"report": report}
- )
+ actual_status = await self._finish_terminal(run["id"], "completed", None, report)
if actual_status is None:
raise LeaseLost()
run = await asyncio.to_thread(db.get_run, run["id"])
diff --git a/studio/backend/storage/research_runs_db.py b/studio/backend/storage/research_runs_db.py
index eca3f27bd7e..dcc889154da 100644
--- a/studio/backend/storage/research_runs_db.py
+++ b/studio/backend/storage/research_runs_db.py
@@ -998,8 +998,9 @@ def finish(
else status
)
actual_error = None if actual_status == "cancelled" else error
+ # A cancelled run stores no report: a stop is the user asking for nothing back.
report_text = None
- if actual_status == "completed" and event_payload:
+ if actual_status in {"completed", "failed"} and event_payload:
candidate = event_payload.get("report")
if isinstance(candidate, str):
report_text = candidate
diff --git a/studio/backend/tests/test_research_interrupted_report.py b/studio/backend/tests/test_research_interrupted_report.py
new file mode 100644
index 00000000000..214f7439e7c
--- /dev/null
+++ b/studio/backend/tests/test_research_interrupted_report.py
@@ -0,0 +1,141 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+
+import asyncio
+import json
+from types import SimpleNamespace
+
+import httpx
+import pytest
+
+from core import research_runs as worker
+from storage import research_runs_db as research_db
+from storage import studio_db
+from .test_research_synthesis_recovery import _claimed_run, research_home
+
+
+REPORT = "## Findings\n\n" + "The evidence explains the result. " * 80
+RAW = f"Private preamble.\n{worker._REPORT_BOUNDARY_MARKER}\n{REPORT}"
+
+
+def _interrupt(monkeypatch, text, error, channel, final_reason):
+ class Stream(httpx.AsyncByteStream):
+ async def __aiter__(self):
+ for offset in range(0, len(text), 300):
+ chunk = {"choices": [{"delta": {channel: text[offset : offset + 300]}}]}
+ yield f"data: {json.dumps(chunk)}\n\n".encode()
+ if final_reason is not None:
+ done = {"choices": [{"delta": {}, "finish_reason": final_reason}]}
+ yield f"data: {json.dumps(done)}\n\n".encode()
+ raise error
+
+ def respond(request):
+ # Only the report call is interrupted; the steps before it have to land, and their
+ # unparseable bodies make each one take its single seed action.
+ if b"untrusted_synthesis_audit_json" in request.content:
+ return httpx.Response(200, stream = Stream())
+ return httpx.Response(200, content = b'data: {"choices":[{"delta":{"content":"x"}}]}\n\n')
+
+ original = httpx.AsyncClient
+ monkeypatch.setattr(
+ worker.httpx,
+ "AsyncClient",
+ lambda **kwargs: original(transport = httpx.MockTransport(respond), **kwargs),
+ )
+ monkeypatch.setattr(
+ worker.auth_storage, "create_api_key", lambda **kwargs: ("fixture", {"id": 1})
+ )
+ monkeypatch.setattr(worker.auth_storage, "revoke_internal_api_key", lambda key_id: None)
+
+ supervisor = worker.ResearchSupervisor(SimpleNamespace(state = SimpleNamespace(server_port = 1)))
+ run = _claimed_run(supervisor)
+ asyncio.run(supervisor._process(run))
+ return research_db.get_run(run["id"])
+
+
+ ("error", "channel", "final_reason", "lead"),
+ [
+ (httpx.ReadError("connection reset"), "content", None, "Incomplete report."),
+ (worker.ModelWallClockTimeout("timed out"), "content", None, "Incomplete report."),
+ # Some local models write the report only as reasoning, which synthesis accepts.
+ (httpx.ReadError("connection reset"), "reasoning_content", None, "Incomplete report."),
+ # Cut after the model said it was done: the report is whole, only delivery was not.
+ (httpx.ReadError("connection reset"), "content", "stop", "Report complete."),
+ ],
+)
+def test_interrupted_stream_retains_last_bytes_in_failed_report_and_chat(
+ research_home, monkeypatch, error, channel, final_reason, lead
+):
+ run = _interrupt(monkeypatch, RAW, error, channel, final_reason)
+ assert run["status"] == "failed"
+ assert run["error"] == worker._safe_error(error)
+ notice = f"> **{lead}** Research failed: `{run['error']}`"
+ assert run["report"] == f"{notice}\n\n{REPORT.strip()}"
+ message = studio_db.get_chat_message(run["threadId"], run["assistantMessageId"])
+ text = "\n".join(part["text"] for part in message["content"] if part["type"] == "text")
+ assert text == run["report"]
+ assert "Private preamble" not in text
+ terminal = research_db.list_events(run["id"], 0)[-1]
+ assert terminal["type"] == "run.failed"
+ assert terminal["data"]["report"] == run["report"]
+
+
+ "text", ["", REPORT, f"```\n{RAW}\n```", f"{worker._REPORT_BOUNDARY_MARKER}\n"]
+)
+def test_unidentified_report_is_not_exposed(research_home, monkeypatch, text):
+ run = _interrupt(monkeypatch, text, httpx.ReadError("connection reset"), "content", None)
+ assert run["status"] == "failed"
+ assert run["report"] is None
+
+
+def test_a_returned_report_is_kept_before_the_checks_that_could_lose_it(research_home, monkeypatch):
+ supervisor = worker.ResearchSupervisor(SimpleNamespace(state = SimpleNamespace(server_port = 1)))
+ claimed = _claimed_run(supervisor)
+
+ async def gone(run_id):
+ raise RuntimeError("database is gone")
+
+ async def streamed(run, messages, **kwargs):
+ if kwargs.get("phase") != "synthesis":
+ return "not json", "", "stop", None
+ monkeypatch.setattr(supervisor, "_check_active", gone)
+ return RAW, "", "stop", None
+
+ monkeypatch.setattr(supervisor, "_stream_completion", streamed)
+ asyncio.run(supervisor._process(claimed))
+
+ # Work remains between a returned report and the row storing it; none of it can lose the report.
+ assert research_db.get_run(claimed["id"])["report"] == (
+ f"> **Report complete.** Research failed: `database is gone`\n\n{REPORT.strip()}"
+ )
+
+
+def test_interrupted_report_validates_citations(research_home, monkeypatch):
+ raw = RAW + "\n[Backed](https://backed.test) [invented](https://invented.test)"
+ raw += "\n\n## Sources\n\nUntrusted source list"
+ monkeypatch.setattr(
+ worker,
+ "execute_tool",
+ lambda *args, **kwargs: "Title: Backed\nURL: https://backed.test\nSnippet: evidence",
+ )
+ run = _interrupt(monkeypatch, raw, httpx.ReadError("connection reset"), "content", None)
+ assert "https://backed.test" in run["report"]
+ assert "invented.test" not in run["report"]
+ assert "Untrusted source list" not in run["report"]
+
+
[email protected]("error", [worker.RunCancelled(), worker.LeaseLost()])
+def test_control_flow_interruptions_do_not_publish_partial_report(
+ research_home, monkeypatch, error
+):
+ run = _interrupt(monkeypatch, RAW, error, "content", None)
+ assert run["report"] is None
+
+
+def test_failed_finish_does_not_promote_raw_progress(research_home):
+ supervisor = worker.ResearchSupervisor(SimpleNamespace(state = SimpleNamespace(server_port = 1)))
+ run = _claimed_run(supervisor)
+ research_db.set_report_progress(run["id"], RAW, worker_id = supervisor.worker_id)
+ assert research_db.finish(run["id"], supervisor.worker_id, "failed", "failed") == "failed"
+ assert research_db.get_run(run["id"])["report"] is None
diff --git a/studio/backend/tests/test_research_runs_storage.py b/studio/backend/tests/test_research_runs_storage.py
index fc288596e2e..200fdaa461e 100644
--- a/studio/backend/tests/test_research_runs_storage.py
+++ b/studio/backend/tests/test_research_runs_storage.py
@@ -2873,7 +2873,8 @@ def test_terminal_sse_event_contains_report_and_complete_snapshot(research_home)
("cancelled", "expected_status", "text"),
[
(True, "cancelled", "Research cancelled."),
- (False, "failed", "Research failed: mocked model failure"),
+ # Provider text reaches a Markdown surface here too, so it is quoted literally.
+ (False, "failed", "Research failed: `mocked model failure`"),
],
)
def test_worker_terminal_paths_create_one_fallback_without_frontend_message(
diff --git a/studio/frontend/src/features/chat/components/research-message.tsx b/studio/frontend/src/features/chat/components/research-message.tsx
index 9e1492fbd5e..5c580917084 100644
--- a/studio/frontend/src/features/chat/components/research-message.tsx
+++ b/studio/frontend/src/features/chat/components/research-message.tsx
@@ -79,7 +79,9 @@ export function ResearchMessage(): ReactElement | null {
);
}
- if (run.status === "completed" && run.report) {
+ // A failed run's report opens with its own notice, so nothing here repeats it.
+ if ((run.status === "completed" || run.status === "failed") && run.report) {
+ const failed = run.status === "failed";
const sources: SourceData[] = run.sources.map((source) => ({
id: String(source.id ?? source.url),
url: source.url,
@@ -109,9 +111,17 @@ export function ResearchMessage(): ReactElement | null {
className="mb-3 flex items-center gap-2 rounded-full text-sm text-muted-foreground transition-colors hover:text-foreground focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring"
>
<span className="flex size-5 items-center justify-center rounded-full bg-primary/10 text-primary">
- <Check className="size-3" />
+ {failed ? (
+ <TriangleAlert className="size-3" />
+ ) : (
+ <Check className="size-3" />
+ )}
+ </span>
+ <span>
+ {failed
+ ? `Deep research failed · ${sourceCount} sources`
+ : `Deep research completed · ${sourceCount} sources`}
</span>
- <span>Deep research completed · {sourceCount} sources</span>
<span className="text-primary">View activity</span>
</button>
<MarkdownPreview