DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.
Python ai-agentsai-tutorclawbotcli-tooldeepresearchinteractive-learninglearningmulti-agent-systemsrag
fix(turns): surface and retry orphaned worker loss (#1570)
Bingxi Zhao (Frank)3a0ed2b56042
diff --git a/deeptutor/api/routers/sessions.py b/deeptutor/api/routers/sessions.pyindex 4cdb66a9d..3b034626b 100644--- a/deeptutor/api/routers/sessions.py+++ b/deeptutor/api/routers/sessions.py@@ -167,6 +167,65 @@ def _redact_private_message_metadata(messages: list[dict[str, Any]]) -> None:_redact_provider_state_metadata(messages)+def _attach_orphaned_failed_turns(+ messages: list[dict[str, Any]], turns: list[dict[str, Any]]+) -> None:+ """Expose a failed turn beside its saved user row when no reply exists.++ Older rows have no turn id, so the first user row written during the+ turn is the safe fallback. A failed preflight with no saved user row is+ deliberately omitted rather than attached to an unrelated question.+ """+ users = [message for message in messages if message.get("role") == "user"]+ for turn in turns:+ turn_id = str(turn.get("turn_id") or turn.get("id") or "")+ started = float(turn.get("created_at") or 0)+ finished = float(turn.get("finished_at") or turn.get("updated_at") or 0)+ user = next(+ (+ message+ for message in users+ if str((message.get("metadata") or {}).get("turn_id") or "") == turn_id+ ),+ None,+ )+ if user is None:+ user = next(+ (+ message+ for message in users+ if started <= float(message.get("created_at") or 0) <= finished+ ),+ None,+ )+ if user is None:+ continue+ # A subsequent retry may have answered this same user row. Its real+ # assistant bubble wins; showing an earlier failure beside it would+ # tell the learner that the answered question is still broken.+ if any(+ message.get("role") == "assistant"+ and str(message.get("parent_message_id") or "") == str(user.get("id") or "")+ and float(message.get("created_at") or 0) >= started+ for message in messages+ ):+ continue+ metadata = user.get("metadata")+ if not isinstance(metadata, dict):+ metadata = {}+ user["metadata"] = metadata+ previous = metadata.get("orphaned_failed_turn")+ if isinstance(previous, dict) and float(previous.get("finished_at") or 0) > finished:+ continue+ metadata["orphaned_failed_turn"] = {+ "turn_id": turn_id,+ "error": str(turn.get("error") or ""),+ "failure_code": str(turn.get("failure_code") or ""),+ "retryable": bool(turn.get("retryable")),+ "finished_at": finished,+ }++def _truncate_oversized_events(messages: list[dict[str, Any]], limit: int = MAX_EVENT_PAYLOAD) -> None:@@ -216,6 +275,9 @@ async def get_session(session_id: str):session = await store.get_session_with_messages(session_id)if session is None:raise HTTPException(status_code=404, detail="Session not found")+ _attach_orphaned_failed_turns(+ session.get("messages", []), await store.list_orphaned_failed_turns(session_id)+ )_redact_private_message_metadata(session.get("messages", []))_truncate_oversized_events(session.get("messages", []))return sessiondiff --git a/deeptutor/services/session/_turn_runtime_shared.py b/deeptutor/services/session/_turn_runtime_shared.pyindex 0f7b4ca12..ac6828690 100644--- a/deeptutor/services/session/_turn_runtime_shared.py+++ b/deeptutor/services/session/_turn_runtime_shared.py@@ -924,6 +924,13 @@ def _request_snapshot_metadata(snapshot["capabilityRoute"] = dict(capability_route)mastery_path_id = _mastery_path_id(payload.get("mastery_path_id"))snapshot["masteryPathId"] = mastery_path_id+ for payload_key, snapshot_key in (+ ("mastery_answer", "masteryAnswer"),+ ("mastery_skip", "masterySkip"),+ ):+ value = payload.get(payload_key)+ if isinstance(value, dict) and value.get("question_id"):+ snapshot[snapshot_key] = dict(value)# Persisted so a regenerate re-runs with the same document open. Without it# the reading capability would be inactive on the retry and the answer would# silently lose its grounding.diff --git a/deeptutor/services/session/pocketbase_store.py b/deeptutor/services/session/pocketbase_store.pyindex f09cd9def..a91c0d4a4 100644--- a/deeptutor/services/session/pocketbase_store.py+++ b/deeptutor/services/session/pocketbase_store.py@@ -1368,6 +1368,35 @@ class PocketBaseSessionStore:except Exception:return []+ @_captured_store_context+ async def list_orphaned_failed_turns(self, session_id: str) -> list[dict[str, Any]]:+ if await self.get_session(session_id) is None:+ return []+ sid = _validate_id(session_id, "session_id")++ def _list():+ return (+ _pb()+ .collection("turns")+ .get_full_list(+ query_params={+ "filter": f'session_id="{sid}" && status="failed"',+ "sort": "turn_created_at",+ }+ )+ )++ try:+ records = await asyncio.to_thread(_list)+ except Exception:+ logger.exception("Could not list orphaned failed turns for session %s", sid)+ return []+ return [+ self._turn_record_to_dict(record)+ for record in records+ if not getattr(record, "assistant_message_id", None)+ ]+@_captured_store_contextasync def list_nonterminal_turns(self) -> list[dict[str, Any]]:def _list():diff --git a/deeptutor/services/session/protocol.py b/deeptutor/services/session/protocol.pyindex 3ab2ca496..a9396cfc6 100644--- a/deeptutor/services/session/protocol.py+++ b/deeptutor/services/session/protocol.py@@ -93,6 +93,8 @@ class TurnRepository(Protocol):async def list_active_turns(self, session_id: str) -> list[dict[str, Any]]: ...+ async def list_orphaned_failed_turns(self, session_id: str) -> list[dict[str, Any]]: ...+async def list_nonterminal_turns(self) -> list[dict[str, Any]]: ...async def transition_turn(@@ -162,6 +164,8 @@ class SessionStoreProtocol(SessionRepository, TurnRepository, MessageRepository,async def list_active_turns(self, session_id: str) -> list[dict[str, Any]]: ...+ async def list_orphaned_failed_turns(self, session_id: str) -> list[dict[str, Any]]: ...+async def update_turn_status(self, turn_id: str, status: str, error: str = "") -> bool: ...async def append_turn_event(self, turn_id: str, event: dict[str, Any]) -> dict[str, Any]: ...diff --git a/deeptutor/services/session/sqlite_store.py b/deeptutor/services/session/sqlite_store.pyindex 5cda472f7..a559e2aaa 100644--- a/deeptutor/services/session/sqlite_store.py+++ b/deeptutor/services/session/sqlite_store.py@@ -1408,6 +1408,23 @@ class SQLiteSessionStore:async def list_active_turns(self, session_id: str) -> list[dict[str, Any]]:return await self._run(self._list_active_turns_sync, session_id)+ def _list_orphaned_failed_turns_sync(self, session_id: str) -> list[dict[str, Any]]:+ with self._connect() as conn:+ rows = conn.execute(+ """+ SELECT t.*, 0 AS last_seq+ FROM turns t+ WHERE t.session_id = ? AND t.status = 'failed'+ AND t.assistant_message_id IS NULL+ ORDER BY t.created_at, t.id+ """,+ (session_id,),+ ).fetchall()+ return [self._serialize_turn(row) for row in rows]++ async def list_orphaned_failed_turns(self, session_id: str) -> list[dict[str, Any]]:+ return await self._run(self._list_orphaned_failed_turns_sync, session_id)+def _list_nonterminal_turns_sync(self) -> list[dict[str, Any]]:with self._connect() as conn:rows = conn.execute(diff --git a/deeptutor/services/session/turns/executor.py b/deeptutor/services/session/turns/executor.pyindex dc2500601..e5828543a 100644--- a/deeptutor/services/session/turns/executor.py+++ b/deeptutor/services/session/turns/executor.py@@ -857,22 +857,27 @@ class TurnExecutor:content=raw_user_content,capability=capability_name,attachments=persisted_attachment_records,- metadata=_request_snapshot_metadata(- payload=payload,- content=raw_user_content,- capability=capability_name,- config=request_config,- attachments=persisted_attachment_records,- notebook_references=notebook_references,- history_references=history_references,- partner_group_references=partner_group_references,- question_notebook_references=question_notebook_references,- book_references=book_references,- reading_references=reading_references,- persona=active_persona,- memory_references=memory_references,- llm_selection=payload.get("llm_selection"),- ),+ metadata={+ **_request_snapshot_metadata(+ payload=payload,+ content=raw_user_content,+ capability=capability_name,+ config=request_config,+ attachments=persisted_attachment_records,+ notebook_references=notebook_references,+ history_references=history_references,+ partner_group_references=partner_group_references,+ question_notebook_references=question_notebook_references,+ book_references=book_references,+ reading_references=reading_references,+ persona=active_persona,+ memory_references=memory_references,+ llm_selection=payload.get("llm_selection"),+ ),+ # A recovered worker_lost turn may never have an+ # assistant row. Keep its exact user-row association.+ "turn_id": turn_id,+ },**parent_kwargs,)diff --git a/deeptutor/services/session/turns/lifecycle.py b/deeptutor/services/session/turns/lifecycle.pyindex 38d06b259..a5348e314 100644--- a/deeptutor/services/session/turns/lifecycle.py+++ b/deeptutor/services/session/turns/lifecycle.py@@ -27,6 +27,7 @@ if TYPE_CHECKING:from deeptutor.services.app_update import UpdateJoblogger = logging.getLogger(__name__)+_SUBSCRIPTION_POLL_SECONDS = 5.0class TurnLifecycle:@@ -363,11 +364,16 @@ class TurnLifecycle:# the frontend's ``isStreaming`` state clears immediately rather than# waiting on the 45s heartbeat-timeout + reconnect catchup path.done_yielded = False+ terminal_error_yielded = Falsedef _track(item: dict[str, Any]) -> dict[str, Any]:- nonlocal done_yielded+ nonlocal done_yielded, terminal_error_yieldedif str(item.get("type") or "") == "done":done_yielded = True+ if str(item.get("type") or "") == "error" and (item.get("metadata") or {}).get(+ "turn_terminal"+ ):+ terminal_error_yielded = Truereturn itemfor item in backlog:@@ -410,7 +416,11 @@ class TurnLifecycle:# persisted history above — synthesise one so the caller can# still close out its streaming state cleanly.if not done_yielded:- if turn is not None and str(turn.get("status") or "") == "failed":+ if (+ not terminal_error_yielded+ and turn is not None+ and str(turn.get("status") or "") == "failed"+ ):error_event = self._synthesize_error_event(turn_id,turn,@@ -425,14 +435,49 @@ class TurnLifecycle:seq=last_seq + 1,)return- # A running turn may be owned by another worker. Subscription is a- # read-only operation; distributed coordinators attach a live event- # source above this local-runtime fallback.- return+ # A running turn may be owned by another worker, including one+ # whose lease is about to be recovered. Keep watching persisted+ # events so recovery can reach this subscriber without requiring+ # the browser to reconnect after its long idle timeout.queue_drained = Falsetry:while True:- item = await queue.get()+ try:+ item = await asyncio.wait_for(queue.get(), timeout=_SUBSCRIPTION_POLL_SECONDS)+ except TimeoutError:+ # A dead producer cannot put a sentinel on its old local+ # queue. Reconcile against the durable turn, including a+ # recovery worker's terminal events, while the socket is+ # still open (#1570).+ for persisted in await self.store.get_turn_events(turn_id, after_seq=last_seq):+ seq = int(persisted.get("seq") or 0)+ if seq <= last_seq:+ continue+ last_seq = seq+ yield _track(persisted)+ persisted_turn = await self.store.get_turn(turn_id)+ if persisted_turn is None or str(persisted_turn.get("status") or "") in {+ "failed",+ "cancelled",+ "completed",+ }:+ if not done_yielded:+ if (+ not terminal_error_yielded+ and persisted_turn is not None+ and persisted_turn.get("status") == "failed"+ ):+ error_event = self._synthesize_error_event(+ turn_id, persisted_turn, seq=last_seq + 1+ )+ if error_event is not None:+ yield error_event+ last_seq += 1+ yield self._synthesize_done_event(+ turn_id, persisted_turn, seq=last_seq + 1+ )+ break+ continueif item is None:queue_drained = Truebreakdiff --git a/deeptutor/services/session/turns/request_preparer.py b/deeptutor/services/session/turns/request_preparer.pyindex 5a9d59442..13965dc67 100644--- a/deeptutor/services/session/turns/request_preparer.py+++ b/deeptutor/services/session/turns/request_preparer.py@@ -67,6 +67,8 @@ _REPLAY_SNAPSHOT_FIELDS: dict[str, str] = {"course_id": "courseId","mastery_path_id": "masteryPathId","mastery_session_mode": "masterySessionMode",+ "mastery_answer": "masteryAnswer",+ "mastery_skip": "masterySkip","reading_material_id": "readingMaterialId","reading_material_revision": "readingMaterialRevision","reading_workspace_id": "readingWorkspaceId",diff --git a/tests/api/test_sessions_trace.py b/tests/api/test_sessions_trace.pyindex a6ca21fa8..de53d001a 100644--- a/tests/api/test_sessions_trace.py+++ b/tests/api/test_sessions_trace.py@@ -68,3 +68,86 @@ def test_message_trace_is_paginated_and_session_scoped(tmp_path, monkeypatch) ->first, last = message["events"][0], message["events"][-1]assert trace["started_at"] <= first["timestamp"]assert trace["ended_at"] >= last["timestamp"]+++def test_worker_lost_without_assistant_is_visible_after_session_reload(+ tmp_path, monkeypatch+) -> None:+ store = SQLiteSessionStore(tmp_path / "chat.db")+ session = asyncio.run(store.ensure_session(None))+ turn = asyncio.run(store.begin_turn(session["id"], capability="chat"))+ user_id = asyncio.run(+ store.add_message(+ session["id"],+ "user",+ "My mastery answer is B",+ metadata={+ "turn_id": turn["id"],+ "request_snapshot": {"content": "My mastery answer is B"},+ },+ )+ )+ assert asyncio.run(+ store.transition_turn(+ turn["id"],+ "failed",+ error="Worker lost during this turn",+ failure_code="worker_lost",+ retryable=True,+ )+ )++ monkeypatch.setattr(sessions_router, "get_session_store", lambda: store)+ app = FastAPI()+ app.include_router(sessions_router.router, prefix="/api/sessions")+ with TestClient(app) as client:+ response = client.get(f"/api/sessions/{session['id']}")+ assert response.status_code == 200+ detail = response.json()+ assert detail["status"] == "failed"+ assert len(detail["messages"]) == 1+ saved = detail["messages"][0]+ assert saved["id"] == user_id+ assert saved["metadata"]["request_snapshot"]["content"] == "My mastery answer is B"+ assert saved["metadata"]["orphaned_failed_turn"] == {+ "turn_id": turn["id"],+ "error": "Worker lost during this turn",+ "failure_code": "worker_lost",+ "retryable": True,+ "finished_at": saved["metadata"]["orphaned_failed_turn"]["finished_at"],+ }+++def test_legacy_worker_lost_matches_saved_user_but_not_an_answered_retry(+ tmp_path, monkeypatch+) -> None:+ store = SQLiteSessionStore(tmp_path / "chat.db")+ session = asyncio.run(store.ensure_session(None))+ turn = asyncio.run(store.begin_turn(session["id"], capability="chat"))+ user_id = asyncio.run(store.add_message(session["id"], "user", "B"))+ asyncio.run(+ store.transition_turn(+ turn["id"],+ "failed",+ error="Worker lost",+ failure_code="worker_lost",+ retryable=True,+ )+ )+ monkeypatch.setattr(sessions_router, "get_session_store", lambda: store)+ app = FastAPI()+ app.include_router(sessions_router.router, prefix="/api/sessions")+ with TestClient(app) as client:+ first = client.get(f"/api/sessions/{session['id']}").json()+ assert first["messages"][0]["metadata"]["orphaned_failed_turn"]["turn_id"] == turn["id"]++ asyncio.run(+ store.add_message(+ session["id"],+ "assistant",+ "The correct answer is B.",+ parent_message_id=user_id,+ )+ )+ answered = client.get(f"/api/sessions/{session['id']}").json()+ assert "orphaned_failed_turn" not in answered["messages"][0]["metadata"]diff --git a/tests/api/test_unified_ws_turn_runtime.py b/tests/api/test_unified_ws_turn_runtime.pyindex 397cd6403..28c7cccb5 100644--- a/tests/api/test_unified_ws_turn_runtime.py+++ b/tests/api/test_unified_ws_turn_runtime.py@@ -205,6 +205,7 @@ async def test_turn_runtime_replays_events_and_materializes_messages("memory_references": ["summary"],"book_references": [{"book_id": "book-1", "page_ids": ["page-1"]}],"mastery_path_id": "path-1",+ "mastery_answer": {"question_id": "question-1", "text": "B"},"config": {},**consultation,}@@ -239,12 +240,17 @@ async def test_turn_runtime_replays_events_and_materializes_messages(assert done_event["metadata"]["user_message_id"] == user_row["id"]assert done_event["metadata"]["assistant_message_id"] == assistant_row["id"]assert user_row["metadata"]["client_submission_id"] == "browser-submission-123"+ assert user_row["metadata"]["turn_id"] == turn["id"]assert detail["messages"][0]["metadata"]["request_snapshot"]["persona"] == "socratic"assert detail["messages"][0]["metadata"]["request_snapshot"]["memoryReferences"] == ["summary"]assert detail["messages"][0]["metadata"]["request_snapshot"]["bookReferences"] == [{"book_id": "book-1", "page_ids": ["page-1"]}]assert detail["messages"][0]["metadata"]["request_snapshot"]["masteryPathId"] == "path-1"+ assert detail["messages"][0]["metadata"]["request_snapshot"]["masteryAnswer"] == {+ "question_id": "question-1",+ "text": "B",+ }snapshot = detail["messages"][0]["metadata"]["request_snapshot"]assert snapshot.get("consultPartnerId") == requested.get("consult_partner_id")assert snapshot.get("partnerDiscussionGroupId") == requested.get("partner_discussion_group_id")@@ -770,6 +776,7 @@ async def test_regenerate_reuses_snapshot_or_override_llm_selection(tmp_path) ->"request_snapshot": {"content": "again","llmSelection": {"profile_id": "p-alt", "model_id": "m-alt"},+ "masteryAnswer": {"question_id": "question-1", "text": "B"},}},)@@ -780,6 +787,9 @@ async def test_regenerate_reuses_snapshot_or_override_llm_selection(tmp_path) ->"model_id": "m-alt",}+ await runtime.regenerate_last_turn(session["id"], overrides={"replay_snapshot": True})+ assert captured_payloads[-1]["mastery_answer"] == {"question_id": "question-1", "text": "B"}+await runtime.regenerate_last_turn(session["id"],overrides={"llm_selection": {"profile_id": "p-default", "model_id": "m-default"}},diff --git a/tests/services/session/test_turn_runtime_subscribe.py b/tests/services/session/test_turn_runtime_subscribe.pyindex 3f578c83f..f64a90705 100644--- a/tests/services/session/test_turn_runtime_subscribe.py+++ b/tests/services/session/test_turn_runtime_subscribe.py@@ -16,6 +16,7 @@ from deeptutor.services.session.turn_runtime import (_resolve_turn_outcome,_TurnExecution,)+from deeptutor.services.session.turns import lifecycle as lifecycle_moduledef _isolate_learning_store(monkeypatch: pytest.MonkeyPatch, tmp_path) -> None:@@ -213,23 +214,67 @@ async def test_replacing_subscription_does_not_synthesize_duplicate_done(tmp_pat@pytest.mark.asyncio-async def test_subscribe_turn_does_not_mutate_remote_running_turn(tmp_path) -> None:+async def test_subscribe_turn_does_not_mutate_remote_running_turn(tmp_path, monkeypatch) -> None:"""A subscriber may be on a different worker from the turn owner."""+ monkeypatch.setattr(lifecycle_module, "_SUBSCRIPTION_POLL_SECONDS", 0.01)store = SQLiteSessionStore(tmp_path / "chat_history.db")runtime = TurnRuntimeManager(store)session = await store.ensure_session(None)turn = await store.create_turn(session["id"], capability="chat")events: list[dict] = []- async for event in runtime.subscribe_turn(turn["id"], after_seq=0):- events.append(event)++ async def _collect() -> None:+ async for event in runtime.subscribe_turn(turn["id"], after_seq=0):+ events.append(event)++ task = asyncio.create_task(_collect())+ await asyncio.sleep(0.04)persisted = await store.get_turn(turn["id"])assert persisted is not Noneassert persisted["status"] == "running"assert persisted["error"] == ""assert events == []+ assert await store.update_turn_status(turn["id"], "completed") is True+ await asyncio.wait_for(task, timeout=1)+ assert [event["type"] for event in events] == ["done"]+++async def test_worker_lost_unblocks_a_stale_local_subscriber(tmp_path, monkeypatch) -> None:+ """Recovery's durable terminal state reaches an abandoned live queue."""++ monkeypatch.setattr(lifecycle_module, "_SUBSCRIPTION_POLL_SECONDS", 0.01)+ store = SQLiteSessionStore(tmp_path / "chat_history.db")+ runtime = TurnRuntimeManager(store)+ session = await store.ensure_session(None)+ turn = await store.create_turn(session["id"], capability="mastery_path")+ execution = _TurnExecution(+ turn_id=turn["id"], session_id=session["id"], capability="mastery_path", payload={}+ )+ runtime._executions[turn["id"]] = execution+ events: list[dict] = []++ async def _collect() -> None:+ async for event in runtime.subscribe_turn(turn["id"]):+ events.append(event)++ task = asyncio.create_task(_collect())+ for _ in range(100):+ if execution.subscribers:+ break+ await asyncio.sleep(0.01)+ assert execution.subscribers+ assert await store.transition_turn(+ turn["id"], "failed", error="Worker lost", failure_code="worker_lost", retryable=True+ )+ await asyncio.wait_for(task, timeout=1)++ assert [event["type"] for event in events] == ["error", "done"]+ assert events[0]["metadata"]["error_code"] == "worker_lost"+ assert events[1]["metadata"]["status"] == "failed"@pytest.mark.asynciodiff --git a/web/components/space/learning/MasteryStudy.tsx b/web/components/space/learning/MasteryStudy.tsxindex 42f127924..8c3b594c7 100644--- a/web/components/space/learning/MasteryStudy.tsx+++ b/web/components/space/learning/MasteryStudy.tsx@@ -177,6 +177,7 @@ export function MasteryStudy({sendMessage,submitUserReply,regenerateLastMessage,+ resendLastMessage,deleteTurn,editMessage,switchBranch,@@ -781,6 +782,8 @@ export function MasteryStudy({language={state.language}onCopyAssistantMessage={copyAssistantMessage}onRegenerateMessage={regenerateLastMessage}+ canResendLastTurn={state.lastTurnFailed}+ onResendLastTurn={resendLastMessage}onDeleteTurn={deleteTurn}selectedBranches={state.selectedBranches}onEditMessage={editMessage}diff --git a/web/features/chat/ChatStateAdapter.tsx b/web/features/chat/ChatStateAdapter.tsxindex c3077d9c9..29b5a8f84 100644--- a/web/features/chat/ChatStateAdapter.tsx+++ b/web/features/chat/ChatStateAdapter.tsx@@ -47,6 +47,7 @@ import {updateSessionReplyLanguage,type MessageTracePage,type SessionMessage,+ type OrphanedFailedTurn,} from "@/lib/session-api";import {TraceCache,@@ -284,6 +285,8 @@ export interface MessageRequestSnapshot {readingReferences?: ReadingReferencePayload[];masteryPathId?: string;masterySessionMode?: string;+ masteryAnswer?: { question_id: string; text: string };+ masterySkip?: { question_id: string };timedMediaId?: string;persona?: string;memoryReferences?: MemoryReferencePayload;@@ -326,6 +329,8 @@ export interface MessageItem {failedSubmissionNeedsReview?: boolean;failedSubmissionNotSaved?: boolean;failedSubmissionId?: string;+ /** A backend failure with a saved user row and no assistant reply (#1570). */+ orphanedFailedTurn?: OrphanedFailedTurn;}interface SessionEntry extends Omit<ChatState, "sessionKey"> {@@ -1740,6 +1745,18 @@ function hydrateRequestSnapshot(if (memoryReferences.length) snapshot.memoryReferences = memoryReferences;if (llmSelection) snapshot.llmSelection = llmSelection;if (masteryPathId) snapshot.masteryPathId = masteryPathId;+ const masteryAnswer = asRecord(stored.masteryAnswer);+ if (typeof masteryAnswer?.question_id === "string" &&+ typeof masteryAnswer.text === "string") {+ snapshot.masteryAnswer = {+ question_id: masteryAnswer.question_id,+ text: masteryAnswer.text,+ };+ }+ const masterySkip = asRecord(stored.masterySkip);+ if (typeof masterySkip?.question_id === "string") {+ snapshot.masterySkip = { question_id: masterySkip.question_id };+ }if (readingMaterialId) {snapshot.readingMaterialId = readingMaterialId;if (readingMaterialRevision) {@@ -1886,6 +1903,9 @@ export function ChatStateAdapterProvider({? null: message.parent_message_id,...(requestSnapshot ? { requestSnapshot } : {}),+ ...(message.role === "user" && message.metadata?.orphaned_failed_turn+ ? { orphanedFailedTurn: message.metadata.orphaned_failed_turn }+ : {}),};});},@@ -2836,6 +2856,8 @@ export function ChatStateAdapterProvider({...(effectiveMasterySessionMode? { masterySessionMode: effectiveMasterySessionMode }: {}),+ ...(options?.masteryAnswer ? { masteryAnswer: options.masteryAnswer } : {}),+ ...(options?.masterySkip ? { masterySkip: options.masterySkip } : {}),...(effectivePersona ? { persona: effectivePersona } : {}),...(effectiveMemoryReferences?.length? { memoryReferences: [...effectiveMemoryReferences] }@@ -3010,8 +3032,8 @@ export function ChatStateAdapterProvider({readingReferences: effectiveReadingReferences,masteryPathId: effectiveMasteryPathId || null,masterySessionMode: effectiveMasterySessionMode || null,- masteryAnswer: options?.masteryAnswer ?? null,- masterySkip: options?.masterySkip ?? null,+ masteryAnswer: options?.masteryAnswer ?? replaySnapshot?.masteryAnswer ?? null,+ masterySkip: options?.masterySkip ?? replaySnapshot?.masterySkip ?? null,// Immersive reading. Gated on the stable workspace mode as well as on// an open document: the reader outlives action switches and new// sessions, so Home must never inherit its source context.diff --git a/web/features/chat/messages/ChatMessageList.tsx b/web/features/chat/messages/ChatMessageList.tsxindex 9ee46396f..0be23080a 100644--- a/web/features/chat/messages/ChatMessageList.tsx+++ b/web/features/chat/messages/ChatMessageList.tsx@@ -105,6 +105,7 @@ import {} from "@/features/chat/trace/TracePresentation";import { hasSettledFinalRound } from "@/features/chat/trace/selectors";import type { MessageTraceMetadata } from "@/features/chat/trace/memory";+import type { OrphanedFailedTurn } from "@/lib/session-api";import { agentGlyph } from "@/components/agents/agent-icons";import { useConsultationReference } from "@/hooks/useConsultationReference";import { useConnectedAgentKinds } from "@/hooks/useConnectedAgentKinds";@@ -145,6 +146,7 @@ interface ChatMessageItem {/** The server never accepted this submission (#1594) — rendered as an* unsent message, not an ordinary sent one. */failedSubmission?: boolean;+ orphanedFailedTurn?: OrphanedFailedTurn;}interface NotebookReferenceGroup {@@ -2150,6 +2152,10 @@ export const ChatMessageList = memo(function ChatMessageList({const sib =msg.id !== undefined ? siblingsByMessageId.get(msg.id) : undefined;const reply = messageRows[rowIndex + 1]?.msg;+ const orphanedFailure =+ reply?.role === "assistant" && reply.parentMessageId === msg.id+ ? null+ : msg.orphanedFailedTurn;const consultationEvents = reply?.role === "assistant"? (reply.events ?? []).filter(event => event.metadata?.trace_kind === "subagent_event"): [];@@ -2175,6 +2181,28 @@ export const ChatMessageList = memo(function ChatMessageList({? () => onOpenConsultation(consultationEvents): undefined}/>+ {orphanedFailure ? (+ <div+ role="alert"+ data-orphaned-failed-turn={orphanedFailure.turn_id}+ className="mt-3 flex w-full max-w-[min(520px,90%)] items-center gap-2 rounded-xl border border-[var(--destructive)]/30 bg-[var(--destructive)]/5 px-3 py-2"+ >+ <AlertCircle className="h-4 w-4 shrink-0 text-[var(--destructive)]" />+ <span className="min-w-0 flex-1 text-[12px] leading-[1.5] text-[var(--foreground)]">+ {orphanedFailure.error || t("The turn was interrupted.")}+ </span>+ {!isStreaming && rowIndex === messageRows.length - 1 &&+ canResendLastTurn && orphanedFailure.retryable && onResendLastTurn ? (+ <button+ type="button"+ onClick={onResendLastTurn}+ className="shrink-0 rounded-md px-2 py-1 text-[11.5px] font-medium text-[var(--destructive)] hover:bg-[var(--destructive)]/10"+ >+ {t("Retry")}+ </button>+ ) : null}+ </div>+ ) : null}</div>);}diff --git a/web/lib/chat-resend.ts b/web/lib/chat-resend.tsindex 860d505ab..51590614a 100644--- a/web/lib/chat-resend.ts+++ b/web/lib/chat-resend.ts@@ -9,6 +9,7 @@ interface ChatBranchMessage extends BranchMessage {failedSubmission?: boolean;/** A quota fallback kept the text but not enough context for safe resend. */failedSubmissionNeedsReview?: boolean;+ orphanedFailedTurn?: { retryable: boolean };}/** A failed turn can be retried only while its tail is on the visible branch:@@ -27,6 +28,9 @@ export function isFailedTurnVisible<T extends ChatBranchMessage>(if (tail?.role === "user" && tail.failedSubmission && !tail.failedSubmissionNeedsReview) {return buildVisiblePath(messages, selectedBranches).messages.at(-1) === tail;}+ if (tail?.role === "user" && tail.orphanedFailedTurn?.retryable) {+ return buildVisiblePath(messages, selectedBranches).messages.at(-1) === tail;+ }return false;}@@ -41,7 +45,7 @@ interface RemoteMessage {role: "user" | "assistant" | "system";content: string;parent_message_id?: MessageId | null;- metadata?: { client_submission_id?: unknown };+ metadata?: Record<string, unknown>;}interface RemoteSession {diff --git a/web/lib/session-api.ts b/web/lib/session-api.tsindex 671fc5150..3aa032fd8 100644--- a/web/lib/session-api.ts+++ b/web/lib/session-api.ts@@ -30,7 +30,9 @@ export interface SessionMessage {title?: string;caption?: string;}>;- metadata?: Record<string, unknown>;+ metadata?: Record<string, unknown> & {+ orphaned_failed_turn?: OrphanedFailedTurn;+ };trace?: MessageTraceMetadata;created_at: number;/** Edit-branching: id of the message this row continues. `null` for the@@ -38,6 +40,15 @@ export interface SessionMessage {parent_message_id?: number | null;}+/** A persisted failed turn with a saved user row but no assistant reply. */+export interface OrphanedFailedTurn {+ turn_id: string;+ error: string;+ failure_code: string;+ retryable: boolean;+ finished_at: number;+}+export interface MessageTraceMetadata {turn_id?: string | null;total?: number;diff --git a/web/tests/chat-failed-submission-recovery.spec.tsx b/web/tests/chat-failed-submission-recovery.spec.tsxindex fd6828c6a..cc359b3f0 100644--- a/web/tests/chat-failed-submission-recovery.spec.tsx+++ b/web/tests/chat-failed-submission-recovery.spec.tsx@@ -225,6 +225,47 @@ beforeEach(() => {afterEach(() => vi.restoreAllMocks());+it("restores a persisted worker loss and retries the same mastery answer snapshot", async () => {+ fixture.session = {+ ...completedServerSession(),+ status: "failed",+ messages: [{+ id: 3,+ session_id: "s1",+ role: "user",+ content: "B",+ events: [],+ attachments: [],+ created_at: 3,+ parent_message_id: null,+ metadata: {+ request_snapshot: {+ content: "B", capability: "mastery_path", masteryPathId: "path-1",+ masteryAnswer: { question_id: "q-1", text: "B" },+ },+ orphaned_failed_turn: {+ turn_id: "lost-turn", error: "Worker lost", failure_code: "worker_lost",+ retryable: true, finished_at: 4,+ },+ },+ }],+ };+ fixture.connected = true;+ render(<ChatStateAdapterProvider><Harness /></ChatStateAdapterProvider>);+ await act(async () => {+ fireEvent.click(screen.getByText("Load"));+ });+ expect(screen.getByTestId("lastTurnFailed").textContent).toBe("true");+ expect(readMessages()).toEqual([{ role: "user", content: "B", failed: false }]);++ await act(async () => {+ fireEvent.click(screen.getByText("Resend"));+ });+ expect(fixture.sent.at(-1)).toMatchObject({+ type: "regenerate", session_id: "s1", overrides: { replay_snapshot: true },+ });+});+it("marks a submission the server never received as unsent, not a failed reply", async () => {vi.useFakeTimers();try {diff --git a/web/tests/chat-resend-decision.test.ts b/web/tests/chat-resend-decision.test.tsindex e9e28dc67..02a0e2670 100644--- a/web/tests/chat-resend-decision.test.ts+++ b/web/tests/chat-resend-decision.test.ts@@ -3,6 +3,24 @@ import test from "node:test";import { decideFailedTurnReplay, isFailedTurnVisible } from "../lib/chat-resend";+test("a saved mastery answer with a worker_lost turn remains retryable after reload", () => {+ const user = {+ id: 23,+ role: "user" as const,+ parentMessageId: null,+ requestSnapshot: { content: "B" },+ orphanedFailedTurn: { retryable: true },+ };+ assert.equal(isFailedTurnVisible([user], {}, "failed", false), true);+ assert.deepEqual(+ decideFailedTurnReplay(+ [user], user,+ { status: "failed", messages: [{ id: 23, role: "user", content: "B" }] },+ ),+ { kind: "regenerate" },+ );+});+test("an earlier completed turn does not discard an unsaved failed attempt", () => {const previous = { id: 1, role: "user" as const, parentMessageId: null };const lastUser = { id: -3, parentMessageId: 2, requestSnapshot: { content: "retry me" } };diff --git a/web/tests/integration/chat-messages.spec.tsx b/web/tests/integration/chat-messages.spec.tsxindex 963c1d539..3f6b90466 100644--- a/web/tests/integration/chat-messages.spec.tsx+++ b/web/tests/integration/chat-messages.spec.tsx@@ -9,6 +9,35 @@ import { initI18n } from '@/i18n/init'initI18n('en')describe('chat message feature', () => {+ it('shows a persisted worker loss beside the saved mastery answer with Retry', async () => {+ const retry = vi.fn()+ render(+ <ChatMessageList+ messages={[{+ id: 23,+ role: 'user',+ content: 'B',+ parentMessageId: null,+ requestSnapshot: { content: 'B', enabledTools: [], knowledgeBases: [], language: 'en' },+ orphanedFailedTurn: {+ turn_id: 'failed-turn', error: 'Worker lost during this turn',+ failure_code: 'worker_lost', retryable: true, finished_at: 1,+ },+ }]}+ isStreaming={false}+ canResendLastTurn+ onResendLastTurn={retry}+ onCopyAssistantMessage={vi.fn()}+ onRegenerateMessage={vi.fn()}+ />,+ )++ expect(screen.getByText('B')).toBeVisible()+ expect(screen.getByRole('alert')).toHaveTextContent('Worker lost during this turn')+ await userEvent.setup().click(screen.getByRole('button', { name: 'Retry' }))+ expect(retry).toHaveBeenCalledOnce()+ })+it('renders a user row with keyboard-accessible message actions', async () => {const copy = vi.fn(async () => undefined)const user = userEvent.setup()