Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

fix(turns): surface and retry orphaned worker loss (#1570)

Bingxi Zhao (Frank)3a0ed2b56042
diff --git a/deeptutor/api/routers/sessions.py b/deeptutor/api/routers/sessions.py
index 4cdb66a9d..3b034626b 100644
--- a/deeptutor/api/routers/sessions.py
+++ b/deeptutor/api/routers/sessions.py
@@ -167,6 +167,65 @@ def _redact_private_message_metadata(messages: list[dict[str, Any]]) -> None:
_redact_provider_state_metadata(messages)
+def _attach_orphaned_failed_turns(
+ messages: list[dict[str, Any]], turns: list[dict[str, Any]]
+) -> None:
+ """Expose a failed turn beside its saved user row when no reply exists.
+
+ Older rows have no turn id, so the first user row written during the
+ turn is the safe fallback. A failed preflight with no saved user row is
+ deliberately omitted rather than attached to an unrelated question.
+ """
+ users = [message for message in messages if message.get("role") == "user"]
+ for turn in turns:
+ turn_id = str(turn.get("turn_id") or turn.get("id") or "")
+ started = float(turn.get("created_at") or 0)
+ finished = float(turn.get("finished_at") or turn.get("updated_at") or 0)
+ user = next(
+ (
+ message
+ for message in users
+ if str((message.get("metadata") or {}).get("turn_id") or "") == turn_id
+ ),
+ None,
+ )
+ if user is None:
+ user = next(
+ (
+ message
+ for message in users
+ if started <= float(message.get("created_at") or 0) <= finished
+ ),
+ None,
+ )
+ if user is None:
+ continue
+ # A subsequent retry may have answered this same user row. Its real
+ # assistant bubble wins; showing an earlier failure beside it would
+ # tell the learner that the answered question is still broken.
+ if any(
+ message.get("role") == "assistant"
+ and str(message.get("parent_message_id") or "") == str(user.get("id") or "")
+ and float(message.get("created_at") or 0) >= started
+ for message in messages
+ ):
+ continue
+ metadata = user.get("metadata")
+ if not isinstance(metadata, dict):
+ metadata = {}
+ user["metadata"] = metadata
+ previous = metadata.get("orphaned_failed_turn")
+ if isinstance(previous, dict) and float(previous.get("finished_at") or 0) > finished:
+ continue
+ metadata["orphaned_failed_turn"] = {
+ "turn_id": turn_id,
+ "error": str(turn.get("error") or ""),
+ "failure_code": str(turn.get("failure_code") or ""),
+ "retryable": bool(turn.get("retryable")),
+ "finished_at": finished,
+ }
+
+
def _truncate_oversized_events(
messages: list[dict[str, Any]], limit: int = MAX_EVENT_PAYLOAD
) -> None:
@@ -216,6 +275,9 @@ async def get_session(session_id: str):
session = await store.get_session_with_messages(session_id)
if session is None:
raise HTTPException(status_code=404, detail="Session not found")
+ _attach_orphaned_failed_turns(
+ session.get("messages", []), await store.list_orphaned_failed_turns(session_id)
+ )
_redact_private_message_metadata(session.get("messages", []))
_truncate_oversized_events(session.get("messages", []))
return session
diff --git a/deeptutor/services/session/_turn_runtime_shared.py b/deeptutor/services/session/_turn_runtime_shared.py
index 0f7b4ca12..ac6828690 100644
--- a/deeptutor/services/session/_turn_runtime_shared.py
+++ b/deeptutor/services/session/_turn_runtime_shared.py
@@ -924,6 +924,13 @@ def _request_snapshot_metadata(
snapshot["capabilityRoute"] = dict(capability_route)
mastery_path_id = _mastery_path_id(payload.get("mastery_path_id"))
snapshot["masteryPathId"] = mastery_path_id
+ for payload_key, snapshot_key in (
+ ("mastery_answer", "masteryAnswer"),
+ ("mastery_skip", "masterySkip"),
+ ):
+ value = payload.get(payload_key)
+ if isinstance(value, dict) and value.get("question_id"):
+ snapshot[snapshot_key] = dict(value)
# Persisted so a regenerate re-runs with the same document open. Without it
# the reading capability would be inactive on the retry and the answer would
# silently lose its grounding.
diff --git a/deeptutor/services/session/pocketbase_store.py b/deeptutor/services/session/pocketbase_store.py
index f09cd9def..a91c0d4a4 100644
--- a/deeptutor/services/session/pocketbase_store.py
+++ b/deeptutor/services/session/pocketbase_store.py
@@ -1368,6 +1368,35 @@ class PocketBaseSessionStore:
except Exception:
return []
+ @_captured_store_context
+ async def list_orphaned_failed_turns(self, session_id: str) -> list[dict[str, Any]]:
+ if await self.get_session(session_id) is None:
+ return []
+ sid = _validate_id(session_id, "session_id")
+
+ def _list():
+ return (
+ _pb()
+ .collection("turns")
+ .get_full_list(
+ query_params={
+ "filter": f'session_id="{sid}" && status="failed"',
+ "sort": "turn_created_at",
+ }
+ )
+ )
+
+ try:
+ records = await asyncio.to_thread(_list)
+ except Exception:
+ logger.exception("Could not list orphaned failed turns for session %s", sid)
+ return []
+ return [
+ self._turn_record_to_dict(record)
+ for record in records
+ if not getattr(record, "assistant_message_id", None)
+ ]
+
@_captured_store_context
async def list_nonterminal_turns(self) -> list[dict[str, Any]]:
def _list():
diff --git a/deeptutor/services/session/protocol.py b/deeptutor/services/session/protocol.py
index 3ab2ca496..a9396cfc6 100644
--- a/deeptutor/services/session/protocol.py
+++ b/deeptutor/services/session/protocol.py
@@ -93,6 +93,8 @@ class TurnRepository(Protocol):
async def list_active_turns(self, session_id: str) -> list[dict[str, Any]]: ...
+ async def list_orphaned_failed_turns(self, session_id: str) -> list[dict[str, Any]]: ...
+
async def list_nonterminal_turns(self) -> list[dict[str, Any]]: ...
async def transition_turn(
@@ -162,6 +164,8 @@ class SessionStoreProtocol(SessionRepository, TurnRepository, MessageRepository,
async def list_active_turns(self, session_id: str) -> list[dict[str, Any]]: ...
+ async def list_orphaned_failed_turns(self, session_id: str) -> list[dict[str, Any]]: ...
+
async def update_turn_status(self, turn_id: str, status: str, error: str = "") -> bool: ...
async def append_turn_event(self, turn_id: str, event: dict[str, Any]) -> dict[str, Any]: ...
diff --git a/deeptutor/services/session/sqlite_store.py b/deeptutor/services/session/sqlite_store.py
index 5cda472f7..a559e2aaa 100644
--- a/deeptutor/services/session/sqlite_store.py
+++ b/deeptutor/services/session/sqlite_store.py
@@ -1408,6 +1408,23 @@ class SQLiteSessionStore:
async def list_active_turns(self, session_id: str) -> list[dict[str, Any]]:
return await self._run(self._list_active_turns_sync, session_id)
+ def _list_orphaned_failed_turns_sync(self, session_id: str) -> list[dict[str, Any]]:
+ with self._connect() as conn:
+ rows = conn.execute(
+ """
+ SELECT t.*, 0 AS last_seq
+ FROM turns t
+ WHERE t.session_id = ? AND t.status = 'failed'
+ AND t.assistant_message_id IS NULL
+ ORDER BY t.created_at, t.id
+ """,
+ (session_id,),
+ ).fetchall()
+ return [self._serialize_turn(row) for row in rows]
+
+ async def list_orphaned_failed_turns(self, session_id: str) -> list[dict[str, Any]]:
+ return await self._run(self._list_orphaned_failed_turns_sync, session_id)
+
def _list_nonterminal_turns_sync(self) -> list[dict[str, Any]]:
with self._connect() as conn:
rows = conn.execute(
diff --git a/deeptutor/services/session/turns/executor.py b/deeptutor/services/session/turns/executor.py
index dc2500601..e5828543a 100644
--- a/deeptutor/services/session/turns/executor.py
+++ b/deeptutor/services/session/turns/executor.py
@@ -857,22 +857,27 @@ class TurnExecutor:
content=raw_user_content,
capability=capability_name,
attachments=persisted_attachment_records,
- metadata=_request_snapshot_metadata(
- payload=payload,
- content=raw_user_content,
- capability=capability_name,
- config=request_config,
- attachments=persisted_attachment_records,
- notebook_references=notebook_references,
- history_references=history_references,
- partner_group_references=partner_group_references,
- question_notebook_references=question_notebook_references,
- book_references=book_references,
- reading_references=reading_references,
- persona=active_persona,
- memory_references=memory_references,
- llm_selection=payload.get("llm_selection"),
- ),
+ metadata={
+ **_request_snapshot_metadata(
+ payload=payload,
+ content=raw_user_content,
+ capability=capability_name,
+ config=request_config,
+ attachments=persisted_attachment_records,
+ notebook_references=notebook_references,
+ history_references=history_references,
+ partner_group_references=partner_group_references,
+ question_notebook_references=question_notebook_references,
+ book_references=book_references,
+ reading_references=reading_references,
+ persona=active_persona,
+ memory_references=memory_references,
+ llm_selection=payload.get("llm_selection"),
+ ),
+ # A recovered worker_lost turn may never have an
+ # assistant row. Keep its exact user-row association.
+ "turn_id": turn_id,
+ },
**parent_kwargs,
)
diff --git a/deeptutor/services/session/turns/lifecycle.py b/deeptutor/services/session/turns/lifecycle.py
index 38d06b259..a5348e314 100644
--- a/deeptutor/services/session/turns/lifecycle.py
+++ b/deeptutor/services/session/turns/lifecycle.py
@@ -27,6 +27,7 @@ if TYPE_CHECKING:
from deeptutor.services.app_update import UpdateJob
logger = logging.getLogger(__name__)
+_SUBSCRIPTION_POLL_SECONDS = 5.0
class TurnLifecycle:
@@ -363,11 +364,16 @@ class TurnLifecycle:
# the frontend's ``isStreaming`` state clears immediately rather than
# waiting on the 45s heartbeat-timeout + reconnect catchup path.
done_yielded = False
+ terminal_error_yielded = False
def _track(item: dict[str, Any]) -> dict[str, Any]:
- nonlocal done_yielded
+ nonlocal done_yielded, terminal_error_yielded
if str(item.get("type") or "") == "done":
done_yielded = True
+ if str(item.get("type") or "") == "error" and (item.get("metadata") or {}).get(
+ "turn_terminal"
+ ):
+ terminal_error_yielded = True
return item
for item in backlog:
@@ -410,7 +416,11 @@ class TurnLifecycle:
# persisted history above — synthesise one so the caller can
# still close out its streaming state cleanly.
if not done_yielded:
- if turn is not None and str(turn.get("status") or "") == "failed":
+ if (
+ not terminal_error_yielded
+ and turn is not None
+ and str(turn.get("status") or "") == "failed"
+ ):
error_event = self._synthesize_error_event(
turn_id,
turn,
@@ -425,14 +435,49 @@ class TurnLifecycle:
seq=last_seq + 1,
)
return
- # A running turn may be owned by another worker. Subscription is a
- # read-only operation; distributed coordinators attach a live event
- # source above this local-runtime fallback.
- return
+ # A running turn may be owned by another worker, including one
+ # whose lease is about to be recovered. Keep watching persisted
+ # events so recovery can reach this subscriber without requiring
+ # the browser to reconnect after its long idle timeout.
queue_drained = False
try:
while True:
- item = await queue.get()
+ try:
+ item = await asyncio.wait_for(queue.get(), timeout=_SUBSCRIPTION_POLL_SECONDS)
+ except TimeoutError:
+ # A dead producer cannot put a sentinel on its old local
+ # queue. Reconcile against the durable turn, including a
+ # recovery worker's terminal events, while the socket is
+ # still open (#1570).
+ for persisted in await self.store.get_turn_events(turn_id, after_seq=last_seq):
+ seq = int(persisted.get("seq") or 0)
+ if seq <= last_seq:
+ continue
+ last_seq = seq
+ yield _track(persisted)
+ persisted_turn = await self.store.get_turn(turn_id)
+ if persisted_turn is None or str(persisted_turn.get("status") or "") in {
+ "failed",
+ "cancelled",
+ "completed",
+ }:
+ if not done_yielded:
+ if (
+ not terminal_error_yielded
+ and persisted_turn is not None
+ and persisted_turn.get("status") == "failed"
+ ):
+ error_event = self._synthesize_error_event(
+ turn_id, persisted_turn, seq=last_seq + 1
+ )
+ if error_event is not None:
+ yield error_event
+ last_seq += 1
+ yield self._synthesize_done_event(
+ turn_id, persisted_turn, seq=last_seq + 1
+ )
+ break
+ continue
if item is None:
queue_drained = True
break
diff --git a/deeptutor/services/session/turns/request_preparer.py b/deeptutor/services/session/turns/request_preparer.py
index 5a9d59442..13965dc67 100644
--- a/deeptutor/services/session/turns/request_preparer.py
+++ b/deeptutor/services/session/turns/request_preparer.py
@@ -67,6 +67,8 @@ _REPLAY_SNAPSHOT_FIELDS: dict[str, str] = {
"course_id": "courseId",
"mastery_path_id": "masteryPathId",
"mastery_session_mode": "masterySessionMode",
+ "mastery_answer": "masteryAnswer",
+ "mastery_skip": "masterySkip",
"reading_material_id": "readingMaterialId",
"reading_material_revision": "readingMaterialRevision",
"reading_workspace_id": "readingWorkspaceId",
diff --git a/tests/api/test_sessions_trace.py b/tests/api/test_sessions_trace.py
index a6ca21fa8..de53d001a 100644
--- a/tests/api/test_sessions_trace.py
+++ b/tests/api/test_sessions_trace.py
@@ -68,3 +68,86 @@ def test_message_trace_is_paginated_and_session_scoped(tmp_path, monkeypatch) ->
first, last = message["events"][0], message["events"][-1]
assert trace["started_at"] <= first["timestamp"]
assert trace["ended_at"] >= last["timestamp"]
+
+
+def test_worker_lost_without_assistant_is_visible_after_session_reload(
+ tmp_path, monkeypatch
+) -> None:
+ store = SQLiteSessionStore(tmp_path / "chat.db")
+ session = asyncio.run(store.ensure_session(None))
+ turn = asyncio.run(store.begin_turn(session["id"], capability="chat"))
+ user_id = asyncio.run(
+ store.add_message(
+ session["id"],
+ "user",
+ "My mastery answer is B",
+ metadata={
+ "turn_id": turn["id"],
+ "request_snapshot": {"content": "My mastery answer is B"},
+ },
+ )
+ )
+ assert asyncio.run(
+ store.transition_turn(
+ turn["id"],
+ "failed",
+ error="Worker lost during this turn",
+ failure_code="worker_lost",
+ retryable=True,
+ )
+ )
+
+ monkeypatch.setattr(sessions_router, "get_session_store", lambda: store)
+ app = FastAPI()
+ app.include_router(sessions_router.router, prefix="/api/sessions")
+ with TestClient(app) as client:
+ response = client.get(f"/api/sessions/{session['id']}")
+ assert response.status_code == 200
+ detail = response.json()
+ assert detail["status"] == "failed"
+ assert len(detail["messages"]) == 1
+ saved = detail["messages"][0]
+ assert saved["id"] == user_id
+ assert saved["metadata"]["request_snapshot"]["content"] == "My mastery answer is B"
+ assert saved["metadata"]["orphaned_failed_turn"] == {
+ "turn_id": turn["id"],
+ "error": "Worker lost during this turn",
+ "failure_code": "worker_lost",
+ "retryable": True,
+ "finished_at": saved["metadata"]["orphaned_failed_turn"]["finished_at"],
+ }
+
+
+def test_legacy_worker_lost_matches_saved_user_but_not_an_answered_retry(
+ tmp_path, monkeypatch
+) -> None:
+ store = SQLiteSessionStore(tmp_path / "chat.db")
+ session = asyncio.run(store.ensure_session(None))
+ turn = asyncio.run(store.begin_turn(session["id"], capability="chat"))
+ user_id = asyncio.run(store.add_message(session["id"], "user", "B"))
+ asyncio.run(
+ store.transition_turn(
+ turn["id"],
+ "failed",
+ error="Worker lost",
+ failure_code="worker_lost",
+ retryable=True,
+ )
+ )
+ monkeypatch.setattr(sessions_router, "get_session_store", lambda: store)
+ app = FastAPI()
+ app.include_router(sessions_router.router, prefix="/api/sessions")
+ with TestClient(app) as client:
+ first = client.get(f"/api/sessions/{session['id']}").json()
+ assert first["messages"][0]["metadata"]["orphaned_failed_turn"]["turn_id"] == turn["id"]
+
+ asyncio.run(
+ store.add_message(
+ session["id"],
+ "assistant",
+ "The correct answer is B.",
+ parent_message_id=user_id,
+ )
+ )
+ answered = client.get(f"/api/sessions/{session['id']}").json()
+ assert "orphaned_failed_turn" not in answered["messages"][0]["metadata"]
diff --git a/tests/api/test_unified_ws_turn_runtime.py b/tests/api/test_unified_ws_turn_runtime.py
index 397cd6403..28c7cccb5 100644
--- a/tests/api/test_unified_ws_turn_runtime.py
+++ b/tests/api/test_unified_ws_turn_runtime.py
@@ -205,6 +205,7 @@ async def test_turn_runtime_replays_events_and_materializes_messages(
"memory_references": ["summary"],
"book_references": [{"book_id": "book-1", "page_ids": ["page-1"]}],
"mastery_path_id": "path-1",
+ "mastery_answer": {"question_id": "question-1", "text": "B"},
"config": {},
**consultation,
}
@@ -239,12 +240,17 @@ async def test_turn_runtime_replays_events_and_materializes_messages(
assert done_event["metadata"]["user_message_id"] == user_row["id"]
assert done_event["metadata"]["assistant_message_id"] == assistant_row["id"]
assert user_row["metadata"]["client_submission_id"] == "browser-submission-123"
+ assert user_row["metadata"]["turn_id"] == turn["id"]
assert detail["messages"][0]["metadata"]["request_snapshot"]["persona"] == "socratic"
assert detail["messages"][0]["metadata"]["request_snapshot"]["memoryReferences"] == ["summary"]
assert detail["messages"][0]["metadata"]["request_snapshot"]["bookReferences"] == [
{"book_id": "book-1", "page_ids": ["page-1"]}
]
assert detail["messages"][0]["metadata"]["request_snapshot"]["masteryPathId"] == "path-1"
+ assert detail["messages"][0]["metadata"]["request_snapshot"]["masteryAnswer"] == {
+ "question_id": "question-1",
+ "text": "B",
+ }
snapshot = detail["messages"][0]["metadata"]["request_snapshot"]
assert snapshot.get("consultPartnerId") == requested.get("consult_partner_id")
assert snapshot.get("partnerDiscussionGroupId") == requested.get("partner_discussion_group_id")
@@ -770,6 +776,7 @@ async def test_regenerate_reuses_snapshot_or_override_llm_selection(tmp_path) ->
"request_snapshot": {
"content": "again",
"llmSelection": {"profile_id": "p-alt", "model_id": "m-alt"},
+ "masteryAnswer": {"question_id": "question-1", "text": "B"},
}
},
)
@@ -780,6 +787,9 @@ async def test_regenerate_reuses_snapshot_or_override_llm_selection(tmp_path) ->
"model_id": "m-alt",
}
+ await runtime.regenerate_last_turn(session["id"], overrides={"replay_snapshot": True})
+ assert captured_payloads[-1]["mastery_answer"] == {"question_id": "question-1", "text": "B"}
+
await runtime.regenerate_last_turn(
session["id"],
overrides={"llm_selection": {"profile_id": "p-default", "model_id": "m-default"}},
diff --git a/tests/services/session/test_turn_runtime_subscribe.py b/tests/services/session/test_turn_runtime_subscribe.py
index 3f578c83f..f64a90705 100644
--- a/tests/services/session/test_turn_runtime_subscribe.py
+++ b/tests/services/session/test_turn_runtime_subscribe.py
@@ -16,6 +16,7 @@ from deeptutor.services.session.turn_runtime import (
_resolve_turn_outcome,
_TurnExecution,
)
+from deeptutor.services.session.turns import lifecycle as lifecycle_module
def _isolate_learning_store(monkeypatch: pytest.MonkeyPatch, tmp_path) -> None:
@@ -213,23 +214,67 @@ async def test_replacing_subscription_does_not_synthesize_duplicate_done(tmp_pat
@pytest.mark.asyncio
-async def test_subscribe_turn_does_not_mutate_remote_running_turn(tmp_path) -> None:
+async def test_subscribe_turn_does_not_mutate_remote_running_turn(tmp_path, monkeypatch) -> None:
"""A subscriber may be on a different worker from the turn owner."""
+ monkeypatch.setattr(lifecycle_module, "_SUBSCRIPTION_POLL_SECONDS", 0.01)
store = SQLiteSessionStore(tmp_path / "chat_history.db")
runtime = TurnRuntimeManager(store)
session = await store.ensure_session(None)
turn = await store.create_turn(session["id"], capability="chat")
events: list[dict] = []
- async for event in runtime.subscribe_turn(turn["id"], after_seq=0):
- events.append(event)
+
+ async def _collect() -> None:
+ async for event in runtime.subscribe_turn(turn["id"], after_seq=0):
+ events.append(event)
+
+ task = asyncio.create_task(_collect())
+ await asyncio.sleep(0.04)
persisted = await store.get_turn(turn["id"])
assert persisted is not None
assert persisted["status"] == "running"
assert persisted["error"] == ""
assert events == []
+ assert await store.update_turn_status(turn["id"], "completed") is True
+ await asyncio.wait_for(task, timeout=1)
+ assert [event["type"] for event in events] == ["done"]
+
+
+async def test_worker_lost_unblocks_a_stale_local_subscriber(tmp_path, monkeypatch) -> None:
+ """Recovery's durable terminal state reaches an abandoned live queue."""
+
+ monkeypatch.setattr(lifecycle_module, "_SUBSCRIPTION_POLL_SECONDS", 0.01)
+ store = SQLiteSessionStore(tmp_path / "chat_history.db")
+ runtime = TurnRuntimeManager(store)
+ session = await store.ensure_session(None)
+ turn = await store.create_turn(session["id"], capability="mastery_path")
+ execution = _TurnExecution(
+ turn_id=turn["id"], session_id=session["id"], capability="mastery_path", payload={}
+ )
+ runtime._executions[turn["id"]] = execution
+ events: list[dict] = []
+
+ async def _collect() -> None:
+ async for event in runtime.subscribe_turn(turn["id"]):
+ events.append(event)
+
+ task = asyncio.create_task(_collect())
+ for _ in range(100):
+ if execution.subscribers:
+ break
+ await asyncio.sleep(0.01)
+ assert execution.subscribers
+ assert await store.transition_turn(
+ turn["id"], "failed", error="Worker lost", failure_code="worker_lost", retryable=True
+ )
+ await asyncio.wait_for(task, timeout=1)
+
+ assert [event["type"] for event in events] == ["error", "done"]
+ assert events[0]["metadata"]["error_code"] == "worker_lost"
+ assert events[1]["metadata"]["status"] == "failed"
@pytest.mark.asyncio
diff --git a/web/components/space/learning/MasteryStudy.tsx b/web/components/space/learning/MasteryStudy.tsx
index 42f127924..8c3b594c7 100644
--- a/web/components/space/learning/MasteryStudy.tsx
+++ b/web/components/space/learning/MasteryStudy.tsx
@@ -177,6 +177,7 @@ export function MasteryStudy({
sendMessage,
submitUserReply,
regenerateLastMessage,
+ resendLastMessage,
deleteTurn,
editMessage,
switchBranch,
@@ -781,6 +782,8 @@ export function MasteryStudy({
language={state.language}
onCopyAssistantMessage={copyAssistantMessage}
onRegenerateMessage={regenerateLastMessage}
+ canResendLastTurn={state.lastTurnFailed}
+ onResendLastTurn={resendLastMessage}
onDeleteTurn={deleteTurn}
selectedBranches={state.selectedBranches}
onEditMessage={editMessage}
diff --git a/web/features/chat/ChatStateAdapter.tsx b/web/features/chat/ChatStateAdapter.tsx
index c3077d9c9..29b5a8f84 100644
--- a/web/features/chat/ChatStateAdapter.tsx
+++ b/web/features/chat/ChatStateAdapter.tsx
@@ -47,6 +47,7 @@ import {
updateSessionReplyLanguage,
type MessageTracePage,
type SessionMessage,
+ type OrphanedFailedTurn,
} from "@/lib/session-api";
import {
TraceCache,
@@ -284,6 +285,8 @@ export interface MessageRequestSnapshot {
readingReferences?: ReadingReferencePayload[];
masteryPathId?: string;
masterySessionMode?: string;
+ masteryAnswer?: { question_id: string; text: string };
+ masterySkip?: { question_id: string };
timedMediaId?: string;
persona?: string;
memoryReferences?: MemoryReferencePayload;
@@ -326,6 +329,8 @@ export interface MessageItem {
failedSubmissionNeedsReview?: boolean;
failedSubmissionNotSaved?: boolean;
failedSubmissionId?: string;
+ /** A backend failure with a saved user row and no assistant reply (#1570). */
+ orphanedFailedTurn?: OrphanedFailedTurn;
}
interface SessionEntry extends Omit<ChatState, "sessionKey"> {
@@ -1740,6 +1745,18 @@ function hydrateRequestSnapshot(
if (memoryReferences.length) snapshot.memoryReferences = memoryReferences;
if (llmSelection) snapshot.llmSelection = llmSelection;
if (masteryPathId) snapshot.masteryPathId = masteryPathId;
+ const masteryAnswer = asRecord(stored.masteryAnswer);
+ if (typeof masteryAnswer?.question_id === "string" &&
+ typeof masteryAnswer.text === "string") {
+ snapshot.masteryAnswer = {
+ question_id: masteryAnswer.question_id,
+ text: masteryAnswer.text,
+ };
+ }
+ const masterySkip = asRecord(stored.masterySkip);
+ if (typeof masterySkip?.question_id === "string") {
+ snapshot.masterySkip = { question_id: masterySkip.question_id };
+ }
if (readingMaterialId) {
snapshot.readingMaterialId = readingMaterialId;
if (readingMaterialRevision) {
@@ -1886,6 +1903,9 @@ export function ChatStateAdapterProvider({
? null
: message.parent_message_id,
...(requestSnapshot ? { requestSnapshot } : {}),
+ ...(message.role === "user" && message.metadata?.orphaned_failed_turn
+ ? { orphanedFailedTurn: message.metadata.orphaned_failed_turn }
+ : {}),
};
});
},
@@ -2836,6 +2856,8 @@ export function ChatStateAdapterProvider({
...(effectiveMasterySessionMode
? { masterySessionMode: effectiveMasterySessionMode }
: {}),
+ ...(options?.masteryAnswer ? { masteryAnswer: options.masteryAnswer } : {}),
+ ...(options?.masterySkip ? { masterySkip: options.masterySkip } : {}),
...(effectivePersona ? { persona: effectivePersona } : {}),
...(effectiveMemoryReferences?.length
? { memoryReferences: [...effectiveMemoryReferences] }
@@ -3010,8 +3032,8 @@ export function ChatStateAdapterProvider({
readingReferences: effectiveReadingReferences,
masteryPathId: effectiveMasteryPathId || null,
masterySessionMode: effectiveMasterySessionMode || null,
- masteryAnswer: options?.masteryAnswer ?? null,
- masterySkip: options?.masterySkip ?? null,
+ masteryAnswer: options?.masteryAnswer ?? replaySnapshot?.masteryAnswer ?? null,
+ masterySkip: options?.masterySkip ?? replaySnapshot?.masterySkip ?? null,
// Immersive reading. Gated on the stable workspace mode as well as on
// an open document: the reader outlives action switches and new
// sessions, so Home must never inherit its source context.
diff --git a/web/features/chat/messages/ChatMessageList.tsx b/web/features/chat/messages/ChatMessageList.tsx
index 9ee46396f..0be23080a 100644
--- a/web/features/chat/messages/ChatMessageList.tsx
+++ b/web/features/chat/messages/ChatMessageList.tsx
@@ -105,6 +105,7 @@ import {
} from "@/features/chat/trace/TracePresentation";
import { hasSettledFinalRound } from "@/features/chat/trace/selectors";
import type { MessageTraceMetadata } from "@/features/chat/trace/memory";
+import type { OrphanedFailedTurn } from "@/lib/session-api";
import { agentGlyph } from "@/components/agents/agent-icons";
import { useConsultationReference } from "@/hooks/useConsultationReference";
import { useConnectedAgentKinds } from "@/hooks/useConnectedAgentKinds";
@@ -145,6 +146,7 @@ interface ChatMessageItem {
/** The server never accepted this submission (#1594) — rendered as an
* unsent message, not an ordinary sent one. */
failedSubmission?: boolean;
+ orphanedFailedTurn?: OrphanedFailedTurn;
}
interface NotebookReferenceGroup {
@@ -2150,6 +2152,10 @@ export const ChatMessageList = memo(function ChatMessageList({
const sib =
msg.id !== undefined ? siblingsByMessageId.get(msg.id) : undefined;
const reply = messageRows[rowIndex + 1]?.msg;
+ const orphanedFailure =
+ reply?.role === "assistant" && reply.parentMessageId === msg.id
+ ? null
+ : msg.orphanedFailedTurn;
const consultationEvents = reply?.role === "assistant"
? (reply.events ?? []).filter(event => event.metadata?.trace_kind === "subagent_event")
: [];
@@ -2175,6 +2181,28 @@ export const ChatMessageList = memo(function ChatMessageList({
? () => onOpenConsultation(consultationEvents)
: undefined}
/>
+ {orphanedFailure ? (
+ <div
+ role="alert"
+ data-orphaned-failed-turn={orphanedFailure.turn_id}
+ className="mt-3 flex w-full max-w-[min(520px,90%)] items-center gap-2 rounded-xl border border-[var(--destructive)]/30 bg-[var(--destructive)]/5 px-3 py-2"
+ >
+ <AlertCircle className="h-4 w-4 shrink-0 text-[var(--destructive)]" />
+ <span className="min-w-0 flex-1 text-[12px] leading-[1.5] text-[var(--foreground)]">
+ {orphanedFailure.error || t("The turn was interrupted.")}
+ </span>
+ {!isStreaming && rowIndex === messageRows.length - 1 &&
+ canResendLastTurn && orphanedFailure.retryable && onResendLastTurn ? (
+ <button
+ type="button"
+ onClick={onResendLastTurn}
+ className="shrink-0 rounded-md px-2 py-1 text-[11.5px] font-medium text-[var(--destructive)] hover:bg-[var(--destructive)]/10"
+ >
+ {t("Retry")}
+ </button>
+ ) : null}
+ </div>
+ ) : null}
</div>
);
}
diff --git a/web/lib/chat-resend.ts b/web/lib/chat-resend.ts
index 860d505ab..51590614a 100644
--- a/web/lib/chat-resend.ts
+++ b/web/lib/chat-resend.ts
@@ -9,6 +9,7 @@ interface ChatBranchMessage extends BranchMessage {
failedSubmission?: boolean;
/** A quota fallback kept the text but not enough context for safe resend. */
failedSubmissionNeedsReview?: boolean;
+ orphanedFailedTurn?: { retryable: boolean };
}
/** A failed turn can be retried only while its tail is on the visible branch:
@@ -27,6 +28,9 @@ export function isFailedTurnVisible<T extends ChatBranchMessage>(
if (tail?.role === "user" && tail.failedSubmission && !tail.failedSubmissionNeedsReview) {
return buildVisiblePath(messages, selectedBranches).messages.at(-1) === tail;
}
+ if (tail?.role === "user" && tail.orphanedFailedTurn?.retryable) {
+ return buildVisiblePath(messages, selectedBranches).messages.at(-1) === tail;
+ }
return false;
}
@@ -41,7 +45,7 @@ interface RemoteMessage {
role: "user" | "assistant" | "system";
content: string;
parent_message_id?: MessageId | null;
- metadata?: { client_submission_id?: unknown };
+ metadata?: Record<string, unknown>;
}
interface RemoteSession {
diff --git a/web/lib/session-api.ts b/web/lib/session-api.ts
index 671fc5150..3aa032fd8 100644
--- a/web/lib/session-api.ts
+++ b/web/lib/session-api.ts
@@ -30,7 +30,9 @@ export interface SessionMessage {
title?: string;
caption?: string;
}>;
- metadata?: Record<string, unknown>;
+ metadata?: Record<string, unknown> & {
+ orphaned_failed_turn?: OrphanedFailedTurn;
+ };
trace?: MessageTraceMetadata;
created_at: number;
/** Edit-branching: id of the message this row continues. `null` for the
@@ -38,6 +40,15 @@ export interface SessionMessage {
parent_message_id?: number | null;
}
+/** A persisted failed turn with a saved user row but no assistant reply. */
+export interface OrphanedFailedTurn {
+ turn_id: string;
+ error: string;
+ failure_code: string;
+ retryable: boolean;
+ finished_at: number;
+}
+
export interface MessageTraceMetadata {
turn_id?: string | null;
total?: number;
diff --git a/web/tests/chat-failed-submission-recovery.spec.tsx b/web/tests/chat-failed-submission-recovery.spec.tsx
index fd6828c6a..cc359b3f0 100644
--- a/web/tests/chat-failed-submission-recovery.spec.tsx
+++ b/web/tests/chat-failed-submission-recovery.spec.tsx
@@ -225,6 +225,47 @@ beforeEach(() => {
afterEach(() => vi.restoreAllMocks());
+it("restores a persisted worker loss and retries the same mastery answer snapshot", async () => {
+ fixture.session = {
+ ...completedServerSession(),
+ status: "failed",
+ messages: [{
+ id: 3,
+ session_id: "s1",
+ role: "user",
+ content: "B",
+ events: [],
+ attachments: [],
+ created_at: 3,
+ parent_message_id: null,
+ metadata: {
+ request_snapshot: {
+ content: "B", capability: "mastery_path", masteryPathId: "path-1",
+ masteryAnswer: { question_id: "q-1", text: "B" },
+ },
+ orphaned_failed_turn: {
+ turn_id: "lost-turn", error: "Worker lost", failure_code: "worker_lost",
+ retryable: true, finished_at: 4,
+ },
+ },
+ }],
+ };
+ fixture.connected = true;
+ render(<ChatStateAdapterProvider><Harness /></ChatStateAdapterProvider>);
+ await act(async () => {
+ fireEvent.click(screen.getByText("Load"));
+ });
+ expect(screen.getByTestId("lastTurnFailed").textContent).toBe("true");
+ expect(readMessages()).toEqual([{ role: "user", content: "B", failed: false }]);
+
+ await act(async () => {
+ fireEvent.click(screen.getByText("Resend"));
+ });
+ expect(fixture.sent.at(-1)).toMatchObject({
+ type: "regenerate", session_id: "s1", overrides: { replay_snapshot: true },
+ });
+});
+
it("marks a submission the server never received as unsent, not a failed reply", async () => {
vi.useFakeTimers();
try {
diff --git a/web/tests/chat-resend-decision.test.ts b/web/tests/chat-resend-decision.test.ts
index e9e28dc67..02a0e2670 100644
--- a/web/tests/chat-resend-decision.test.ts
+++ b/web/tests/chat-resend-decision.test.ts
@@ -3,6 +3,24 @@ import test from "node:test";
import { decideFailedTurnReplay, isFailedTurnVisible } from "../lib/chat-resend";
+test("a saved mastery answer with a worker_lost turn remains retryable after reload", () => {
+ const user = {
+ id: 23,
+ role: "user" as const,
+ parentMessageId: null,
+ requestSnapshot: { content: "B" },
+ orphanedFailedTurn: { retryable: true },
+ };
+ assert.equal(isFailedTurnVisible([user], {}, "failed", false), true);
+ assert.deepEqual(
+ decideFailedTurnReplay(
+ [user], user,
+ { status: "failed", messages: [{ id: 23, role: "user", content: "B" }] },
+ ),
+ { kind: "regenerate" },
+ );
+});
+
test("an earlier completed turn does not discard an unsaved failed attempt", () => {
const previous = { id: 1, role: "user" as const, parentMessageId: null };
const lastUser = { id: -3, parentMessageId: 2, requestSnapshot: { content: "retry me" } };
diff --git a/web/tests/integration/chat-messages.spec.tsx b/web/tests/integration/chat-messages.spec.tsx
index 963c1d539..3f6b90466 100644
--- a/web/tests/integration/chat-messages.spec.tsx
+++ b/web/tests/integration/chat-messages.spec.tsx
@@ -9,6 +9,35 @@ import { initI18n } from '@/i18n/init'
initI18n('en')
describe('chat message feature', () => {
+ it('shows a persisted worker loss beside the saved mastery answer with Retry', async () => {
+ const retry = vi.fn()
+ render(
+ <ChatMessageList
+ messages={[{
+ id: 23,
+ role: 'user',
+ content: 'B',
+ parentMessageId: null,
+ requestSnapshot: { content: 'B', enabledTools: [], knowledgeBases: [], language: 'en' },
+ orphanedFailedTurn: {
+ turn_id: 'failed-turn', error: 'Worker lost during this turn',
+ failure_code: 'worker_lost', retryable: true, finished_at: 1,
+ },
+ }]}
+ isStreaming={false}
+ canResendLastTurn
+ onResendLastTurn={retry}
+ onCopyAssistantMessage={vi.fn()}
+ onRegenerateMessage={vi.fn()}
+ />,
+ )
+
+ expect(screen.getByText('B')).toBeVisible()
+ expect(screen.getByRole('alert')).toHaveTextContent('Worker lost during this turn')
+ await userEvent.setup().click(screen.getByRole('button', { name: 'Retry' }))
+ expect(retry).toHaveBeenCalledOnce()
+ })
+
it('renders a user row with keyboard-accessible message actions', async () => {
const copy = vi.fn(async () => undefined)
const user = userEvent.setup()