DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.
Python ai-agentsai-tutorclawbotcli-tooldeepresearchinteractive-learninglearningmulti-agent-systemsrag
fix(runtime): 补全恢复中断后的失败事件回放
AleFlyXa9d6c99d89a8
diff --git a/deeptutor/app/service.py b/deeptutor/app/service.pyindex 286bb962b..d9e18316f 100644--- a/deeptutor/app/service.py+++ b/deeptutor/app/service.py@@ -321,13 +321,29 @@ class TurnApplicationService:# Ownership/cleanup can change between the reads above.if await store.get_events(turn_id, last_seq):continue- # Only legacy terminal rows with no journal need synthesis.+ # Recovery can commit FAILED before its terminal journal is+ # written. A client may also have seen volatile events beyond+ # the durable cursor. In either case preserve the failure+ # envelope: DONE alone hides errors on partial responses.metadata = {"status": status,"synthesized": True,"error_code": str(turn.get("failure_code") or ""),"retryable": bool(turn.get("retryable")),}+ if status == "failed":+ last_seq += 1+ yield {+ "type": "error",+ "source": "turn_application",+ "stage": "",+ "content": str(turn.get("error") or "Turn failed; retry to continue"),+ "metadata": {**metadata, "turn_terminal": True},+ "session_id": turn.get("session_id", ""),+ "turn_id": turn_id,+ "seq": last_seq,+ "timestamp": time.time(),+ }yield {"type": "done","source": "turn_application",diff --git a/tests/app/test_turn_application_service.py b/tests/app/test_turn_application_service.pyindex 575ab11ae..2d2e89e98 100644--- a/tests/app/test_turn_application_service.py+++ b/tests/app/test_turn_application_service.py@@ -540,3 +540,57 @@ async def test_replacement_availability_matches_configured_store_before_mutationawait sessions.replace_submission(body)assert error.value.status_code == 501assert calls["count"] == 0++[email protected]("interruption", ["append", "cancel", "publish"])[email protected]("after_seq", [0, 1, 10])+async def test_interrupted_worker_loss_recovery_replays_failure(+ tmp_path, monkeypatch, interruption, after_seq+):+ from deeptutor.runtime.coordination.recovery import TurnRecoveryService++ store = SQLiteSessionStore(tmp_path / "interrupted.db")+ coordinator = MemoryCoordinator()+ session = await store.ensure_session(None)+ turn = await store.begin_turn(session["id"], capability="chat")+ await store.append_events(turn["id"], [{"type": "content", "content": "Partial answer"}])++ async def interrupt(*args, **kwargs):+ if interruption == "cancel":+ raise asyncio.CancelledError+ raise RuntimeError("Injected recovery interruption")++ with monkeypatch.context() as patch:+ if interruption == "publish":+ patch.setattr(coordinator, "publish_event", interrupt)+ else:+ patch.setattr(store, "append_events", interrupt)+ recovery = TurnRecoveryService(coordinator, store)+ if interruption == "cancel":+ with pytest.raises(asyncio.CancelledError):+ await recovery.recover_once()+ else:+ await recovery.recover_once()++ assert (await store.get_turn(turn["id"]))["status"] == "failed"+ # A restarted coordinator cannot rediscover an already terminal row.+ restarted = MemoryCoordinator()+ assert await TurnRecoveryService(restarted, store).recover_once() == 0+ service, _ = _service(store, restarted, "replacement")++ async def collect():+ return [event async for event in service.subscribe_turn(turn["id"], after_seq)]++ events = await asyncio.wait_for(collect(), timeout=2)+ assert [event["type"] for event in events][-2:] == ["error", "done"]+ assert events[-2]["content"]+ assert events[-2]["metadata"]["turn_terminal"] is True+ for event in events[-2:]:+ assert event["metadata"]["error_code"] == "worker_lost"+ assert event["metadata"]["retryable"] is True+ assert event["turn_id"] == turn["id"]+ assert event["seq"] > after_seq+ assert events[-1]["seq"] == events[-2]["seq"] + 1+ if after_seq == 0:+ assert events[0]["content"] == "Partial answer"