Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

fix(runtime): 补全恢复中断后的失败事件回放

AleFlyXa9d6c99d89a8
diff --git a/deeptutor/app/service.py b/deeptutor/app/service.py
index 286bb962b..d9e18316f 100644
--- a/deeptutor/app/service.py
+++ b/deeptutor/app/service.py
@@ -321,13 +321,29 @@ class TurnApplicationService:
# Ownership/cleanup can change between the reads above.
if await store.get_events(turn_id, last_seq):
continue
- # Only legacy terminal rows with no journal need synthesis.
+ # Recovery can commit FAILED before its terminal journal is
+ # written. A client may also have seen volatile events beyond
+ # the durable cursor. In either case preserve the failure
+ # envelope: DONE alone hides errors on partial responses.
metadata = {
"status": status,
"synthesized": True,
"error_code": str(turn.get("failure_code") or ""),
"retryable": bool(turn.get("retryable")),
}
+ if status == "failed":
+ last_seq += 1
+ yield {
+ "type": "error",
+ "source": "turn_application",
+ "stage": "",
+ "content": str(turn.get("error") or "Turn failed; retry to continue"),
+ "metadata": {**metadata, "turn_terminal": True},
+ "session_id": turn.get("session_id", ""),
+ "turn_id": turn_id,
+ "seq": last_seq,
+ "timestamp": time.time(),
+ }
yield {
"type": "done",
"source": "turn_application",
diff --git a/tests/app/test_turn_application_service.py b/tests/app/test_turn_application_service.py
index 575ab11ae..2d2e89e98 100644
--- a/tests/app/test_turn_application_service.py
+++ b/tests/app/test_turn_application_service.py
@@ -540,3 +540,57 @@ async def test_replacement_availability_matches_configured_store_before_mutation
await sessions.replace_submission(body)
assert error.value.status_code == 501
assert calls["count"] == 0
+
+
[email protected]("interruption", ["append", "cancel", "publish"])
[email protected]("after_seq", [0, 1, 10])
+async def test_interrupted_worker_loss_recovery_replays_failure(
+ tmp_path, monkeypatch, interruption, after_seq
+):
+ from deeptutor.runtime.coordination.recovery import TurnRecoveryService
+
+ store = SQLiteSessionStore(tmp_path / "interrupted.db")
+ coordinator = MemoryCoordinator()
+ session = await store.ensure_session(None)
+ turn = await store.begin_turn(session["id"], capability="chat")
+ await store.append_events(turn["id"], [{"type": "content", "content": "Partial answer"}])
+
+ async def interrupt(*args, **kwargs):
+ if interruption == "cancel":
+ raise asyncio.CancelledError
+ raise RuntimeError("Injected recovery interruption")
+
+ with monkeypatch.context() as patch:
+ if interruption == "publish":
+ patch.setattr(coordinator, "publish_event", interrupt)
+ else:
+ patch.setattr(store, "append_events", interrupt)
+ recovery = TurnRecoveryService(coordinator, store)
+ if interruption == "cancel":
+ with pytest.raises(asyncio.CancelledError):
+ await recovery.recover_once()
+ else:
+ await recovery.recover_once()
+
+ assert (await store.get_turn(turn["id"]))["status"] == "failed"
+ # A restarted coordinator cannot rediscover an already terminal row.
+ restarted = MemoryCoordinator()
+ assert await TurnRecoveryService(restarted, store).recover_once() == 0
+ service, _ = _service(store, restarted, "replacement")
+
+ async def collect():
+ return [event async for event in service.subscribe_turn(turn["id"], after_seq)]
+
+ events = await asyncio.wait_for(collect(), timeout=2)
+ assert [event["type"] for event in events][-2:] == ["error", "done"]
+ assert events[-2]["content"]
+ assert events[-2]["metadata"]["turn_terminal"] is True
+ for event in events[-2:]:
+ assert event["metadata"]["error_code"] == "worker_lost"
+ assert event["metadata"]["retryable"] is True
+ assert event["turn_id"] == turn["id"]
+ assert event["seq"] > after_seq
+ assert events[-1]["seq"] == events[-2]["seq"] + 1
+ if after_seq == 0:
+ assert events[0]["content"] == "Partial answer"