DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.
Python ai-agentsai-tutorclawbotcli-tooldeepresearchinteractive-learninglearningmulti-agent-systemsrag
fix(runtime): 自动回收工作进程重启后遗留的轮次
AleFlyX9b721a71d259
diff --git a/deeptutor/app/service.py b/deeptutor/app/service.pyindex c84f8960e..286bb962b 100644--- a/deeptutor/app/service.py+++ b/deeptutor/app/service.py@@ -472,10 +472,10 @@ class TurnApplicationService:seconds whether or not it is producing output, so "no lease" is the onesignal that separates a zombie from a slow human.- Doing it here rather than on a timer is what makes it safe: the only- rows ever considered are ones already blocking a real request, so- there is no window in which a just-inserted turn can be swept before- its first renewal — the hazard that kept the periodic version out.+ This synchronous path avoids waiting for the background recovery+ pass when a new request is already blocked. Admission acquires its+ lease before inserting a turn, so a live new row is protected even+ before its first renewal.Returns ``True`` when something was reaped and the caller should retry."""diff --git a/deeptutor/runtime/coordination/recovery.py b/deeptutor/runtime/coordination/recovery.pyindex 39989eee0..b2c69c6d6 100644--- a/deeptutor/runtime/coordination/recovery.py+++ b/deeptutor/runtime/coordination/recovery.py@@ -1,4 +1,4 @@-"""Leader-only recovery of turns whose owner lease has expired."""+"""Leader-only recovery of turns whose owner lease has been lost."""from __future__ import annotations@@ -24,6 +24,18 @@ class TurnRecoveryService:async def recover_once(self) -> int:turn_ids = await self.coordinator.list_expired_turn_ids()+ if self.coordinator.mode == "memory":+ # A process restart loses the in-memory lease index entirely. The+ # durable active rows are then the only way to discover old turns.+ # Admission acquires its lease before inserting the row; the lease+ # check in _recover_turn protects both new and parked live turns.+ persisted = await self.repository.list_nonterminal_turns()+ candidates = set(turn_ids)+ for turn in persisted:+ turn_id = str(turn["id"])+ if await self.coordinator.get_lease(turn_id) is None:+ candidates.add(turn_id)+ turn_ids = sorted(candidates)self.backlog = len(turn_ids)recovered = 0for turn_id in turn_ids:@@ -52,53 +64,63 @@ class TurnRecoveryService:TurnStatus.WAITING_INPUT.value,}:return False- await self.coordinator.publish_event(- turn_id,- {- "type": "error",- "source": "turn_recovery",- "stage": "recovery",- "content": "The worker executing this turn was lost; regenerate to retry",- "metadata": {- "turn_terminal": True,- "status": TurnStatus.FAILED.value,- "error_code": TurnFailureCode.WORKER_LOST.value,- "retryable": True,- },- "session_id": turn.get("session_id", ""),- },- )- await self.coordinator.publish_event(- turn_id,- {- "type": "done",- "source": "turn_recovery",- "stage": "recovery",- "content": "",- "metadata": {- "status": TurnStatus.FAILED.value,- "error_code": TurnFailureCode.WORKER_LOST.value,- "retryable": True,- },- "session_id": turn.get("session_id", ""),- },- )- events = await self.coordinator.read_events(turn_id)- if events:- await self.repository.append_events(- turn_id,- events,- fencing_token=int(turn.get("fencing_token") or 0),- )- return await self.repository.transition_turn(+ # Recheck at recovery time, not just when collecting expired IDs.+ # None also prevents acknowledging a candidate whose lease is live.+ if await self.coordinator.get_lease(turn_id) is not None:+ return None+ token = int(turn.get("fencing_token") or 0)+ # Preserve any events still available from an expired worker before+ # allocating terminal sequence numbers from the durable journal.+ pending = await self.coordinator.read_events(turn_id)+ if pending:+ await self.repository.append_events(turn_id, pending, fencing_token=token)+ transitioned = await self.repository.transition_turn(turn_id,TurnStatus.FAILED.value,expected_status=str(turn["status"]),- fencing_token=int(turn.get("fencing_token") or 0),+ fencing_token=token,error="The worker executing this turn was lost; regenerate to retry",failure_code=TurnFailureCode.WORKER_LOST.value,retryable=True,)+ if not transitioned:+ return False+ # Persist first: a fresh coordinator starts at seq=1, which would+ # collide with pre-crash events and be ignored by resumed clients.+ events = await self.repository.append_events(+ turn_id,+ [+ {+ "type": "error",+ "source": "turn_recovery",+ "stage": "recovery",+ "content": "The worker executing this turn was lost; regenerate to retry",+ "metadata": {+ "turn_terminal": True,+ "status": TurnStatus.FAILED.value,+ "error_code": TurnFailureCode.WORKER_LOST.value,+ "retryable": True,+ },+ "session_id": turn.get("session_id", ""),+ },+ {+ "type": "done",+ "source": "turn_recovery",+ "stage": "recovery",+ "content": "",+ "metadata": {+ "status": TurnStatus.FAILED.value,+ "error_code": TurnFailureCode.WORKER_LOST.value,+ "retryable": True,+ },+ "session_id": turn.get("session_id", ""),+ },+ ],+ fencing_token=token,+ )+ for event in events:+ await self.coordinator.publish_event(turn_id, event)+ return True__all__ = ["TurnRecoveryService"]diff --git a/tests/runtime/coordination/test_journal_recovery.py b/tests/runtime/coordination/test_journal_recovery.pyindex 6e52b99ae..a8f23bc4e 100644--- a/tests/runtime/coordination/test_journal_recovery.py+++ b/tests/runtime/coordination/test_journal_recovery.py@@ -1,6 +1,7 @@from __future__ import annotationsimport asyncio+from unittest.mock import AsyncMockimport pytest@@ -69,6 +70,121 @@ async def test_recovery_preserves_unflushed_events_and_marks_retryable_failure(tassert await coordinator.list_expired_turn_ids() == [][email protected]("status", ["queued", "running", "waiting_input"])+async def test_restart_recovers_durable_turn_without_new_submission(tmp_path, status) -> None:+ repository = SQLiteSessionStore(tmp_path / "turns.db")+ session = await repository.ensure_session(None)+ old_coordinator = MemoryCoordinator()+ lease = await old_coordinator.acquire_turn("interrupted", session["id"], "old-worker")+ await repository.begin_turn(+ session["id"],+ "chat",+ turn_id=lease.turn_id,+ owner_id=lease.owner_id,+ fencing_token=lease.fencing_token,+ )+ await repository.transition_turn(lease.turn_id, status)+ await repository.append_events(+ lease.turn_id,+ [+ {"type": "session", "content": ""},+ {"type": "content", "content": "partial answer"},+ ],+ )+ original = await repository.get_events(lease.turn_id)+ # A new process has neither the old lease index nor its event stream.+ coordinator = MemoryCoordinator()+ assert await coordinator.list_expired_turn_ids() == []+ recovery = TurnRecoveryService(coordinator, repository)++ assert await recovery.recover_once() == 1+ turn = await repository.get_turn(lease.turn_id)+ assert turn["status"] == "failed"+ assert turn["failure_code"] == "worker_lost"+ assert turn["retryable"] is True+ events = await repository.get_events(lease.turn_id)+ assert events[:2] == original+ assert [e["seq"] for e in events] == [1, 2, 3, 4]+ assert [e["type"] for e in events] == ["session", "content", "error", "done"]+ assert events[-1]["metadata"]["status"] == "failed"+ assert await recovery.recover_once() == 0+ assert await repository.get_events(lease.turn_id) == events+ # Recovery unblocks admission without a user-triggered reclaim path.+ assert (await repository.begin_turn(session["id"], "chat"))["id"] != lease.turn_id++[email protected]("status", ["queued", "running", "waiting_input"])+async def test_durable_scan_preserves_live_turns(tmp_path, status) -> None:+ repository = SQLiteSessionStore(tmp_path / "turns.db")+ session = await repository.ensure_session(None)+ coordinator = MemoryCoordinator()+ lease = await coordinator.acquire_turn("live", session["id"], "worker")+ await repository.begin_turn(+ session["id"],+ "chat",+ turn_id=lease.turn_id,+ owner_id=lease.owner_id,+ fencing_token=lease.fencing_token,+ )+ await repository.transition_turn(lease.turn_id, status)+ assert await TurnRecoveryService(coordinator, repository).recover_once() == 0+ assert (await repository.get_turn(lease.turn_id))["status"] == status+ assert await coordinator.get_lease(lease.turn_id) == lease+ assert await repository.get_events(lease.turn_id) == []+++async def test_recovery_rechecks_lease_after_expired_scan(tmp_path, monkeypatch) -> None:+ repository = SQLiteSessionStore(tmp_path / "turns.db")+ session = await repository.ensure_session(None)+ coordinator = MemoryCoordinator()+ lease = await coordinator.acquire_turn("live", session["id"], "worker")+ await repository.begin_turn(+ session["id"],+ "chat",+ turn_id=lease.turn_id,+ owner_id=lease.owner_id,+ fencing_token=lease.fencing_token,+ )+ monkeypatch.setattr(coordinator, "list_expired_turn_ids", AsyncMock(return_value=["live"]))+ assert await TurnRecoveryService(coordinator, repository).recover_once() == 0+ assert (await repository.get_turn("live"))["status"] != "failed"+ assert await repository.get_events("live") == []+++async def test_recovery_does_not_publish_failure_if_terminal_transition_loses(+ tmp_path, monkeypatch+):+ repository = SQLiteSessionStore(tmp_path / "turns.db")+ session = await repository.ensure_session(None)+ coordinator = MemoryCoordinator()+ lease = await coordinator.acquire_turn("finished", session["id"], "worker")+ await repository.begin_turn(+ session["id"],+ "chat",+ turn_id=lease.turn_id,+ owner_id=lease.owner_id,+ fencing_token=lease.fencing_token,+ )+ await coordinator.release_turn(lease)+ transition = repository.transition_turn++ async def complete_first(*args, **kwargs):+ await transition("finished", "completed")+ return False++ monkeypatch.setattr(repository, "transition_turn", complete_first)+ publish = AsyncMock(wraps=coordinator.publish_event)+ monkeypatch.setattr(coordinator, "publish_event", publish)+ assert await TurnRecoveryService(coordinator, repository).recover_once() == 0+ assert (await repository.get_turn("finished"))["status"] == "completed"+ assert await repository.get_events("finished") == []+ publish.assert_not_awaited()++@pytest.mark.asyncioasync def test_wrong_user_repository_does_not_acknowledge_expired_turn(tmp_path) -> None:owner_repository = SQLiteSessionStore(tmp_path / "owner.db")diff --git a/tests/runtime/coordination/test_redis_integration.py b/tests/runtime/coordination/test_redis_integration.pyindex d4d6602cc..b19062a1b 100644--- a/tests/runtime/coordination/test_redis_integration.py+++ b/tests/runtime/coordination/test_redis_integration.py@@ -1,5 +1,6 @@from __future__ import annotations+import asyncioimport osimport uuid@@ -7,6 +8,8 @@ import pytestimport pytest_asynciofrom deeptutor.runtime.coordination import RedisCoordinator+from deeptutor.runtime.coordination.recovery import TurnRecoveryService+from deeptutor.services.session.sqlite_store import SQLiteSessionStorepytestmark = [pytest.mark.asyncio, pytest.mark.redis_integration]@@ -103,3 +106,34 @@ async def test_two_redis_coordinators_share_turn_and_background_streams(coordinaassert await coordinator.release_turn(lease) is Truefinally:await second.close()+++async def test_recovery_preserves_shared_events_and_live_owner(coordinator, tmp_path) -> None:+ store = SQLiteSessionStore(tmp_path / "recovery.db")+ session = await store.ensure_session(None)+ lease = await coordinator.acquire_turn("recover-me", session["id"], "worker")+ assert lease is not None+ await store.begin_turn(+ session["id"],+ "chat",+ turn_id=lease.turn_id,+ owner_id=lease.owner_id,+ fencing_token=lease.fencing_token,+ )+ content = await coordinator.publish_event(lease.turn_id, {"type": "content", "content": "kept"})+ recovery = TurnRecoveryService(coordinator, store)+ assert await recovery.recover_once() == 0+ assert (await store.get_turn(lease.turn_id))["status"] == "running"+ # Simulate lease expiry, not a graceful release (which removes the+ # expired-candidate index). Keep the shared event stream intact.+ await coordinator.client.pexpire(coordinator._key("lease", "turn", lease.turn_id), 1)+ await coordinator.client.zadd(coordinator._key("leases", "turns"), {lease.turn_id: 0})+ await asyncio.sleep(0.02)+ assert await recovery.recover_once() == 1+ events = await store.get_events(lease.turn_id)+ assert [event["type"] for event in events] == ["content", "error", "done"]+ assert [event["seq"] for event in events] == [1, 2, 3]+ assert events[0]["content"] == content["content"]+ assert [event["seq"] for event in await coordinator.read_events(lease.turn_id)] == [1, 2, 3]+ assert (await store.get_turn(lease.turn_id))["failure_code"] == "worker_lost"+ assert await recovery.recover_once() == 0