Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

fix(runtime): 自动回收工作进程重启后遗留的轮次

AleFlyX9b721a71d259
diff --git a/deeptutor/app/service.py b/deeptutor/app/service.py
index c84f8960e..286bb962b 100644
--- a/deeptutor/app/service.py
+++ b/deeptutor/app/service.py
@@ -472,10 +472,10 @@ class TurnApplicationService:
seconds whether or not it is producing output, so "no lease" is the one
signal that separates a zombie from a slow human.
- Doing it here rather than on a timer is what makes it safe: the only
- rows ever considered are ones already blocking a real request, so
- there is no window in which a just-inserted turn can be swept before
- its first renewal — the hazard that kept the periodic version out.
+ This synchronous path avoids waiting for the background recovery
+ pass when a new request is already blocked. Admission acquires its
+ lease before inserting a turn, so a live new row is protected even
+ before its first renewal.
Returns ``True`` when something was reaped and the caller should retry.
"""
diff --git a/deeptutor/runtime/coordination/recovery.py b/deeptutor/runtime/coordination/recovery.py
index 39989eee0..b2c69c6d6 100644
--- a/deeptutor/runtime/coordination/recovery.py
+++ b/deeptutor/runtime/coordination/recovery.py
@@ -1,4 +1,4 @@
-"""Leader-only recovery of turns whose owner lease has expired."""
+"""Leader-only recovery of turns whose owner lease has been lost."""
from __future__ import annotations
@@ -24,6 +24,18 @@ class TurnRecoveryService:
async def recover_once(self) -> int:
turn_ids = await self.coordinator.list_expired_turn_ids()
+ if self.coordinator.mode == "memory":
+ # A process restart loses the in-memory lease index entirely. The
+ # durable active rows are then the only way to discover old turns.
+ # Admission acquires its lease before inserting the row; the lease
+ # check in _recover_turn protects both new and parked live turns.
+ persisted = await self.repository.list_nonterminal_turns()
+ candidates = set(turn_ids)
+ for turn in persisted:
+ turn_id = str(turn["id"])
+ if await self.coordinator.get_lease(turn_id) is None:
+ candidates.add(turn_id)
+ turn_ids = sorted(candidates)
self.backlog = len(turn_ids)
recovered = 0
for turn_id in turn_ids:
@@ -52,53 +64,63 @@ class TurnRecoveryService:
TurnStatus.WAITING_INPUT.value,
}:
return False
- await self.coordinator.publish_event(
- turn_id,
- {
- "type": "error",
- "source": "turn_recovery",
- "stage": "recovery",
- "content": "The worker executing this turn was lost; regenerate to retry",
- "metadata": {
- "turn_terminal": True,
- "status": TurnStatus.FAILED.value,
- "error_code": TurnFailureCode.WORKER_LOST.value,
- "retryable": True,
- },
- "session_id": turn.get("session_id", ""),
- },
- )
- await self.coordinator.publish_event(
- turn_id,
- {
- "type": "done",
- "source": "turn_recovery",
- "stage": "recovery",
- "content": "",
- "metadata": {
- "status": TurnStatus.FAILED.value,
- "error_code": TurnFailureCode.WORKER_LOST.value,
- "retryable": True,
- },
- "session_id": turn.get("session_id", ""),
- },
- )
- events = await self.coordinator.read_events(turn_id)
- if events:
- await self.repository.append_events(
- turn_id,
- events,
- fencing_token=int(turn.get("fencing_token") or 0),
- )
- return await self.repository.transition_turn(
+ # Recheck at recovery time, not just when collecting expired IDs.
+ # None also prevents acknowledging a candidate whose lease is live.
+ if await self.coordinator.get_lease(turn_id) is not None:
+ return None
+ token = int(turn.get("fencing_token") or 0)
+ # Preserve any events still available from an expired worker before
+ # allocating terminal sequence numbers from the durable journal.
+ pending = await self.coordinator.read_events(turn_id)
+ if pending:
+ await self.repository.append_events(turn_id, pending, fencing_token=token)
+ transitioned = await self.repository.transition_turn(
turn_id,
TurnStatus.FAILED.value,
expected_status=str(turn["status"]),
- fencing_token=int(turn.get("fencing_token") or 0),
+ fencing_token=token,
error="The worker executing this turn was lost; regenerate to retry",
failure_code=TurnFailureCode.WORKER_LOST.value,
retryable=True,
)
+ if not transitioned:
+ return False
+ # Persist first: a fresh coordinator starts at seq=1, which would
+ # collide with pre-crash events and be ignored by resumed clients.
+ events = await self.repository.append_events(
+ turn_id,
+ [
+ {
+ "type": "error",
+ "source": "turn_recovery",
+ "stage": "recovery",
+ "content": "The worker executing this turn was lost; regenerate to retry",
+ "metadata": {
+ "turn_terminal": True,
+ "status": TurnStatus.FAILED.value,
+ "error_code": TurnFailureCode.WORKER_LOST.value,
+ "retryable": True,
+ },
+ "session_id": turn.get("session_id", ""),
+ },
+ {
+ "type": "done",
+ "source": "turn_recovery",
+ "stage": "recovery",
+ "content": "",
+ "metadata": {
+ "status": TurnStatus.FAILED.value,
+ "error_code": TurnFailureCode.WORKER_LOST.value,
+ "retryable": True,
+ },
+ "session_id": turn.get("session_id", ""),
+ },
+ ],
+ fencing_token=token,
+ )
+ for event in events:
+ await self.coordinator.publish_event(turn_id, event)
+ return True
__all__ = ["TurnRecoveryService"]
diff --git a/tests/runtime/coordination/test_journal_recovery.py b/tests/runtime/coordination/test_journal_recovery.py
index 6e52b99ae..a8f23bc4e 100644
--- a/tests/runtime/coordination/test_journal_recovery.py
+++ b/tests/runtime/coordination/test_journal_recovery.py
@@ -1,6 +1,7 @@
from __future__ import annotations
import asyncio
+from unittest.mock import AsyncMock
import pytest
@@ -69,6 +70,121 @@ async def test_recovery_preserves_unflushed_events_and_marks_retryable_failure(t
assert await coordinator.list_expired_turn_ids() == []
[email protected]("status", ["queued", "running", "waiting_input"])
+async def test_restart_recovers_durable_turn_without_new_submission(tmp_path, status) -> None:
+ repository = SQLiteSessionStore(tmp_path / "turns.db")
+ session = await repository.ensure_session(None)
+ old_coordinator = MemoryCoordinator()
+ lease = await old_coordinator.acquire_turn("interrupted", session["id"], "old-worker")
+ await repository.begin_turn(
+ session["id"],
+ "chat",
+ turn_id=lease.turn_id,
+ owner_id=lease.owner_id,
+ fencing_token=lease.fencing_token,
+ )
+ await repository.transition_turn(lease.turn_id, status)
+ await repository.append_events(
+ lease.turn_id,
+ [
+ {"type": "session", "content": ""},
+ {"type": "content", "content": "partial answer"},
+ ],
+ )
+ original = await repository.get_events(lease.turn_id)
+ # A new process has neither the old lease index nor its event stream.
+ coordinator = MemoryCoordinator()
+ assert await coordinator.list_expired_turn_ids() == []
+ recovery = TurnRecoveryService(coordinator, repository)
+
+ assert await recovery.recover_once() == 1
+ turn = await repository.get_turn(lease.turn_id)
+ assert turn["status"] == "failed"
+ assert turn["failure_code"] == "worker_lost"
+ assert turn["retryable"] is True
+ events = await repository.get_events(lease.turn_id)
+ assert events[:2] == original
+ assert [e["seq"] for e in events] == [1, 2, 3, 4]
+ assert [e["type"] for e in events] == ["session", "content", "error", "done"]
+ assert events[-1]["metadata"]["status"] == "failed"
+ assert await recovery.recover_once() == 0
+ assert await repository.get_events(lease.turn_id) == events
+ # Recovery unblocks admission without a user-triggered reclaim path.
+ assert (await repository.begin_turn(session["id"], "chat"))["id"] != lease.turn_id
+
+
[email protected]("status", ["queued", "running", "waiting_input"])
+async def test_durable_scan_preserves_live_turns(tmp_path, status) -> None:
+ repository = SQLiteSessionStore(tmp_path / "turns.db")
+ session = await repository.ensure_session(None)
+ coordinator = MemoryCoordinator()
+ lease = await coordinator.acquire_turn("live", session["id"], "worker")
+ await repository.begin_turn(
+ session["id"],
+ "chat",
+ turn_id=lease.turn_id,
+ owner_id=lease.owner_id,
+ fencing_token=lease.fencing_token,
+ )
+ await repository.transition_turn(lease.turn_id, status)
+ assert await TurnRecoveryService(coordinator, repository).recover_once() == 0
+ assert (await repository.get_turn(lease.turn_id))["status"] == status
+ assert await coordinator.get_lease(lease.turn_id) == lease
+ assert await repository.get_events(lease.turn_id) == []
+
+
+async def test_recovery_rechecks_lease_after_expired_scan(tmp_path, monkeypatch) -> None:
+ repository = SQLiteSessionStore(tmp_path / "turns.db")
+ session = await repository.ensure_session(None)
+ coordinator = MemoryCoordinator()
+ lease = await coordinator.acquire_turn("live", session["id"], "worker")
+ await repository.begin_turn(
+ session["id"],
+ "chat",
+ turn_id=lease.turn_id,
+ owner_id=lease.owner_id,
+ fencing_token=lease.fencing_token,
+ )
+ monkeypatch.setattr(coordinator, "list_expired_turn_ids", AsyncMock(return_value=["live"]))
+ assert await TurnRecoveryService(coordinator, repository).recover_once() == 0
+ assert (await repository.get_turn("live"))["status"] != "failed"
+ assert await repository.get_events("live") == []
+
+
+async def test_recovery_does_not_publish_failure_if_terminal_transition_loses(
+ tmp_path, monkeypatch
+):
+ repository = SQLiteSessionStore(tmp_path / "turns.db")
+ session = await repository.ensure_session(None)
+ coordinator = MemoryCoordinator()
+ lease = await coordinator.acquire_turn("finished", session["id"], "worker")
+ await repository.begin_turn(
+ session["id"],
+ "chat",
+ turn_id=lease.turn_id,
+ owner_id=lease.owner_id,
+ fencing_token=lease.fencing_token,
+ )
+ await coordinator.release_turn(lease)
+ transition = repository.transition_turn
+
+ async def complete_first(*args, **kwargs):
+ await transition("finished", "completed")
+ return False
+
+ monkeypatch.setattr(repository, "transition_turn", complete_first)
+ publish = AsyncMock(wraps=coordinator.publish_event)
+ monkeypatch.setattr(coordinator, "publish_event", publish)
+ assert await TurnRecoveryService(coordinator, repository).recover_once() == 0
+ assert (await repository.get_turn("finished"))["status"] == "completed"
+ assert await repository.get_events("finished") == []
+ publish.assert_not_awaited()
+
+
@pytest.mark.asyncio
async def test_wrong_user_repository_does_not_acknowledge_expired_turn(tmp_path) -> None:
owner_repository = SQLiteSessionStore(tmp_path / "owner.db")
diff --git a/tests/runtime/coordination/test_redis_integration.py b/tests/runtime/coordination/test_redis_integration.py
index d4d6602cc..b19062a1b 100644
--- a/tests/runtime/coordination/test_redis_integration.py
+++ b/tests/runtime/coordination/test_redis_integration.py
@@ -1,5 +1,6 @@
from __future__ import annotations
+import asyncio
import os
import uuid
@@ -7,6 +8,8 @@ import pytest
import pytest_asyncio
from deeptutor.runtime.coordination import RedisCoordinator
+from deeptutor.runtime.coordination.recovery import TurnRecoveryService
+from deeptutor.services.session.sqlite_store import SQLiteSessionStore
pytestmark = [pytest.mark.asyncio, pytest.mark.redis_integration]
@@ -103,3 +106,34 @@ async def test_two_redis_coordinators_share_turn_and_background_streams(coordina
assert await coordinator.release_turn(lease) is True
finally:
await second.close()
+
+
+async def test_recovery_preserves_shared_events_and_live_owner(coordinator, tmp_path) -> None:
+ store = SQLiteSessionStore(tmp_path / "recovery.db")
+ session = await store.ensure_session(None)
+ lease = await coordinator.acquire_turn("recover-me", session["id"], "worker")
+ assert lease is not None
+ await store.begin_turn(
+ session["id"],
+ "chat",
+ turn_id=lease.turn_id,
+ owner_id=lease.owner_id,
+ fencing_token=lease.fencing_token,
+ )
+ content = await coordinator.publish_event(lease.turn_id, {"type": "content", "content": "kept"})
+ recovery = TurnRecoveryService(coordinator, store)
+ assert await recovery.recover_once() == 0
+ assert (await store.get_turn(lease.turn_id))["status"] == "running"
+ # Simulate lease expiry, not a graceful release (which removes the
+ # expired-candidate index). Keep the shared event stream intact.
+ await coordinator.client.pexpire(coordinator._key("lease", "turn", lease.turn_id), 1)
+ await coordinator.client.zadd(coordinator._key("leases", "turns"), {lease.turn_id: 0})
+ await asyncio.sleep(0.02)
+ assert await recovery.recover_once() == 1
+ events = await store.get_events(lease.turn_id)
+ assert [event["type"] for event in events] == ["content", "error", "done"]
+ assert [event["seq"] for event in events] == [1, 2, 3]
+ assert events[0]["content"] == content["content"]
+ assert [event["seq"] for event in await coordinator.read_events(lease.turn_id)] == [1, 2, 3]
+ assert (await store.get_turn(lease.turn_id))["failure_code"] == "worker_lost"
+ assert await recovery.recover_once() == 0