DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.
Python ai-agentsai-tutorclawbotcli-tooldeepresearchinteractive-learninglearningmulti-agent-systemsrag
fix(runtime): persist live prefixes and reclaim completed streams (#1971)
Bingxi Zhao (Frank)e28c8036361c
diff --git a/deeptutor/app/service.py b/deeptutor/app/service.pyindex 268537cbf..c84f8960e 100644--- a/deeptutor/app/service.py+++ b/deeptutor/app/service.py@@ -268,20 +268,20 @@ class TurnApplicationService:done = Truetail_possible = self._done_has_tail(event)yield event- if done:- # The turn had already finished before this subscription opened, so- # the durable replay above carried its post-DONE events too. Only- # bail out once the terminal row confirms that; a turn still marked- # running with a DONE in its journal is mid-teardown and its title- # may not be written yet.- turn = await store.get_turn(turn_id)- if not tail_possible or turn is None or str(turn.get("status") or "") != "running":- return+ if done and not tail_possible:+ return+ # A completed row can still have an owned title tail. Drain it below,+ # including durable events committed after the initial replay (#1971).first_done_at: float | None = time.monotonic() if done else Nonewhile True:emitted = False- for event in await self.coordinator.read_events(turn_id, last_seq):+ events = await self.coordinator.read_events(turn_id, last_seq)+ if not events and await self.coordinator.get_lease(turn_id) is None:+ # The completed in-memory stream may have been reclaimed+ # while this subscriber was paused. Its durable tail remains.+ events = await store.get_events(turn_id, last_seq)+ for event in events:seq = int(event.get("seq") or 0)if seq <= last_seq:continue@@ -313,9 +313,15 @@ class TurnApplicationService:returnstatus = str(turn.get("status") or "")if status in {"completed", "failed", "cancelled"}:- # A terminal row without DONE can only be legacy data. Keep the- # compatibility envelope stable while all new paths journal- # DONE before the terminal CAS.+ # Completion can commit before the owner publishes DONE. Do+ # not synthesize an id-less envelope in that window (#1971).+ if await self.coordinator.get_lease(turn_id) is not None:+ await asyncio.sleep(0.1)+ continue+ # Ownership/cleanup can change between the reads above.+ if await store.get_events(turn_id, last_seq):+ continue+ # Only legacy terminal rows with no journal need synthesis.metadata = {"status": status,"synthesized": True,diff --git a/deeptutor/runtime/coordination/memory.py b/deeptutor/runtime/coordination/memory.pyindex 25e0ff97e..be4d6d610 100644--- a/deeptutor/runtime/coordination/memory.py+++ b/deeptutor/runtime/coordination/memory.py@@ -97,8 +97,15 @@ class MemoryCoordinator:async def acknowledge_expired_turn(self, turn_id: str) -> None:async with self._lock:- self._known_turns.discard(turn_id)lease = self._turn_leases.get(turn_id)+ if self._active(lease):+ return+ self._known_turns.discard(turn_id)+ # Recovery has verified the durable terminal row. Keeping these+ # payloads forever leaks every completed conversation (#1971).+ self._events.pop(turn_id, None)+ for _, command in self._commands.pop(turn_id, []):+ self._command_ids.discard(command.command_id)if lease is not None and not self._active(lease):self._turn_leases.pop(turn_id, None)if self._session_turns.get(lease.session_id) == turn_id:diff --git a/deeptutor/services/session/_turn_runtime_shared.py b/deeptutor/services/session/_turn_runtime_shared.pyindex 3250cabe9..f7aed022d 100644--- a/deeptutor/services/session/_turn_runtime_shared.py+++ b/deeptutor/services/session/_turn_runtime_shared.py@@ -1575,6 +1575,7 @@ class _TurnExecution:events: list[dict[str, Any]] = field(default_factory=list)next_seq: int = 1flush_lock: asyncio.Lock = field(default_factory=asyncio.Lock)+ flush_task: asyncio.Task[None] | None = Noneevents_persisted: bool = Falsepersisted_events: list[dict[str, Any]] = field(default_factory=list)events_flushed: bool = Falsediff --git a/deeptutor/services/session/turns/executor.py b/deeptutor/services/session/turns/executor.pyindex 3417c0ab0..639a0a34d 100644--- a/deeptutor/services/session/turns/executor.py+++ b/deeptutor/services/session/turns/executor.py@@ -1404,6 +1404,10 @@ class TurnExecutor:retryable=resolved_retryable,)finally:+ if execution.flush_task is not None:+ execution.flush_task.cancel()+ with contextlib.suppress(asyncio.CancelledError):+ await execution.flush_taskresource_scope.close()if llm_scope_token is not None and reset_active_llm_selection is not None:reset_active_llm_selection(llm_scope_token)diff --git a/deeptutor/services/session/turns/lifecycle.py b/deeptutor/services/session/turns/lifecycle.pyindex a5348e314..419f5d518 100644--- a/deeptutor/services/session/turns/lifecycle.py+++ b/deeptutor/services/session/turns/lifecycle.py@@ -680,18 +680,41 @@ class TurnLifecycle:if current is not execution:execution.events.append(payload)subscribers = list(current.subscribers)- if event.type == StreamEventType.DONE:+ if event.type == StreamEventType.DONE or (+ len(execution.events) - len(execution.persisted_events) >= 25+ ):# Never expose DONE to a process-local subscriber before the# complete event prefix is durable. Redis subscribers already# read the same canonical payload from the shared journal.await self._flush_buffered_events(execution)+ elif execution.flush_task is None or execution.flush_task.done():+ execution.flush_task = asyncio.create_task(self._flush_live_prefix(execution))for subscriber in subscribers:with contextlib.suppress(asyncio.QueueFull):subscriber.queue.put_nowait(payload)return payload+ async def _flush_live_prefix(self, execution: _TurnExecution) -> None:+ """Bound the crash-loss window even when the next model/tool is quiet."""+ delay = 0.25+ while True:+ await asyncio.sleep(delay)+ if execution.lease_lost:+ return+ try:+ await self._flush_buffered_events(execution)+ except Exception:+ # Keep the pending prefix and retry transient storage errors.+ # Terminal flushes still propagate failure to the turn owner.+ logger.exception("Live event persistence failed for turn %s", execution.turn_id)+ delay = min(delay * 2, 5.0)+ continue+ if len(execution.persisted_events) == len(execution.events):+ return+ delay = 0.25+async def _flush_buffered_events(self, execution: _TurnExecution) -> None:- """Persist buffered turn events after the live stream has already drained."""+ """Persist the pending prefix, serialized with periodic and terminal writes."""async with execution.flush_lock:await self._flush_buffered_events_once(execution)diff --git a/tests/app/test_turn_application_service.py b/tests/app/test_turn_application_service.pyindex 60bf74db0..575ab11ae 100644--- a/tests/app/test_turn_application_service.py+++ b/tests/app/test_turn_application_service.py@@ -86,6 +86,126 @@ def submission_service(monkeypatch, tmp_path):return service, store, coordinator, registry, calls+async def test_subscriber_waits_for_owned_terminal_envelope_and_replays_after_eviction(tmp_path):+ """A slow subscriber must get the real DONE ids even across cleanup (#1971)."""+ store = SQLiteSessionStore(tmp_path / "turns.db")+ coordinator = MemoryCoordinator()+ service, _ = _service(store, coordinator, "worker")+ session = await store.ensure_session(None)+ lease = await coordinator.acquire_turn("owned-terminal", session["id"], "worker")+ assert lease is not None+ await store.begin_turn(+ session["id"],+ turn_id=lease.turn_id,+ owner_id=lease.owner_id,+ fencing_token=lease.fencing_token,+ )+ await store.transition_turn(+ lease.turn_id, "completed", expected_status="running", fencing_token=lease.fencing_token+ )+ subscription = service.subscribe_turn(lease.turn_id)+ waiting = asyncio.create_task(anext(subscription))+ try:+ await asyncio.sleep(0.15)+ assert not waiting.done(), "live owner has not published the real DONE yet"+ done = await coordinator.publish_event(+ lease.turn_id,+ {+ "type": "done",+ "metadata": {"status": "completed", "assistant_message_id": "answer"},+ },+ )+ await store.append_events(lease.turn_id, [done], fencing_token=lease.fencing_token)+ await coordinator.release_turn(lease)+ await coordinator.acknowledge_expired_turn(lease.turn_id)+ received = await asyncio.wait_for(waiting, 2)+ assert received["metadata"]["assistant_message_id"] == "answer"+ assert not received["metadata"].get("synthesized")+ finally:+ waiting.cancel()+ await asyncio.gather(waiting, return_exceptions=True)+ await subscription.aclose()+++async def test_running_turn_flushes_visible_prefix_without_waiting_for_more_events(+ submission_service,+ monkeypatch,+):+ """A quiet model/tool must not leave already visible work only in RAM (#1971)."""+ service, store, coordinator, registry, _ = submission_service+ streamed = asyncio.Event()+ finish = asyncio.Event()++ class SlowOrchestrator:+ async def handle(self, context):+ yield StreamEvent(type=StreamEventType.CONTENT, content="visible prefix", source="chat")+ streamed.set()+ await finish.wait()+ yield StreamEvent(type=StreamEventType.DONE, source="chat")++ monkeypatch.setattr("deeptutor.runtime.orchestrator.ChatOrchestrator", SlowOrchestrator)+ _, turn = await service.start_turn({"content": "Hello"})+ try:+ await asyncio.wait_for(streamed.wait(), 3)+ await asyncio.sleep(0.6)+ restarted = SQLiteSessionStore(store.db_path)+ events = await restarted.get_events(turn["id"])+ assert any(e.get("content") == "visible prefix" for e in events)+ assert (await restarted.get_turn(turn["id"]))["status"] == "running"+ finally:+ finish.set()+ [event async for event in service.subscribe_turn(turn["id"])]+++async def test_reconnect_after_done_keeps_the_live_post_turn_title(tmp_path):+ store = SQLiteSessionStore(tmp_path / "turns.db")+ coordinator = MemoryCoordinator()+ service, _ = _service(store, coordinator, "worker")+ session = await store.ensure_session(None)+ lease = await coordinator.acquire_turn("title-tail", session["id"], "worker")+ await store.begin_turn(+ session["id"],+ turn_id=lease.turn_id,+ owner_id=lease.owner_id,+ fencing_token=lease.fencing_token,+ )+ done = await coordinator.publish_event(+ lease.turn_id,+ {+ "type": "done",+ "metadata": {"status": "completed"},+ },+ )+ await store.append_events(lease.turn_id, [done], fencing_token=lease.fencing_token)+ await store.transition_turn(+ lease.turn_id, "completed", expected_status="running", fencing_token=lease.fencing_token+ )+ stream = service.subscribe_turn(lease.turn_id)+ assert (await anext(stream))["type"] == "done"+ waiting = asyncio.create_task(anext(stream))+ try:+ await asyncio.sleep(0.05)+ assert not waiting.done(), "terminal status is not the end of the owned title tail"+ title = await coordinator.publish_event(+ lease.turn_id,+ {+ "type": "session_meta",+ "metadata": {"title": "Cell membrane"},+ },+ )+ await store.append_events(lease.turn_id, [title], fencing_token=lease.fencing_token)+ await coordinator.release_turn(lease)+ await coordinator.acknowledge_expired_turn(lease.turn_id)+ assert (await asyncio.wait_for(waiting, 2))["metadata"]["title"] == "Cell membrane"+ finally:+ waiting.cancel()+ await asyncio.gather(waiting, return_exceptions=True)+ await stream.aclose()++@pytest.mark.asyncioasync def test_replacement_endpoint_workflow_starts_once_and_branches_admitted_user(submission_service,diff --git a/tests/runtime/coordination/test_memory_coordinator.py b/tests/runtime/coordination/test_memory_coordinator.pyindex a2c8410be..2705b8cda 100644--- a/tests/runtime/coordination/test_memory_coordinator.py+++ b/tests/runtime/coordination/test_memory_coordinator.py@@ -133,3 +133,21 @@ async def test_background_leader_is_singleton_and_transferable() -> None:successor = await coordinator.acquire_leader("worker-b")assert successor is not Noneassert successor.fencing_token > leader.fencing_token+++async def test_acknowledged_turn_releases_payloads_but_live_owner_keeps_them() -> None:+ coordinator = MemoryCoordinator()+ lease = await coordinator.acquire_turn("turn-a", "session-a", "worker")+ assert lease is not None+ await coordinator.publish_event("turn-a", {"type": "content", "content": "large answer"})+ await coordinator.submit_command("turn-a", "cancel", command_id="cancel-a")+ await coordinator.acknowledge_expired_turn("turn-a")+ assert len(await coordinator.read_events("turn-a")) == 1+ assert len(await coordinator.read_commands("turn-a")) == 1+ await coordinator.release_turn(lease)+ await coordinator.acknowledge_expired_turn("turn-a")+ assert await coordinator.read_events("turn-a") == []+ assert await coordinator.read_commands("turn-a") == []+ assert "cancel-a" not in coordinator._command_ids+ assert await coordinator.list_expired_turn_ids() == []diff --git a/tests/services/session/test_turn_event_flush.py b/tests/services/session/test_turn_event_flush.pyindex 73625d456..dd8442783 100644--- a/tests/services/session/test_turn_event_flush.py+++ b/tests/services/session/test_turn_event_flush.py@@ -1,9 +1,7 @@-"""Post-stream turn-event flush: batching and PocketBase upload.+"""Turn-event batching, prefix durability and PocketBase upload.-The turn runtime buffers every live event in memory and persists the whole-batch, including DONE and any post-turn ``session_meta`` title update, once-the turn has fully finished. Everything on that path must stay O(1) round-trips with respect to the event count.+Live prefixes persist in bounded batches; terminal and post-turn metadata are+flushed before teardown. Concurrent writers share one persistence attempt."""from __future__ import annotations