Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

fix(runtime): persist live prefixes and reclaim completed streams (#1971)

Bingxi Zhao (Frank)e28c8036361c
diff --git a/deeptutor/app/service.py b/deeptutor/app/service.py
index 268537cbf..c84f8960e 100644
--- a/deeptutor/app/service.py
+++ b/deeptutor/app/service.py
@@ -268,20 +268,20 @@ class TurnApplicationService:
done = True
tail_possible = self._done_has_tail(event)
yield event
- if done:
- # The turn had already finished before this subscription opened, so
- # the durable replay above carried its post-DONE events too. Only
- # bail out once the terminal row confirms that; a turn still marked
- # running with a DONE in its journal is mid-teardown and its title
- # may not be written yet.
- turn = await store.get_turn(turn_id)
- if not tail_possible or turn is None or str(turn.get("status") or "") != "running":
- return
+ if done and not tail_possible:
+ return
+ # A completed row can still have an owned title tail. Drain it below,
+ # including durable events committed after the initial replay (#1971).
first_done_at: float | None = time.monotonic() if done else None
while True:
emitted = False
- for event in await self.coordinator.read_events(turn_id, last_seq):
+ events = await self.coordinator.read_events(turn_id, last_seq)
+ if not events and await self.coordinator.get_lease(turn_id) is None:
+ # The completed in-memory stream may have been reclaimed
+ # while this subscriber was paused. Its durable tail remains.
+ events = await store.get_events(turn_id, last_seq)
+ for event in events:
seq = int(event.get("seq") or 0)
if seq <= last_seq:
continue
@@ -313,9 +313,15 @@ class TurnApplicationService:
return
status = str(turn.get("status") or "")
if status in {"completed", "failed", "cancelled"}:
- # A terminal row without DONE can only be legacy data. Keep the
- # compatibility envelope stable while all new paths journal
- # DONE before the terminal CAS.
+ # Completion can commit before the owner publishes DONE. Do
+ # not synthesize an id-less envelope in that window (#1971).
+ if await self.coordinator.get_lease(turn_id) is not None:
+ await asyncio.sleep(0.1)
+ continue
+ # Ownership/cleanup can change between the reads above.
+ if await store.get_events(turn_id, last_seq):
+ continue
+ # Only legacy terminal rows with no journal need synthesis.
metadata = {
"status": status,
"synthesized": True,
diff --git a/deeptutor/runtime/coordination/memory.py b/deeptutor/runtime/coordination/memory.py
index 25e0ff97e..be4d6d610 100644
--- a/deeptutor/runtime/coordination/memory.py
+++ b/deeptutor/runtime/coordination/memory.py
@@ -97,8 +97,15 @@ class MemoryCoordinator:
async def acknowledge_expired_turn(self, turn_id: str) -> None:
async with self._lock:
- self._known_turns.discard(turn_id)
lease = self._turn_leases.get(turn_id)
+ if self._active(lease):
+ return
+ self._known_turns.discard(turn_id)
+ # Recovery has verified the durable terminal row. Keeping these
+ # payloads forever leaks every completed conversation (#1971).
+ self._events.pop(turn_id, None)
+ for _, command in self._commands.pop(turn_id, []):
+ self._command_ids.discard(command.command_id)
if lease is not None and not self._active(lease):
self._turn_leases.pop(turn_id, None)
if self._session_turns.get(lease.session_id) == turn_id:
diff --git a/deeptutor/services/session/_turn_runtime_shared.py b/deeptutor/services/session/_turn_runtime_shared.py
index 3250cabe9..f7aed022d 100644
--- a/deeptutor/services/session/_turn_runtime_shared.py
+++ b/deeptutor/services/session/_turn_runtime_shared.py
@@ -1575,6 +1575,7 @@ class _TurnExecution:
events: list[dict[str, Any]] = field(default_factory=list)
next_seq: int = 1
flush_lock: asyncio.Lock = field(default_factory=asyncio.Lock)
+ flush_task: asyncio.Task[None] | None = None
events_persisted: bool = False
persisted_events: list[dict[str, Any]] = field(default_factory=list)
events_flushed: bool = False
diff --git a/deeptutor/services/session/turns/executor.py b/deeptutor/services/session/turns/executor.py
index 3417c0ab0..639a0a34d 100644
--- a/deeptutor/services/session/turns/executor.py
+++ b/deeptutor/services/session/turns/executor.py
@@ -1404,6 +1404,10 @@ class TurnExecutor:
retryable=resolved_retryable,
)
finally:
+ if execution.flush_task is not None:
+ execution.flush_task.cancel()
+ with contextlib.suppress(asyncio.CancelledError):
+ await execution.flush_task
resource_scope.close()
if llm_scope_token is not None and reset_active_llm_selection is not None:
reset_active_llm_selection(llm_scope_token)
diff --git a/deeptutor/services/session/turns/lifecycle.py b/deeptutor/services/session/turns/lifecycle.py
index a5348e314..419f5d518 100644
--- a/deeptutor/services/session/turns/lifecycle.py
+++ b/deeptutor/services/session/turns/lifecycle.py
@@ -680,18 +680,41 @@ class TurnLifecycle:
if current is not execution:
execution.events.append(payload)
subscribers = list(current.subscribers)
- if event.type == StreamEventType.DONE:
+ if event.type == StreamEventType.DONE or (
+ len(execution.events) - len(execution.persisted_events) >= 25
+ ):
# Never expose DONE to a process-local subscriber before the
# complete event prefix is durable. Redis subscribers already
# read the same canonical payload from the shared journal.
await self._flush_buffered_events(execution)
+ elif execution.flush_task is None or execution.flush_task.done():
+ execution.flush_task = asyncio.create_task(self._flush_live_prefix(execution))
for subscriber in subscribers:
with contextlib.suppress(asyncio.QueueFull):
subscriber.queue.put_nowait(payload)
return payload
+ async def _flush_live_prefix(self, execution: _TurnExecution) -> None:
+ """Bound the crash-loss window even when the next model/tool is quiet."""
+ delay = 0.25
+ while True:
+ await asyncio.sleep(delay)
+ if execution.lease_lost:
+ return
+ try:
+ await self._flush_buffered_events(execution)
+ except Exception:
+ # Keep the pending prefix and retry transient storage errors.
+ # Terminal flushes still propagate failure to the turn owner.
+ logger.exception("Live event persistence failed for turn %s", execution.turn_id)
+ delay = min(delay * 2, 5.0)
+ continue
+ if len(execution.persisted_events) == len(execution.events):
+ return
+ delay = 0.25
+
async def _flush_buffered_events(self, execution: _TurnExecution) -> None:
- """Persist buffered turn events after the live stream has already drained."""
+ """Persist the pending prefix, serialized with periodic and terminal writes."""
async with execution.flush_lock:
await self._flush_buffered_events_once(execution)
diff --git a/tests/app/test_turn_application_service.py b/tests/app/test_turn_application_service.py
index 60bf74db0..575ab11ae 100644
--- a/tests/app/test_turn_application_service.py
+++ b/tests/app/test_turn_application_service.py
@@ -86,6 +86,126 @@ def submission_service(monkeypatch, tmp_path):
return service, store, coordinator, registry, calls
+async def test_subscriber_waits_for_owned_terminal_envelope_and_replays_after_eviction(tmp_path):
+ """A slow subscriber must get the real DONE ids even across cleanup (#1971)."""
+ store = SQLiteSessionStore(tmp_path / "turns.db")
+ coordinator = MemoryCoordinator()
+ service, _ = _service(store, coordinator, "worker")
+ session = await store.ensure_session(None)
+ lease = await coordinator.acquire_turn("owned-terminal", session["id"], "worker")
+ assert lease is not None
+ await store.begin_turn(
+ session["id"],
+ turn_id=lease.turn_id,
+ owner_id=lease.owner_id,
+ fencing_token=lease.fencing_token,
+ )
+ await store.transition_turn(
+ lease.turn_id, "completed", expected_status="running", fencing_token=lease.fencing_token
+ )
+ subscription = service.subscribe_turn(lease.turn_id)
+ waiting = asyncio.create_task(anext(subscription))
+ try:
+ await asyncio.sleep(0.15)
+ assert not waiting.done(), "live owner has not published the real DONE yet"
+ done = await coordinator.publish_event(
+ lease.turn_id,
+ {
+ "type": "done",
+ "metadata": {"status": "completed", "assistant_message_id": "answer"},
+ },
+ )
+ await store.append_events(lease.turn_id, [done], fencing_token=lease.fencing_token)
+ await coordinator.release_turn(lease)
+ await coordinator.acknowledge_expired_turn(lease.turn_id)
+ received = await asyncio.wait_for(waiting, 2)
+ assert received["metadata"]["assistant_message_id"] == "answer"
+ assert not received["metadata"].get("synthesized")
+ finally:
+ waiting.cancel()
+ await asyncio.gather(waiting, return_exceptions=True)
+ await subscription.aclose()
+
+
+async def test_running_turn_flushes_visible_prefix_without_waiting_for_more_events(
+ submission_service,
+ monkeypatch,
+):
+ """A quiet model/tool must not leave already visible work only in RAM (#1971)."""
+ service, store, coordinator, registry, _ = submission_service
+ streamed = asyncio.Event()
+ finish = asyncio.Event()
+
+ class SlowOrchestrator:
+ async def handle(self, context):
+ yield StreamEvent(type=StreamEventType.CONTENT, content="visible prefix", source="chat")
+ streamed.set()
+ await finish.wait()
+ yield StreamEvent(type=StreamEventType.DONE, source="chat")
+
+ monkeypatch.setattr("deeptutor.runtime.orchestrator.ChatOrchestrator", SlowOrchestrator)
+ _, turn = await service.start_turn({"content": "Hello"})
+ try:
+ await asyncio.wait_for(streamed.wait(), 3)
+ await asyncio.sleep(0.6)
+ restarted = SQLiteSessionStore(store.db_path)
+ events = await restarted.get_events(turn["id"])
+ assert any(e.get("content") == "visible prefix" for e in events)
+ assert (await restarted.get_turn(turn["id"]))["status"] == "running"
+ finally:
+ finish.set()
+ [event async for event in service.subscribe_turn(turn["id"])]
+
+
+async def test_reconnect_after_done_keeps_the_live_post_turn_title(tmp_path):
+ store = SQLiteSessionStore(tmp_path / "turns.db")
+ coordinator = MemoryCoordinator()
+ service, _ = _service(store, coordinator, "worker")
+ session = await store.ensure_session(None)
+ lease = await coordinator.acquire_turn("title-tail", session["id"], "worker")
+ await store.begin_turn(
+ session["id"],
+ turn_id=lease.turn_id,
+ owner_id=lease.owner_id,
+ fencing_token=lease.fencing_token,
+ )
+ done = await coordinator.publish_event(
+ lease.turn_id,
+ {
+ "type": "done",
+ "metadata": {"status": "completed"},
+ },
+ )
+ await store.append_events(lease.turn_id, [done], fencing_token=lease.fencing_token)
+ await store.transition_turn(
+ lease.turn_id, "completed", expected_status="running", fencing_token=lease.fencing_token
+ )
+ stream = service.subscribe_turn(lease.turn_id)
+ assert (await anext(stream))["type"] == "done"
+ waiting = asyncio.create_task(anext(stream))
+ try:
+ await asyncio.sleep(0.05)
+ assert not waiting.done(), "terminal status is not the end of the owned title tail"
+ title = await coordinator.publish_event(
+ lease.turn_id,
+ {
+ "type": "session_meta",
+ "metadata": {"title": "Cell membrane"},
+ },
+ )
+ await store.append_events(lease.turn_id, [title], fencing_token=lease.fencing_token)
+ await coordinator.release_turn(lease)
+ await coordinator.acknowledge_expired_turn(lease.turn_id)
+ assert (await asyncio.wait_for(waiting, 2))["metadata"]["title"] == "Cell membrane"
+ finally:
+ waiting.cancel()
+ await asyncio.gather(waiting, return_exceptions=True)
+ await stream.aclose()
+
+
@pytest.mark.asyncio
async def test_replacement_endpoint_workflow_starts_once_and_branches_admitted_user(
submission_service,
diff --git a/tests/runtime/coordination/test_memory_coordinator.py b/tests/runtime/coordination/test_memory_coordinator.py
index a2c8410be..2705b8cda 100644
--- a/tests/runtime/coordination/test_memory_coordinator.py
+++ b/tests/runtime/coordination/test_memory_coordinator.py
@@ -133,3 +133,21 @@ async def test_background_leader_is_singleton_and_transferable() -> None:
successor = await coordinator.acquire_leader("worker-b")
assert successor is not None
assert successor.fencing_token > leader.fencing_token
+
+
+async def test_acknowledged_turn_releases_payloads_but_live_owner_keeps_them() -> None:
+ coordinator = MemoryCoordinator()
+ lease = await coordinator.acquire_turn("turn-a", "session-a", "worker")
+ assert lease is not None
+ await coordinator.publish_event("turn-a", {"type": "content", "content": "large answer"})
+ await coordinator.submit_command("turn-a", "cancel", command_id="cancel-a")
+ await coordinator.acknowledge_expired_turn("turn-a")
+ assert len(await coordinator.read_events("turn-a")) == 1
+ assert len(await coordinator.read_commands("turn-a")) == 1
+ await coordinator.release_turn(lease)
+ await coordinator.acknowledge_expired_turn("turn-a")
+ assert await coordinator.read_events("turn-a") == []
+ assert await coordinator.read_commands("turn-a") == []
+ assert "cancel-a" not in coordinator._command_ids
+ assert await coordinator.list_expired_turn_ids() == []
diff --git a/tests/services/session/test_turn_event_flush.py b/tests/services/session/test_turn_event_flush.py
index 73625d456..dd8442783 100644
--- a/tests/services/session/test_turn_event_flush.py
+++ b/tests/services/session/test_turn_event_flush.py
@@ -1,9 +1,7 @@
-"""Post-stream turn-event flush: batching and PocketBase upload.
+"""Turn-event batching, prefix durability and PocketBase upload.
-The turn runtime buffers every live event in memory and persists the whole
-batch, including DONE and any post-turn ``session_meta`` title update, once
-the turn has fully finished. Everything on that path must stay O(1) round
-trips with respect to the event count.
+Live prefixes persist in bounded batches; terminal and post-turn metadata are
+flushed before teardown. Concurrent writers share one persistence attempt.
"""
from __future__ import annotations