Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

fix(session): stop waiting turns before deletion

XZH7cf2fc94a351
diff --git a/deeptutor/api/routers/mastery_path.py b/deeptutor/api/routers/mastery_path.py
index 82c052aca..4ea4c8084 100644
--- a/deeptutor/api/routers/mastery_path.py
+++ b/deeptutor/api/routers/mastery_path.py
@@ -55,6 +55,12 @@ def get_learning_service() -> LearningService:
return LearningService(store)
+def _turn_application_service():
+ from deeptutor.app.container import get_application_container
+
+ return get_application_container().turns
+
+
def _validate_book_id(book_id: str) -> None:
"""Reject empty or path-traversal-bearing book ids (shared by all endpoints)."""
if not book_id or ".." in book_id or "/" in book_id or "\\" in book_id or ":" in book_id:
@@ -99,34 +105,45 @@ def _validate_runnable_modules(modules: list[LearningModule], *, status_code: in
async def _cancel_active_learning_turn(book_id: str) -> None:
- from deeptutor.services.session import get_turn_runtime_manager
-
learning_store = LearningStore()
- runtime = get_turn_runtime_manager()
+ turns = _turn_application_service()
lease = await asyncio.to_thread(learning_store.get_path_lease, book_id)
if lease is not None:
if lease.session_id == "__path_api__":
# Another administrative mutation owns the path. The caller's
# acquisition attempt will return a deterministic HTTP 409.
return
- await runtime.cancel_turn(lease.turn_id)
- # ``cancel_turn`` can finalize a restart orphan without an in-memory
- # task, so its normal runtime ``finally`` cannot release the lease.
- await asyncio.to_thread(
- learning_store.release_path_lease,
- book_id,
- turn_id=lease.turn_id,
- )
+ if await turns.cancel_turn_and_wait(lease.turn_id):
+ # ``cancel_turn`` can finalize a restart orphan without an
+ # in-memory task, so its normal runtime ``finally`` cannot release
+ # the mastery path lease.
+ await asyncio.to_thread(
+ learning_store.release_path_lease,
+ book_id,
+ turn_id=lease.turn_id,
+ )
+ else:
+ raise HTTPException(
+ status_code=409,
+ detail="Active learning turn could not be stopped before changing this path",
+ )
return
# Compatibility for turns started before explicit path leases existed.
session_ids = await asyncio.to_thread(learning_store.list_session_ids, book_id)
if book_id not in session_ids:
session_ids.append(book_id)
+ from deeptutor.services.session import get_session_store
+
+ session_store = get_session_store()
for session_id in session_ids:
- for turn in await runtime.store.list_active_turns(session_id):
+ for turn in await session_store.list_active_turns(session_id):
if str(turn.get("capability") or "") == "mastery_path":
- await runtime.cancel_turn(turn["id"])
+ if not await turns.cancel_turn_and_wait(turn["id"]):
+ raise HTTPException(
+ status_code=409,
+ detail="Active learning turn could not be stopped before changing this path",
+ )
@asynccontextmanager
diff --git a/deeptutor/api/routers/sessions.py b/deeptutor/api/routers/sessions.py
index 633c3364b..310c86d40 100644
--- a/deeptutor/api/routers/sessions.py
+++ b/deeptutor/api/routers/sessions.py
@@ -30,6 +30,12 @@ logger = logging.getLogger(__name__)
router = APIRouter()
+def _turn_application_service():
+ from deeptutor.app.container import get_application_container
+
+ return get_application_container().turns
+
+
class SessionRenameRequest(BaseModel):
title: str = Field(..., min_length=1, max_length=100)
@@ -480,12 +486,14 @@ async def delete_session(session_id: str):
list_active_turns = getattr(store, "list_active_turns", None)
if callable(list_active_turns):
- from deeptutor.services.session import get_turn_runtime_manager
-
- runtime = get_turn_runtime_manager()
+ turns = _turn_application_service()
for target in ordered:
for turn in await list_active_turns(target):
- await runtime.cancel_turn(turn["id"])
+ if not await turns.cancel_turn_and_wait(turn["id"]):
+ raise HTTPException(
+ status_code=409,
+ detail="Active conversation turn could not be stopped before deletion",
+ )
for target in reversed(ordered):
if not await store.delete_session(target):
raise HTTPException(status_code=409, detail="Unable to delete conversation")
diff --git a/deeptutor/app/service.py b/deeptutor/app/service.py
index face96390..9c999b27d 100644
--- a/deeptutor/app/service.py
+++ b/deeptutor/app/service.py
@@ -133,6 +133,8 @@ class TurnApplicationService:
#
# The cap below exists only for a leaked lease; it is not the normal path.
_POST_DONE_MAX_SECONDS = 30.0
+ _CANCEL_WAIT_INTERVAL_SECONDS = 0.05
+ _ACTIVE_TURN_STATUSES = frozenset({"queued", "running", "waiting_input"})
@staticmethod
def _done_has_tail(event: dict[str, Any]) -> bool:
@@ -280,6 +282,40 @@ class TurnApplicationService:
# that lost the first ACK can retire its durable outbox entry.
return True
+ @workspace_writer
+ async def cancel_turn_and_wait(
+ self,
+ turn_id: str,
+ *,
+ command_id: str | None = None,
+ timeout_seconds: float = 15.0,
+ ) -> bool:
+ """Stop a live turn and wait until its owner has finished teardown."""
+ if not await self.cancel_turn(turn_id, command_id=command_id):
+ return False
+ store, _runtime = self._resolve()
+ deadline = time.monotonic() + max(0.0, float(timeout_seconds))
+ released = False
+ while True:
+ turn = await store.get_turn(turn_id)
+ status = str((turn or {}).get("status") or "")
+ lease = await self.coordinator.get_lease(turn_id)
+ if turn is not None and status not in self._ACTIVE_TURN_STATUSES and lease is None:
+ released = True
+ break
+ if turn is None and lease is None:
+ released = True
+ break
+ if time.monotonic() >= deadline:
+ break
+ await asyncio.sleep(self._CANCEL_WAIT_INTERVAL_SECONDS)
+ if released:
+ # Lease release happens just before the owner's activity guard
+ # exits. Leave one polling interval so deletion does not race that
+ # final unwind.
+ await asyncio.sleep(self._CANCEL_WAIT_INTERVAL_SECONDS)
+ return released
+
@workspace_writer
async def submit_user_reply(
self,
diff --git a/tests/api/test_mastery_path_turn_cancellation.py b/tests/api/test_mastery_path_turn_cancellation.py
new file mode 100644
index 000000000..d55fefb5a
--- /dev/null
+++ b/tests/api/test_mastery_path_turn_cancellation.py
@@ -0,0 +1,67 @@
+from types import SimpleNamespace
+from unittest.mock import AsyncMock, Mock
+
+from fastapi import HTTPException
+import pytest
+
+from deeptutor.api.routers import mastery_path as router
+
+
+async def test_path_mutation_waits_for_waiting_turn_and_releases_lease(monkeypatch):
+ learning_store = SimpleNamespace(
+ get_path_lease=lambda _book_id: SimpleNamespace(
+ session_id="session", turn_id="waiting-turn"
+ ),
+ release_path_lease=Mock(),
+ )
+ turns = SimpleNamespace(cancel_turn_and_wait=AsyncMock(return_value=True))
+ monkeypatch.setattr(router, "LearningStore", lambda: learning_store)
+ monkeypatch.setattr(router, "_turn_application_service", lambda: turns)
+
+ await router._cancel_active_learning_turn("path")
+
+ turns.cancel_turn_and_wait.assert_awaited_once_with("waiting-turn")
+ learning_store.release_path_lease.assert_called_once_with("path", turn_id="waiting-turn")
+
+
+async def test_path_mutation_keeps_lease_when_cancellation_times_out(monkeypatch):
+ learning_store = SimpleNamespace(
+ get_path_lease=lambda _book_id: SimpleNamespace(
+ session_id="session", turn_id="waiting-turn"
+ ),
+ release_path_lease=Mock(),
+ )
+ turns = SimpleNamespace(cancel_turn_and_wait=AsyncMock(return_value=False))
+ monkeypatch.setattr(router, "LearningStore", lambda: learning_store)
+ monkeypatch.setattr(router, "_turn_application_service", lambda: turns)
+
+ with pytest.raises(HTTPException) as exc:
+ await router._cancel_active_learning_turn("path")
+
+ assert exc.value.status_code == 409
+ learning_store.release_path_lease.assert_not_called()
+
+
+async def test_legacy_mastery_session_lookup_uses_application_cancellation(monkeypatch):
+ learning_store = SimpleNamespace(
+ get_path_lease=lambda _book_id: None,
+ list_session_ids=lambda _book_id: ["session"],
+ )
+
+ async def list_active_turns(session_id):
+ if session_id == "session":
+ return [{"id": "turn", "capability": "mastery_path"}]
+ return []
+
+ session_store = SimpleNamespace(list_active_turns=list_active_turns)
+ turns = SimpleNamespace(cancel_turn_and_wait=AsyncMock(return_value=True))
+ monkeypatch.setattr(router, "LearningStore", lambda: learning_store)
+ monkeypatch.setattr("deeptutor.services.session.get_session_store", lambda: session_store)
+ monkeypatch.setattr(router, "_turn_application_service", lambda: turns)
+
+ await router._cancel_active_learning_turn("path")
+
+ turns.cancel_turn_and_wait.assert_awaited_once_with("turn")
diff --git a/tests/api/test_sessions_permanent_delete.py b/tests/api/test_sessions_permanent_delete.py
index 93d2cbfd8..f7322e7ee 100644
--- a/tests/api/test_sessions_permanent_delete.py
+++ b/tests/api/test_sessions_permanent_delete.py
@@ -22,8 +22,8 @@ async def test_delete_removes_conversation_descendants_and_messages(tmp_path, mo
monkeypatch.setattr(router, "LearningStore", lambda: learning)
reading = SimpleNamespace(forget_session=Mock())
monkeypatch.setattr(router, "ReadingCatalogStore", lambda: reading)
- runtime = SimpleNamespace(cancel_turn=AsyncMock())
- monkeypatch.setattr("deeptutor.services.session.get_turn_runtime_manager", lambda: runtime)
+ turns = SimpleNamespace(cancel_turn_and_wait=AsyncMock(return_value=True))
+ monkeypatch.setattr(router, "_turn_application_service", lambda: turns)
monkeypatch.setattr(
store, "list_active_turns", AsyncMock(side_effect=lambda sid: [{"id": f"turn-{sid}"}])
)
@@ -38,7 +38,7 @@ async def test_delete_removes_conversation_descendants_and_messages(tmp_path, mo
attachments.delete_session.assert_any_await(sid)
learning.detach_session.assert_any_call(sid)
reading.forget_session.assert_any_call(sid)
- runtime.cancel_turn.assert_any_await(f"turn-{sid}")
+ turns.cancel_turn_and_wait.assert_any_await(f"turn-{sid}")
assert await store.get_session("other") is not None
assert len(await store.get_messages("other")) == 1
assert await store.list_deleted_sessions() == []
@@ -54,6 +54,24 @@ async def test_missing_or_unowned_session_has_no_deletion_side_effects(monkeypat
store.delete_session.assert_not_awaited()
+async def test_delete_keeps_session_when_turn_cancellation_times_out(tmp_path, monkeypatch):
+ store = SQLiteSessionStore(tmp_path / "sessions.db")
+ await store.create_session(session_id="busy", title="Busy")
+ await store.begin_turn("busy", "mastery_path", turn_id="waiting-turn")
+ await store.update_turn_status("waiting-turn", "waiting_input")
+ monkeypatch.setattr(router, "get_session_store", lambda: store)
+ turns = SimpleNamespace(cancel_turn_and_wait=AsyncMock(return_value=False))
+ monkeypatch.setattr(router, "_turn_application_service", lambda: turns)
+
+ with pytest.raises(HTTPException) as exc:
+ await router.delete_session("busy")
+
+ assert exc.value.status_code == 409
+ assert await store.get_session("busy") is not None
+ assert await store.get_turn("waiting-turn") is not None
+
+
@pytest.mark.asyncio
async def test_legacy_deleted_chats_migrate_to_archive_without_data_loss(tmp_path):
path = tmp_path / "sessions.db"
diff --git a/tests/app/test_multiworker_turn_application.py b/tests/app/test_multiworker_turn_application.py
index 063abf78a..c06a6be00 100644
--- a/tests/app/test_multiworker_turn_application.py
+++ b/tests/app/test_multiworker_turn_application.py
@@ -10,6 +10,7 @@ from deeptutor.core.stream import StreamEvent, StreamEventType
from deeptutor.runtime.coordination import MemoryCoordinator
from deeptutor.services.session.sqlite_store import SQLiteSessionStore
from deeptutor.services.session.turn_runtime import TurnRuntimeManager
+from deeptutor.services.workspace.activity import data_activity
class _ContextBuilder:
@@ -177,3 +178,64 @@ async def test_remote_worker_reply_reaches_owner_waiter(monkeypatch, tmp_path) -
assert [event["seq"] for event in received] == list(range(1, len(received) + 1))
await runtime_a.close()
await runtime_b.close()
+
+
+async def test_remote_cancel_waits_for_waiting_turn_teardown(monkeypatch, tmp_path) -> None:
+ waiting = asyncio.Event()
+
+ class Engine:
+ async def execute(self, context):
+ yield StreamEvent(
+ type=StreamEventType.WAIT_FOR_INPUT,
+ source="chat",
+ content="Continue?",
+ )
+ waiting.set()
+ await context.runtime.wait_for_user_reply()
+
+ monkeypatch.setattr("deeptutor.services.llm.config.get_llm_config", lambda: SimpleNamespace())
+ monkeypatch.setattr(
+ "deeptutor.services.session.context_builder.ContextBuilder", _ContextBuilder
+ )
+ coordinator = MemoryCoordinator(lease_ttl_seconds=5)
+ path = tmp_path / "shared.sqlite3"
+ store_a = SQLiteSessionStore(path)
+ store_b = SQLiteSessionStore(path)
+ runtime_a = TurnRuntimeManager(
+ store_a, coordinator=coordinator, owner_id="worker-a", turn_engine=Engine()
+ )
+ runtime_b = TurnRuntimeManager(
+ store_b, coordinator=coordinator, owner_id="worker-b", turn_engine=Engine()
+ )
+
+ async def _noop_title(**_kwargs):
+ return None
+
+ monkeypatch.setattr(runtime_a, "_maybe_generate_session_title", _noop_title)
+ app_a = _application(store_a, runtime_a, coordinator)
+ app_b = _application(store_b, runtime_b, coordinator)
+
+ _session, turn = await app_a.start_turn(_payload())
+ await asyncio.wait_for(waiting.wait(), timeout=2)
+ for _ in range(100):
+ active = await app_b.check_active_turn(turn["session_id"])
+ if active and active["status"] == "waiting_input":
+ break
+ await asyncio.sleep(0.01)
+ assert active is not None
+ assert active["status"] == "waiting_input"
+
+ assert await app_b.cancel_turn_and_wait(
+ turn["id"], command_id="cancel-waiting-from-b", timeout_seconds=3
+ )
+
+ persisted = await store_b.get_turn(turn["id"])
+ assert persisted is not None
+ assert persisted["status"] == "cancelled"
+ assert await coordinator.get_lease(turn["id"]) is None
+ assert turn["id"] not in runtime_a._executions
+ with data_activity(exclusive=True):
+ pass
+ await runtime_a.close()
+ await runtime_b.close()