Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

test(app): lock lost-ack resend session parity behind regression tests

XZHc5b8c9c7ae36
diff --git a/tests/app/test_lost_ack_resend_reuses_session.py b/tests/app/test_lost_ack_resend_reuses_session.py
new file mode 100644
index 000000000..a945b87fe
--- /dev/null
+++ b/tests/app/test_lost_ack_resend_reuses_session.py
@@ -0,0 +1,290 @@
+"""A lost-ack resend must reuse the original conversation and its message.
+
+When the worker owning a turn dies (or the reply/cancel acknowledgement is
+lost), recovery fails the zombie with ``worker_lost`` and tells the learner
+to resend their answer as a new message. The resend contract: the resent
+submission lands in the ORIGINAL conversation, reuses the ORIGINAL user
+message row (never a duplicate), and never forks the session into a second
+conversation — the guarantee the submission reservation (#1793) exists to
+uphold. Resending while the original is still parked replays it instead of
+starting a competing turn.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from types import SimpleNamespace
+
+import pytest
+
+from deeptutor.app.service import TurnApplicationService
+from deeptutor.core.stream import StreamEvent, StreamEventType
+from deeptutor.runtime.coordination import MemoryCoordinator
+from deeptutor.services.session.sqlite_store import SQLiteSessionStore
+from deeptutor.services.session.turn_runtime import TurnRuntimeManager
+
+
+class _ContextBuilder:
+ def __init__(self, *_args, **_kwargs) -> None:
+ pass
+
+ async def build(self, **_kwargs):
+ return SimpleNamespace(
+ conversation_history=[],
+ conversation_summary="",
+ context_text="",
+ token_count=0,
+ budget=0,
+ )
+
+
+class _PausableEngine:
+ """First run parks the turn on a waiting card; later runs answer at once.
+
+ The parked run is the submission whose worker is about to be "lost":
+ the learner's answer is persisted, the row sits in ``waiting_input``,
+ and the answer never reaches the conversation.
+ """
+
+ def __init__(self) -> None:
+ self.parked = asyncio.Event()
+ self.started = 0
+ self.finish_without_waiting = False
+
+ async def execute(self, context):
+ self.started += 1
+ if not self.finish_without_waiting:
+ yield StreamEvent(
+ type=StreamEventType.WAIT_FOR_INPUT,
+ source="chat",
+ content="Continue?",
+ )
+ self.parked.set()
+ reply = await context.runtime.wait_for_user_reply()
+ yield StreamEvent(
+ type=StreamEventType.CONTENT,
+ source="chat",
+ content=f"reply:{reply['text']}",
+ )
+ else:
+ yield StreamEvent(
+ type=StreamEventType.CONTENT,
+ source="chat",
+ content="Recovered answer",
+ )
+ yield StreamEvent(type=StreamEventType.DONE, source="chat")
+
+
+def _payload() -> dict:
+ return {
+ "content": "my original answer",
+ "client_submission_id": "lost-ack-resend-1",
+ "capability": "chat",
+ "tools": [],
+ "knowledge_bases": [],
+ "attachments": [],
+ "language": "en",
+ "config": {},
+ }
+
+
[email protected](autouse=True)
+def _workspace_root(tmp_path, monkeypatch: pytest.MonkeyPatch):
+ """Isolate the runtime home and workspace binding for this test.
+
+ Without ``DEEPTUTOR_HOME`` the runtime resolves paths from the current
+ working directory, so a catalog row written by an earlier process would
+ point later runs at a deleted folder ("The selected workspace folder
+ does not exist"). A private home plus a fresh ``PathService`` keeps
+ settings, the workspace catalog, and the deployment root tmp-scoped.
+ """
+ monkeypatch.setenv("DEEPTUTOR_HOME", str(tmp_path))
+ from deeptutor.services.path_service import PathService
+
+ monkeypatch.setattr(PathService, "_instance", PathService(workspace_root=tmp_path / "data"))
+ root = tmp_path / "workspace"
+ root.mkdir()
+ monkeypatch.setenv("DEEPTUTOR_WORKSPACE_ROOT", str(root))
+ monkeypatch.delenv("DEEPTUTOR_WORKSPACE_ALLOWED_ROOTS", raising=False)
+
+
+def llm_and_context(monkeypatch: pytest.MonkeyPatch):
+ monkeypatch.setattr("deeptutor.services.llm.config.get_llm_config", lambda: SimpleNamespace())
+ monkeypatch.setattr(
+ "deeptutor.services.session.context_builder.ContextBuilder", _ContextBuilder
+ )
+
+
+def _service(
+ store: SQLiteSessionStore,
+ coordinator: MemoryCoordinator,
+ worker_id: str,
+ engine: _PausableEngine,
+) -> TurnApplicationService:
+ runtime = TurnRuntimeManager(
+ store, coordinator=coordinator, owner_id=worker_id, turn_engine=engine
+ )
+ return TurnApplicationService(
+ SimpleNamespace(get=lambda: store),
+ SimpleNamespace(get=lambda _store: runtime),
+ coordinator,
+ )
+
+
+async def _wait_for_status(store: SQLiteSessionStore, turn_id: str, status: str) -> dict:
+ loop = asyncio.get_running_loop()
+ deadline = loop.time() + 5.0
+ while loop.time() < deadline:
+ row = await store.get_turn(turn_id)
+ if row is not None and str(row.get("status")) == status:
+ return row
+ await asyncio.sleep(0.02)
+ raise AssertionError(f"turn {turn_id} never reached {status}")
+
+
+async def _park_lost_submission(
+ tmp_path, engine: _PausableEngine
+) -> tuple[SQLiteSessionStore, dict, dict]:
+ """Run the first submission for real, then drop the worker's lease view.
+
+ A fresh coordinator for the recovery worker mirrors the restarted
+ process: its lease view is empty, so the parked ``waiting_input`` row is
+ exactly the unowned zombie a dead process leaves behind.
+ """
+ coordinator_a = MemoryCoordinator(lease_ttl_seconds=30)
+ store = SQLiteSessionStore(tmp_path / "shared.sqlite3")
+ app_a = _service(store, coordinator_a, "worker-a", engine)
+ session, turn = await app_a.start_turn(_payload())
+ await asyncio.wait_for(engine.parked.wait(), timeout=5)
+ await _wait_for_status(store, turn["id"], "waiting_input")
+ return store, session, turn
+
+
+async def test_resend_while_submission_is_still_parked_replays_the_same_turn(
+ tmp_path, llm_and_context
+) -> None:
+ """A client retry that races the parked original replays it, never forks.
+
+ The learner's answer is persisted exactly once and no second turn is
+ started for the same submission id while the first is still live — and
+ the replay must not reap or otherwise disturb the parked original.
+ """
+ engine = _PausableEngine()
+ store, session, turn = await _park_lost_submission(tmp_path, engine)
+ app = _service(store, MemoryCoordinator(lease_ttl_seconds=30), "worker-b", engine)
+
+ resent_session, resent_turn = await app.start_turn(_payload())
+
+ assert resent_session["id"] == session["id"], "the retry must stay in its conversation"
+ assert resent_turn["id"] == turn["id"], "the live original must be replayed, not raced"
+ assert len(await store.list_sessions()) == 1
+ rows = await store.get_messages(session["id"])
+ assert [row["role"] for row in rows] == ["user"], "no duplicate answer may be recorded"
+ assert rows[0]["metadata"]["client_submission_id"] == "lost-ack-resend-1"
+ parked = await store.get_turn(turn["id"])
+ assert parked["status"] == "waiting_input", "the replay must leave the original parked"
+ assert engine.started == 1, "the replay must not execute the submission a second time"
+
+
+async def test_resend_after_worker_lost_reap_reuses_session_and_user_message(
+ tmp_path, llm_and_context
+) -> None:
+ """After the zombie is reaped, the resend lands in the same conversation.
+
+ The new turn reuses the original user row (no duplicate answer), the
+ done event points back at it, and no second session is created.
+ """
+ engine = _PausableEngine()
+ store, session, turn = await _park_lost_submission(tmp_path, engine)
+ original_user = await store.get_last_message(session["id"], role="user")
+ assert original_user is not None
+
+ coordinator_b = MemoryCoordinator(lease_ttl_seconds=30)
+ app_b = _service(store, coordinator_b, "worker-b", engine)
+
+ # The learner's answer reaches the dead worker first: the reply ack is
+ # rejected and the zombie is reaped synchronously.
+ assert await app_b.submit_user_reply(turn["id"], text="my original answer") is False
+ reaped = await _wait_for_status(store, turn["id"], "failed")
+ assert reaped["failure_code"] == "worker_lost"
+ assert reaped["retryable"] in (True, 1)
+
+ engine.finish_without_waiting = True
+ resent_session, resent_turn = await app_b.start_turn(_payload())
+
+ assert resent_session["id"] == session["id"], "the resend must not fork the conversation"
+ assert resent_turn["id"] != turn["id"]
+ assert len(await store.list_sessions()) == 1
+ async for _event in app_b.subscribe_turn(resent_turn["id"]):
+ pass
+ rows = await store.get_messages(session["id"])
+ assert [row["role"] for row in rows] == ["user", "assistant"]
+ assert rows[0]["id"] == original_user["id"], "the original answer must be reused verbatim"
+ done = [
+ event async for event in app_b.subscribe_turn(resent_turn["id"]) if event["type"] == "done"
+ ]
+ assert done and done[-1]["metadata"]["user_message_id"] == original_user["id"]
+ assert engine.started == 2, "only the resend may execute; the replay must not re-run it"
+
+
+async def test_cancelling_the_zombie_opens_the_same_resend_path(tmp_path, llm_and_context) -> None:
+ """A give-up cancel on the unowned zombie enables the same resend parity.
+
+ Cancelling a turn nobody executes must reap it (not acknowledge into the
+ void), and the following resend must reuse the conversation and the
+ original user message exactly like the reply-ack path does.
+ """
+ engine = _PausableEngine()
+ store, session, turn = await _park_lost_submission(tmp_path, engine)
+ original_user = await store.get_last_message(session["id"], role="user")
+ assert original_user is not None
+
+ coordinator_b = MemoryCoordinator(lease_ttl_seconds=30)
+ app_b = _service(store, coordinator_b, "worker-b", engine)
+
+ assert await app_b.cancel_turn(turn["id"]) is True
+ reaped = await _wait_for_status(store, turn["id"], "failed")
+ assert reaped["failure_code"] == "worker_lost"
+ assert reaped["retryable"] in (True, 1)
+
+ engine.finish_without_waiting = True
+ resent_session, resent_turn = await app_b.start_turn(_payload())
+
+ assert resent_session["id"] == session["id"]
+ assert resent_turn["id"] != turn["id"]
+ assert len(await store.list_sessions()) == 1
+ async for _event in app_b.subscribe_turn(resent_turn["id"]):
+ pass
+ rows = await store.get_messages(session["id"])
+ assert [row["role"] for row in rows] == ["user", "assistant"]
+ assert rows[0]["id"] == original_user["id"]
+
+
+async def test_resend_cannot_swap_content_after_the_zombie_failed(
+ tmp_path, llm_and_context
+) -> None:
+ """The submission id stays bound to its content even after a lost ack.
+
+ A reap makes the submission resendable, not hijackable: different
+ content under the same id is refused and no fork session is created.
+ """
+ engine = _PausableEngine()
+ store, session, turn = await _park_lost_submission(tmp_path, engine)
+
+ coordinator_b = MemoryCoordinator(lease_ttl_seconds=30)
+ app_b = _service(store, coordinator_b, "worker-b", engine)
+ assert await app_b.submit_user_reply(turn["id"], text="my original answer") is False
+ reaped = await _wait_for_status(store, turn["id"], "failed")
+ assert reaped["failure_code"] == "worker_lost"
+
+ with pytest.raises(RuntimeError, match="different request"):
+ await app_b.start_turn({**_payload(), "content": "a different answer"})
+ assert len(await store.list_sessions()) == 1
+ rows = await store.get_messages(session["id"])
+ assert [row["role"] for row in rows] == ["user"], "the refusal must not write messages"
+ assert engine.started == 1