DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.
Python ai-agentsai-tutorclawbotcli-tooldeepresearchinteractive-learninglearningmulti-agent-systemsrag
from __future__ import annotations
import asyncio
from types import SimpleNamespace
import pytest
from deeptutor.app.container import RuntimeRegistry
from deeptutor.app.service import TurnApplicationService
from deeptutor.core.stream import StreamEvent, StreamEventType
from deeptutor.runtime.coordination import MemoryCoordinator
from deeptutor.services.session.sqlite_store import SQLiteSessionStore
from deeptutor.services.session.turn_runtime import _TurnExecution
class _FixedStoreProvider:
def __init__(self, store: SQLiteSessionStore) -> None:
self.store = store
def get(self) -> SQLiteSessionStore:
return self.store
def _service(
store: SQLiteSessionStore,
coordinator: MemoryCoordinator,
worker_id: str,
) -> tuple[TurnApplicationService, RuntimeRegistry]:
registry = RuntimeRegistry(coordinator, worker_id)
return (
TurnApplicationService(_FixedStoreProvider(store), registry, coordinator),
registry,
)
@pytest.fixture
def submission_service(monkeypatch, tmp_path):
from deeptutor.services.path_service import PathService
from deeptutor.services.session.turns.title_service import SessionTitleService
monkeypatch.setenv("DEEPTUTOR_HOME", str(tmp_path))
monkeypatch.setattr(PathService, "_instance", PathService(workspace_root=tmp_path / "data"))
calls = {"count": 0, "fail": False}
class Builder:
def __init__(self, *args, **kwargs):
pass
async def build(self, **kwargs):
return SimpleNamespace(
conversation_history=[],
conversation_summary="",
context_text="",
token_count=0,
budget=0,
)
class Orchestrator:
async def handle(self, context):
calls["count"] += 1
if calls["fail"]:
from deeptutor.services.llm.exceptions import LLMProviderTransportError
raise LLMProviderTransportError("controlled upstream failure")
yield StreamEvent(
type=StreamEventType.CONTENT,
content="Recovered answer",
source="chat",
metadata={"call_kind": "llm_final_response"},
)
yield StreamEvent(type=StreamEventType.DONE, source="chat")
async def noop(*args, **kwargs):
pass
monkeypatch.setattr("deeptutor.services.llm.config.get_llm_config", lambda: SimpleNamespace())
monkeypatch.setattr("deeptutor.services.session.context_builder.ContextBuilder", Builder)
monkeypatch.setattr("deeptutor.runtime.orchestrator.ChatOrchestrator", Orchestrator)
monkeypatch.setattr(
"deeptutor.services.memory.get_memory_store", lambda: SimpleNamespace(emit=noop)
)
monkeypatch.setattr(SessionTitleService, "_maybe_generate_session_title", noop)
store = SQLiteSessionStore(tmp_path / "submissions.db")
coordinator = MemoryCoordinator(lease_ttl_seconds=30)
service, registry = _service(store, coordinator, "worker-a")
return service, store, coordinator, registry, calls
@pytest.mark.asyncio
async def test_subscriber_waits_for_owned_terminal_envelope_and_replays_after_eviction(tmp_path):
"""A slow subscriber must get the real DONE ids even across cleanup (#1971)."""
store = SQLiteSessionStore(tmp_path / "turns.db")
coordinator = MemoryCoordinator()
service, _ = _service(store, coordinator, "worker")
session = await store.ensure_session(None)
lease = await coordinator.acquire_turn("owned-terminal", session["id"], "worker")
assert lease is not None
await store.begin_turn(
session["id"],
turn_id=lease.turn_id,
owner_id=lease.owner_id,
fencing_token=lease.fencing_token,
)
await store.transition_turn(
lease.turn_id, "completed", expected_status="running", fencing_token=lease.fencing_token
)
subscription = service.subscribe_turn(lease.turn_id)
waiting = asyncio.create_task(anext(subscription))
try:
await asyncio.sleep(0.15)
assert not waiting.done(), "live owner has not published the real DONE yet"
done = await coordinator.publish_event(
lease.turn_id,
{
"type": "done",
"metadata": {"status": "completed", "assistant_message_id": "answer"},
},
)
await store.append_events(lease.turn_id, [done], fencing_token=lease.fencing_token)
await coordinator.release_turn(lease)
await coordinator.acknowledge_expired_turn(lease.turn_id)
received = await asyncio.wait_for(waiting, 2)
assert received["metadata"]["assistant_message_id"] == "answer"
assert not received["metadata"].get("synthesized")
finally:
waiting.cancel()
await asyncio.gather(waiting, return_exceptions=True)
await subscription.aclose()
@pytest.mark.asyncio
async def test_running_turn_flushes_visible_prefix_without_waiting_for_more_events(
submission_service,
monkeypatch,
):
"""A quiet model/tool must not leave already visible work only in RAM (#1971)."""
service, store, coordinator, registry, _ = submission_service
streamed = asyncio.Event()
finish = asyncio.Event()
class SlowOrchestrator:
async def handle(self, context):
yield StreamEvent(type=StreamEventType.CONTENT, content="visible prefix", source="chat")
streamed.set()
await finish.wait()
yield StreamEvent(type=StreamEventType.DONE, source="chat")
monkeypatch.setattr("deeptutor.runtime.orchestrator.ChatOrchestrator", SlowOrchestrator)
_, turn = await service.start_turn({"content": "Hello"})
try:
await asyncio.wait_for(streamed.wait(), 3)
await asyncio.sleep(0.6)
restarted = SQLiteSessionStore(store.db_path)
events = await restarted.get_events(turn["id"])
assert any(e.get("content") == "visible prefix" for e in events)
assert (await restarted.get_turn(turn["id"]))["status"] == "running"
finally:
finish.set()
[event async for event in service.subscribe_turn(turn["id"])]
@pytest.mark.asyncio
async def test_reconnect_after_done_keeps_the_live_post_turn_title(tmp_path):
store = SQLiteSessionStore(tmp_path / "turns.db")
coordinator = MemoryCoordinator()
service, _ = _service(store, coordinator, "worker")
session = await store.ensure_session(None)
lease = await coordinator.acquire_turn("title-tail", session["id"], "worker")
await store.begin_turn(
session["id"],
turn_id=lease.turn_id,
owner_id=lease.owner_id,
fencing_token=lease.fencing_token,
)
done = await coordinator.publish_event(
lease.turn_id,
{
"type": "done",
"metadata": {"status": "completed"},
},
)
await store.append_events(lease.turn_id, [done], fencing_token=lease.fencing_token)
await store.transition_turn(
lease.turn_id, "completed", expected_status="running", fencing_token=lease.fencing_token
)
stream = service.subscribe_turn(lease.turn_id)
assert (await anext(stream))["type"] == "done"
waiting = asyncio.create_task(anext(stream))
try:
await asyncio.sleep(0.05)
assert not waiting.done(), "terminal status is not the end of the owned title tail"
title = await coordinator.publish_event(
lease.turn_id,
{
"type": "session_meta",
"metadata": {"title": "Cell membrane"},
},
)
await store.append_events(lease.turn_id, [title], fencing_token=lease.fencing_token)
await coordinator.release_turn(lease)
await coordinator.acknowledge_expired_turn(lease.turn_id)
assert (await asyncio.wait_for(waiting, 2))["metadata"]["title"] == "Cell membrane"
finally:
waiting.cancel()
await asyncio.gather(waiting, return_exceptions=True)
await stream.aclose()
@pytest.mark.asyncio
async def test_replacement_endpoint_workflow_starts_once_and_branches_admitted_user(
submission_service,
):
service, store, _, _, calls = submission_service
session, original_turn = await service.start_turn(
{"content": "Original", "client_submission_id": "already-admitted"}
)
[event async for event in service.subscribe_turn(original_turn["id"])]
payload = {
"content": "Edited",
"client_submission_id": "branch-edit",
"session_id": session["id"],
}
receipt = await service.replace_submission("already-admitted", payload, start=True)
assert receipt["status"] == "started"
[event async for event in service.subscribe_turn(receipt["turn"]["id"])]
replay = await service.replace_submission("already-admitted", payload, start=True)
assert replay["turn"]["id"] == receipt["turn"]["id"]
users = [row for row in await store.get_messages(session["id"]) if row["role"] == "user"]
assert [row["content"] for row in users] == ["Original", "Edited"]
assert users[0]["parent_message_id"] == users[1]["parent_message_id"]
assert calls["count"] == 2
@pytest.mark.asyncio
async def test_edit_admitted_before_user_persistence_waits_for_cancellation(submission_service):
service, store, _, _, calls = submission_service
sid = await store.reserve_submission("before-user", "original-digest")
original = await store.begin_turn(sid, submission_id="before-user")
receipt = await service.replace_submission(
"before-user",
{
"content": "Edited",
"client_submission_id": "edit-before-user",
"session_id": sid,
},
start=True,
)
assert (await store.get_turn(original["id"]))["status"] in {"cancelled", "failed"}
[event async for event in service.subscribe_turn(receipt["turn"]["id"])]
assert calls["count"] == 1
assert [row["content"] for row in await store.get_messages(sid) if row["role"] == "user"] == [
"Edited"
]
@pytest.mark.asyncio
async def test_reserved_edit_replays_exact_request_after_lost_receipt(submission_service):
service, store, _, _, calls = submission_service
payload = {
"content": "Edited",
"client_submission_id": "edited",
"tools": ["reason"],
"config": {},
"attachments": [],
}
receipt = await service.replace_submission("unsent", payload)
assert await service.replace_submission("unsent", payload) == receipt
with pytest.raises(RuntimeError, match="superseded"):
await service.start_turn({"content": "Original", "client_submission_id": "unsent"})
session, turn = await service.start_turn({**payload, "session_id": receipt["session_id"]})
[event async for event in service.subscribe_turn(turn["id"])]
replay_session, replay_turn = await service.start_turn(payload)
assert replay_session["id"] == session["id"]
assert replay_turn["id"] == turn["id"]
assert calls["count"] == 1
@pytest.mark.asyncio
async def test_lost_first_session_ack_replays_the_same_submission_after_worker_restart(
submission_service,
):
service, store, coordinator, _registry, calls = submission_service
payload = {"content": "Hello", "client_submission_id": "lost-first-ack"}
first_session, first_turn = await service.start_turn(payload)
[event async for event in service.subscribe_turn(first_turn["id"])]
restarted_store = SQLiteSessionStore(store.db_path)
restarted, _ = _service(restarted_store, coordinator, "worker-b")
second_session, second_turn = await restarted.start_turn(payload)
assert second_session["id"] == first_session["id"]
assert second_turn["id"] == first_turn["id"]
assert calls["count"] == 1
assert len(await store.list_sessions()) == 1
assert [row["role"] for row in await store.get_messages(first_session["id"])] == [
"user",
"assistant",
]
@pytest.mark.asyncio
async def test_failed_first_submission_resends_in_its_original_conversation_without_a_duplicate_user(
submission_service,
):
service, store, _coordinator, _registry, calls = submission_service
payload = {"content": "Hello", "client_submission_id": "failed-first-ack"}
calls["fail"] = True
first_session, failed_turn = await service.start_turn(payload)
[event async for event in service.subscribe_turn(failed_turn["id"])]
original_user = await store.get_last_message(first_session["id"], role="user")
assert original_user is not None
calls["fail"] = False
session, retried_turn = await service.start_turn(payload)
events = [event async for event in service.subscribe_turn(retried_turn["id"])]
assert session["id"] == first_session["id"]
assert retried_turn["id"] != failed_turn["id"]
rows = await store.get_messages(session["id"])
assert [row["role"] for row in rows] == ["user", "assistant"]
assert rows[0]["id"] == original_user["id"]
done = next(event for event in events if event["type"] == "done")
assert done["metadata"]["user_message_id"] == original_user["id"]
assert calls["count"] == 2
@pytest.mark.asyncio
async def test_submission_key_cannot_be_reused_for_different_content(submission_service):
service, store, _coordinator, _registry, calls = submission_service
payload = {"content": "Hello", "client_submission_id": "same-key"}
session, turn = await service.start_turn(payload)
[event async for event in service.subscribe_turn(turn["id"])]
with pytest.raises(RuntimeError, match="different request"):
await service.start_turn({**payload, "content": "Changed input"})
assert len(await store.list_sessions()) == 1
assert calls["count"] == 1
@pytest.mark.asyncio
async def test_second_worker_reads_shared_event_tail_without_mutating_turn(tmp_path) -> None:
db_path = tmp_path / "chat_history.db"
owner_store = SQLiteSessionStore(db_path)
subscriber_store = SQLiteSessionStore(db_path)
coordinator = MemoryCoordinator(lease_ttl_seconds=30)
owner_service, owner_registry = _service(owner_store, coordinator, "worker-a")
subscriber_service, _subscriber_registry = _service(subscriber_store, coordinator, "worker-b")
session = await owner_store.ensure_session("shared-session")
owner_runtime = owner_registry.get(owner_store)
turn_id = "turn-shared-events"
lease = await coordinator.acquire_turn(
turn_id,
f"{owner_runtime._coordination_scope}:{session['id']}",
"worker-a",
)
assert lease is not None
await owner_store.begin_turn(
session["id"],
capability="chat",
turn_id=turn_id,
owner_id="worker-a",
fencing_token=lease.fencing_token,
)
await coordinator.publish_event(
turn_id,
{
"type": "content",
"content": "from worker a",
"session_id": session["id"],
},
)
await coordinator.publish_event(
turn_id,
{
"type": "done",
"content": "",
"session_id": session["id"],
"metadata": {"status": "completed"},
},
)
active = await subscriber_service.check_active_turn(session["id"])
assert active == {
"turn_id": turn_id,
"status": "running",
"owner_id": "worker-a",
}
persisted = await subscriber_store.get_turn(turn_id)
assert persisted is not None and persisted["status"] == "running"
events = [event async for event in subscriber_service.subscribe_turn(turn_id)]
assert [(event["seq"], event["type"]) for event in events] == [
(1, "content"),
(2, "done"),
]
await owner_store.append_events(turn_id, events, fencing_token=lease.fencing_token)
await owner_store.transition_turn(
turn_id,
"completed",
expected_status="running",
fencing_token=lease.fencing_token,
)
await coordinator.release_turn(lease)
del owner_service
@pytest.mark.asyncio
async def test_durable_done_remains_terminal_when_post_turn_metadata_follows(tmp_path) -> None:
store = SQLiteSessionStore(tmp_path / "chat_history.db")
coordinator = MemoryCoordinator(lease_ttl_seconds=30)
service, _registry = _service(store, coordinator, "worker-a")
session = await store.ensure_session("post-turn-metadata")
turn = await store.begin_turn(session["id"], capability="chat")
await store.append_events(
turn["id"],
[
{
"type": "done",
"metadata": {"status": "completed"},
"session_id": session["id"],
"turn_id": turn["id"],
},
{
"type": "session_meta",
"stage": "title",
"content": "Recovered title",
"metadata": {"title": "Recovered title"},
"session_id": session["id"],
"turn_id": turn["id"],
},
],
)
assert await store.update_turn_status(turn["id"], "completed") is True
events = [event async for event in service.subscribe_turn(turn["id"], after_seq=0)]
assert [(event["seq"], event["type"]) for event in events] == [
(1, "done"),
(2, "session_meta"),
]
@pytest.mark.asyncio
async def test_legacy_terminal_row_synthesizes_protocol_valid_done(tmp_path) -> None:
store = SQLiteSessionStore(tmp_path / "chat_history.db")
coordinator = MemoryCoordinator(lease_ttl_seconds=30)
service, _registry = _service(store, coordinator, "worker-a")
session = await store.ensure_session("legacy-terminal")
turn = await store.begin_turn(session["id"], capability="chat")
assert await store.update_turn_status(turn["id"], "completed") is True
events = [event async for event in service.subscribe_turn(turn["id"], after_seq=0)]
assert len(events) == 1
assert events[0]["type"] == "done"
assert events[0]["seq"] == 1
assert isinstance(events[0]["timestamp"], float)
assert events[0]["metadata"]["synthesized"] is True
@pytest.mark.asyncio
async def test_second_worker_cancel_is_consumed_by_owner_worker(tmp_path) -> None:
db_path = tmp_path / "chat_history.db"
owner_store = SQLiteSessionStore(db_path)
remote_store = SQLiteSessionStore(db_path)
coordinator = MemoryCoordinator(lease_ttl_seconds=30)
_owner_service, owner_registry = _service(owner_store, coordinator, "worker-a")
remote_service, _remote_registry = _service(remote_store, coordinator, "worker-b")
session = await owner_store.ensure_session("shared-session")
owner_runtime = owner_registry.get(owner_store)
turn_id = "turn-remote-cancel"
lease = await coordinator.acquire_turn(
turn_id,
f"{owner_runtime._coordination_scope}:{session['id']}",
"worker-a",
)
assert lease is not None
await owner_store.begin_turn(
session["id"],
capability="chat",
turn_id=turn_id,
owner_id="worker-a",
fencing_token=lease.fencing_token,
)
execution = _TurnExecution(
turn_id=turn_id,
session_id=session["id"],
capability="chat",
payload={},
lease=lease,
)
execution.task = asyncio.create_task(asyncio.Event().wait())
owner_runtime._executions[turn_id] = execution
execution.coordination_task = asyncio.create_task(
owner_runtime._coordinate_execution(execution)
)
assert await remote_service.cancel_turn(turn_id, command_id="cancel-once") is True
# A retry of an already accepted command is acknowledged as success so a
# client that lost the first ACK can safely retire its outbox entry.
assert await remote_service.cancel_turn(turn_id, command_id="cancel-once") is True
with pytest.raises(asyncio.CancelledError):
await asyncio.wait_for(execution.task, timeout=1)
await asyncio.wait_for(execution.coordination_task, timeout=1)
# Command submission and observation are not allowed to write status; the
# real owner coroutine performs that transition in its cancellation path.
persisted = await remote_store.get_turn(turn_id)
assert persisted is not None and persisted["status"] == "running"
await owner_store.transition_turn(
turn_id,
"cancelled",
expected_status="running",
fencing_token=lease.fencing_token,
)
await coordinator.release_turn(lease)
@pytest.mark.asyncio
async def test_replacement_availability_matches_configured_store_before_mutation(
submission_service, monkeypatch
):
from fastapi import HTTPException
from deeptutor.api.routers import sessions
from deeptutor.core.turn_request import TurnRequest
service, store, _, _, calls = submission_service
monkeypatch.setattr(sessions, "get_session_store", lambda: store)
assert (await sessions.session_store_capabilities()).submission_replacement
unsupported = SimpleNamespace()
monkeypatch.setattr(sessions, "get_session_store", lambda: unsupported)
monkeypatch.setattr(service, "_resolve", lambda: (unsupported, None))
monkeypatch.setattr(sessions, "_turn_application_service", lambda: service)
assert not (await sessions.session_store_capabilities()).submission_replacement
body = sessions.SubmissionReplacementRequest(
original_submission_id="original",
request=TurnRequest(content="Edited", client_submission_id="edited"),
)
with pytest.raises(HTTPException) as error:
await sessions.replace_submission(body)
assert error.value.status_code == 501
assert calls["count"] == 0
@pytest.mark.asyncio
@pytest.mark.parametrize("interruption", ["append", "cancel", "publish"])
@pytest.mark.parametrize("after_seq", [0, 1, 10])
async def test_interrupted_worker_loss_recovery_replays_failure(
tmp_path, monkeypatch, interruption, after_seq
):
from deeptutor.runtime.coordination.recovery import TurnRecoveryService
store = SQLiteSessionStore(tmp_path / "interrupted.db")
coordinator = MemoryCoordinator()
session = await store.ensure_session(None)
turn = await store.begin_turn(session["id"], capability="chat")
await store.append_events(turn["id"], [{"type": "content", "content": "Partial answer"}])
async def interrupt(*args, **kwargs):
if interruption == "cancel":
raise asyncio.CancelledError
raise RuntimeError("Injected recovery interruption")
with monkeypatch.context() as patch:
if interruption == "publish":
patch.setattr(coordinator, "publish_event", interrupt)
else:
patch.setattr(store, "append_events", interrupt)
recovery = TurnRecoveryService(coordinator, store)
if interruption == "cancel":
with pytest.raises(asyncio.CancelledError):
await recovery.recover_once()
else:
await recovery.recover_once()
assert (await store.get_turn(turn["id"]))["status"] == "failed"
# A restarted coordinator cannot rediscover an already terminal row.
restarted = MemoryCoordinator()
assert await TurnRecoveryService(restarted, store).recover_once() == 0
service, _ = _service(store, restarted, "replacement")
async def collect():
return [event async for event in service.subscribe_turn(turn["id"], after_seq)]
events = await asyncio.wait_for(collect(), timeout=2)
assert [event["type"] for event in events][-2:] == ["error", "done"]
assert events[-2]["content"]
assert events[-2]["metadata"]["turn_terminal"] is True
for event in events[-2:]:
assert event["metadata"]["error_code"] == "worker_lost"
assert event["metadata"]["retryable"] is True
assert event["turn_id"] == turn["id"]
assert event["seq"] > after_seq
assert events[-1]["seq"] == events[-2]["seq"] + 1
if after_seq == 0:
assert events[0]["content"] == "Partial answer"