Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

27.5 KB
Raw Blame
"""One application service used by WebSocket, CLI, and Python SDK adapters."""

from __future__ import annotations

import asyncio
from collections.abc import AsyncIterator
import contextlib
import hashlib
import json
import time
from typing import Any

from deeptutor.runtime.coordination import RuntimeCoordinator
from deeptutor.services.session.protocol import ActiveTurnConflict, SessionStoreProtocol
from deeptutor.services.session.turn_runtime import TurnRuntimeManager
from deeptutor.services.workspace.activity import workspace_writer

from .contracts import TurnRequest


class TurnApplicationService:
    @staticmethod
    def _submission_digest(payload: dict[str, Any]) -> str:
        identity = {
            key: value
            for key, value in payload.items()
            if key not in {"session_id", "workspace_id"}
        }
        return hashlib.sha256(
            json.dumps(identity, sort_keys=True, default=str).encode()
        ).hexdigest()

    @workspace_writer
    async def replace_submission(
        self, original_id: str, payload: TurnRequest | dict[str, Any], *, start: bool = False
    ) -> dict[str, Any]:
        request = (
            payload if isinstance(payload, TurnRequest) else TurnRequest.model_validate(payload)
        )
        if (
            not request.client_submission_id
            or not request.persist_user_message
            or request.regenerate
        ):
            raise RuntimeError("An edited submission requires a new user message identity.")
        from deeptutor.services.workspace.context import get_workspace_scope, workspace_context

        wire = request.to_payload()
        if get_workspace_scope() is None and "workspace_id" in wire:
            with workspace_context(wire.get("workspace_id")):
                return await self.replace_submission(original_id, request, start=start)
        store, _ = self._resolve()
        replace = getattr(store, "replace_submission", None)
        if not callable(replace):
            raise NotImplementedError("This session store does not support submission replacement.")
        receipt = await replace(
            original_id,
            request.client_submission_id,
            self._submission_digest(wire),
            request.session_id,
        )
        if (
            receipt["status"] == "admitted"
            and receipt["turn"]["status"] in self._ACTIVE_TURN_STATUSES
        ):
            if not await self.cancel_turn_and_wait(receipt["turn"]["id"]):
                raise RuntimeError("The original turn has not stopped yet. Please retry the edit.")
            # Re-read only after the owner finishes: user persistence and
            # cancellation can overlap before the original row exists.
            receipt = await replace(
                original_id,
                request.client_submission_id,
                self._submission_digest(wire),
                receipt["session_id"],
            )
        if not start:
            return receipt
        wire["session_id"] = receipt["session_id"]
        if receipt["status"] == "admitted":
            if receipt["user_message_id"] is not None:
                messages = await store.get_messages(receipt["session_id"])
                original = next(
                    (row for row in messages if row["id"] == receipt["user_message_id"]), None
                )
                if original is None:
                    raise RuntimeError("The original message changed. Refresh before editing it.")
                wire["parent_message_id"] = original.get("parent_message_id")
            elif receipt["turn"]["status"] not in {"failed", "cancelled"}:
                raise RuntimeError("The admitted message is not yet available for editing.")
        session, turn = await self.start_turn(wire)
        return {
            "status": "started",
            "session_id": session["id"],
            "submission_id": request.client_submission_id,
            "turn": turn,
        }

    def __init__(
        self,
        store_provider: Any,
        runtime_registry: Any,
        coordinator: RuntimeCoordinator,
    ) -> None:
        self.store_provider = store_provider
        self.runtime_registry = runtime_registry
        self.coordinator = coordinator

    def _resolve(self) -> tuple[SessionStoreProtocol, TurnRuntimeManager]:
        store = self.store_provider.get()
        return store, self.runtime_registry.get(store)

    @workspace_writer
    async def start_turn(
        self, payload: TurnRequest | dict[str, Any]
    ) -> tuple[dict[str, Any], dict[str, Any]]:
        request = (
            payload if isinstance(payload, TurnRequest) else TurnRequest.model_validate(payload)
        )
        payload = request.to_payload()
        from deeptutor.services.workspace.context import get_workspace_scope, workspace_context

        # SDK/CLI callers may specify the scope in the request instead of
        # wrapping their application in workspace_context. Resolve it before
        # the store/runtime, and let the spawned turn inherit this context.
        if get_workspace_scope() is None and "workspace_id" in payload:
            with workspace_context(payload.get("workspace_id")):
                return await self.start_turn(payload)
        store, runtime = self._resolve()
        reserve = getattr(store, "reserve_submission", None)
        submission_turn = getattr(store, "submission_turn", None)
        if (
            request.client_submission_id
            and request.persist_user_message
            and not request.regenerate
            and callable(reserve)
            and callable(submission_turn)
        ):
            digest = self._submission_digest(payload)
            payload["session_id"] = await reserve(
                request.client_submission_id, digest, request.session_id
            )
            prior = await submission_turn(request.client_submission_id)
            if prior is not None and prior["status"] not in {"failed", "cancelled"}:
                session = await store.get_session(payload["session_id"])
                if session is None:
                    raise RuntimeError("Conversation not found in this workspace.")
                return session, prior
        try:
            session, turn = await runtime.start_turn(payload)
        except ActiveTurnConflict:
            if request.client_submission_id and callable(reserve) and callable(submission_turn):
                prior = await submission_turn(request.client_submission_id)
                if prior is not None:
                    session = await store.get_session(prior["session_id"])
                    if session is not None:
                        return session, prior
            # Retry once, and only after something was actually reclaimed, so a
            # session busy with a live turn still gets its conflict.
            session_id = str(payload.get("session_id") or "")
            if not session_id or not await self._reclaim_unowned_active_turn(session_id):
                raise
            session, turn = await runtime.start_turn(payload)
        await store.update_session_preferences(
            session["id"],
            {
                "language": str(payload.get("language") or "en"),
                "notebook_references": list(payload.get("notebook_references") or []),
                "history_references": list(payload.get("history_references") or []),
                "book_references": list(payload.get("book_references") or []),
                "reading_references": list(payload.get("reading_references") or []),
                "question_notebook_references": list(
                    payload.get("question_notebook_references") or []
                ),
                "knowledge_bases": list(payload.get("knowledge_bases") or []),
                "partner_group_references": list(payload.get("partner_group_references") or []),
            },
        )
        # Keep historical source references even when the current turn no
        # longer selects them. Migration must preserve earlier citations too.
        previous = (session.get("preferences") or {}).get("workspace_dependencies", {})
        dependencies = {}
        for key in (
            "notebook_references",
            "history_references",
            "book_references",
            "reading_references",
            "question_notebook_references",
            "knowledge_bases",
        ):
            values = list(previous.get(key) or [])
            for value in list((session.get("preferences") or {}).get(key) or []) + list(
                payload.get(key) or []
            ):
                if value not in values:
                    values.append(value)
            dependencies[key] = values
        await store.update_session_preferences(
            session["id"], {"workspace_dependencies": dependencies}
        )
        return session, turn

    async def regenerate_last_turn(
        self,
        session_id: str,
        overrides: dict[str, Any] | None = None,
    ) -> tuple[dict[str, Any], dict[str, Any]]:
        _store, runtime = self._resolve()
        try:
            return await runtime.regenerate_last_turn(session_id, overrides=overrides)
        except ActiveTurnConflict:
            if not await self._reclaim_unowned_active_turn(session_id):
                raise
            return await runtime.regenerate_last_turn(session_id, overrides=overrides)

    # How long to keep reading after DONE before closing the stream.
    #
    # DONE is not a turn's last event. The runtime deliberately publishes
    # post-turn metadata after it — notably the LLM-written session title,
    # which ``SessionTitleService`` emits once the answer is saved so the
    # composer and the duration clock stop immediately rather than waiting on
    # the title model. Both loops below used to ``return`` the instant they
    # saw DONE, so every post-DONE event was dropped: the title landed in
    # ``turn_events`` and in the database while no subscriber ever received
    # it, which is why a finished conversation could sit on "New conversation"
    # indefinitely. The frontend even holds its socket open for 15s waiting
    # for that frame; it never had a chance to arrive.
    #
    # End of stream is the turn's lease disappearing, not a timeout.
    #
    # The turn task releases its lease in its own ``finally``, which runs after
    # every post-turn event has been published. So "lease gone" is an honest
    # end-of-stream signal and needs no guessing about how long a title model
    # might take — an idle timeout would have to exceed that model's own 20s
    # ceiling to be safe, and would then hold every finished stream open for
    # 20s whenever no title was written at all.
    #
    # The cap below exists only for a leaked lease; it is not the normal path.
    _POST_DONE_MAX_SECONDS = 30.0
    _CANCEL_WAIT_INTERVAL_SECONDS = 0.05
    _ACTIVE_TURN_STATUSES = frozenset({"queued", "running", "waiting_input"})

    @staticmethod
    def _done_has_tail(event: dict[str, Any]) -> bool:
        """Can more events follow this DONE?"""
        status = str((event.get("metadata") or {}).get("status") or "completed")
        return status == "completed"

    async def subscribe_turn(
        self,
        turn_id: str,
        after_seq: int = 0,
    ) -> AsyncIterator[dict[str, Any]]:
        store, _runtime = self._resolve()
        if await store.get_turn(turn_id) is None:
            return
        last_seq = max(0, int(after_seq))
        done = False
        tail_possible = False

        # Durable replay covers a process/Redis restart. Shared-stream replay
        # then fills the live tail owned by any worker.
        for event in await store.get_events(turn_id, last_seq):
            seq = int(event.get("seq") or 0)
            if seq <= last_seq:
                continue
            last_seq = seq
            if str(event.get("type") or "") == "done":
                done = True
                tail_possible = self._done_has_tail(event)
            yield event
        if done and not tail_possible:
            return
        # A completed row can still have an owned title tail. Drain it below,
        # including durable events committed after the initial replay (#1971).

        first_done_at: float | None = time.monotonic() if done else None
        while True:
            emitted = False
            events = await self.coordinator.read_events(turn_id, last_seq)
            if not events and await self.coordinator.get_lease(turn_id) is None:
                # The completed in-memory stream may have been reclaimed
                # while this subscriber was paused. Its durable tail remains.
                events = await store.get_events(turn_id, last_seq)
            for event in events:
                seq = int(event.get("seq") or 0)
                if seq <= last_seq:
                    continue
                emitted = True
                last_seq = seq
                if str(event.get("type") or "") == "done":
                    done = True
                    tail_possible = self._done_has_tail(event)
                    if first_done_at is None:
                        first_done_at = time.monotonic()
                yield event
            if done and not tail_possible:
                return
            if done:
                now = time.monotonic()
                if first_done_at is None:
                    first_done_at = now
                # The turn task publishes its post-turn events before releasing
                # the lease, so once the lease is gone the tail is complete.
                if await self.coordinator.get_lease(turn_id) is None:
                    return
                if now - first_done_at >= self._POST_DONE_MAX_SECONDS:
                    return
                if not emitted:
                    await self._wait_for_turn_events(turn_id, last_seq)
                continue
            turn = await store.get_turn(turn_id)
            if turn is None:
                return
            status = str(turn.get("status") or "")
            if status in {"completed", "failed", "cancelled"}:
                # Completion can commit before the owner publishes DONE. Do
                # not synthesize an id-less envelope in that window (#1971).
                if await self.coordinator.get_lease(turn_id) is not None:
                    await asyncio.sleep(0.1)
                    continue
                # Ownership/cleanup can change between the reads above.
                if await store.get_events(turn_id, last_seq):
                    continue
                # Recovery can commit FAILED before its terminal journal is
                # written. A client may also have seen volatile events beyond
                # the durable cursor. In either case preserve the failure
                # envelope: DONE alone hides errors on partial responses.
                metadata = {
                    "status": status,
                    "synthesized": True,
                    "error_code": str(turn.get("failure_code") or ""),
                    "retryable": bool(turn.get("retryable")),
                }
                if status == "failed":
                    last_seq += 1
                    yield {
                        "type": "error",
                        "source": "turn_application",
                        "stage": "",
                        "content": str(turn.get("error") or "Turn failed; retry to continue"),
                        "metadata": {**metadata, "turn_terminal": True},
                        "session_id": turn.get("session_id", ""),
                        "turn_id": turn_id,
                        "seq": last_seq,
                        "timestamp": time.time(),
                    }
                yield {
                    "type": "done",
                    "source": "turn_application",
                    "stage": "",
                    "content": "",
                    "metadata": metadata,
                    "session_id": turn.get("session_id", ""),
                    "turn_id": turn_id,
                    "seq": last_seq + 1,
                    "timestamp": time.time(),
                }
                return
            if not emitted:
                await self._wait_for_turn_events(turn_id, last_seq)

    async def _wait_for_turn_events(self, turn_id: str, after_seq: int) -> None:
        # Notifications accelerate the live path; they never replace durable
        # replay or the periodic terminal-state checks. Older/distributed
        # coordinator adapters can keep their existing polling behavior.
        wait = getattr(self.coordinator, "wait_for_events", None)
        if wait is None:
            await asyncio.sleep(0.1)
        else:
            await wait(turn_id, after_seq, timeout=0.5)

    async def subscribe_session(
        self,
        session_id: str,
        after_seq: int = 0,
    ) -> AsyncIterator[dict[str, Any]]:
        active = await self.check_active_turn(session_id)
        turn_id = str((active or {}).get("turn_id") or "")
        if not turn_id:
            return
        async for event in self.subscribe_turn(turn_id, after_seq):
            yield event

    async def check_active_turn(self, session_id: str) -> dict[str, Any] | None:
        store, _runtime = self._resolve()
        turn = await store.get_active_turn(session_id)
        if turn is None:
            return None
        turn_id = str(turn.get("id") or turn.get("turn_id") or "")
        lease = await self.coordinator.get_lease(turn_id)
        return {
            "turn_id": turn_id,
            "status": str(turn.get("status") or "running") if lease else "recovering",
            "owner_id": lease.owner_id if lease else str(turn.get("owner_id") or ""),
        }

    @workspace_writer
    async def cancel_turn(self, turn_id: str, *, command_id: str | None = None) -> bool:
        store, _runtime = self._resolve()
        turn = await store.get_turn(turn_id)
        if turn is None or turn.get("status") not in {
            "queued",
            "running",
            "waiting_input",
        }:
            return False
        if await self.coordinator.get_lease(turn_id) is None:
            # Nobody owns it, so a queued cancel would never be read. Stopping
            # a turn nothing is executing is what the learner asked for, and
            # the row is what blocks the session — reap it here for the same
            # reason ``submit_user_reply`` does, and report the stop as done
            # rather than as a refusal the UI cannot act on.
            return await self._reap_unowned_live_turn(turn_id)
        await self.coordinator.submit_command(turn_id, "cancel", {}, command_id=command_id)
        # A duplicate command ID means the mutation was already accepted. The
        # WebSocket adapter must acknowledge that retry as success so a client
        # that lost the first ACK can retire its durable outbox entry.
        return True

    @workspace_writer
    async def cancel_turn_and_wait(
        self,
        turn_id: str,
        *,
        command_id: str | None = None,
        timeout_seconds: float = 15.0,
    ) -> bool:
        """Stop a live turn and wait until its owner has finished teardown."""
        if not await self.cancel_turn(turn_id, command_id=command_id):
            return False
        store, _runtime = self._resolve()
        deadline = time.monotonic() + max(0.0, float(timeout_seconds))
        released = False
        while True:
            turn = await store.get_turn(turn_id)
            status = str((turn or {}).get("status") or "")
            lease = await self.coordinator.get_lease(turn_id)
            if turn is not None and status not in self._ACTIVE_TURN_STATUSES and lease is None:
                released = True
                break
            if turn is None and lease is None:
                released = True
                break
            if time.monotonic() >= deadline:
                break
            await asyncio.sleep(self._CANCEL_WAIT_INTERVAL_SECONDS)
        if released:
            # Lease release happens just before the owner's activity guard
            # exits. Leave one polling interval so deletion does not race that
            # final unwind.
            await asyncio.sleep(self._CANCEL_WAIT_INTERVAL_SECONDS)
        return released

    @workspace_writer
    async def submit_user_reply(
        self,
        turn_id: str,
        text: str | None = None,
        *,
        answers: list[dict[str, Any]] | None = None,
        command_id: str | None = None,
    ) -> bool:
        store, _runtime = self._resolve()
        if await store.get_turn(turn_id) is None:
            return False
        if await self.coordinator.get_lease(turn_id) is None:
            # Nobody owns the turn: a queued command would never be read. If
            # the durable row still says ``waiting_input`` it is a zombie —
            # the worker that owned the waiter is gone, and without this the
            # session is blocked forever while the card sits on screen
            # (#1297). Reap it synchronously (the background recovery pass
            # would only sweep it on its next tick) so the client's ack
            # rejection comes with a terminal state to recover from.
            await self._reap_unowned_live_turn(turn_id)
            return False
        await self.coordinator.submit_command(
            turn_id,
            "submit_user_reply",
            {"text": text or "", "answers": answers},
            command_id=command_id,
        )
        return True

    async def _reclaim_unowned_active_turn(self, session_id: str) -> bool:
        """Clear the rows blocking this session that nothing is executing.

        ``_begin_turn_sync`` refuses a new turn while the session owns any row
        in ``queued``/``running``/``waiting_input``. That is right while a turn
        is alive and catastrophic once one is not: a single row left behind by
        a lost worker or a restart blocks *every* later message, with no way
        out from the UI (#1297, #1359). Users reported exactly that — one
        window, no refresh, and a conversation that never accepts another word.

        Liveness is the lease, never the row's age: a turn parked on an
        ``ask_user`` card emits no events, so ``updated_at`` stops advancing
        while it legitimately waits for a person to type. A sweep on age would
        cancel that reader mid-answer. A live worker renews its lease every few
        seconds whether or not it is producing output, so "no lease" is the one
        signal that separates a zombie from a slow human.

        This synchronous path avoids waiting for the background recovery
        pass when a new request is already blocked. Admission acquires its
        lease before inserting a turn, so a live new row is protected even
        before its first renewal.

        Returns ``True`` when something was reaped and the caller should retry.
        """
        store, _runtime = self._resolve()
        try:
            active = await store.list_active_turns(session_id)
        except Exception:
            return False
        reclaimed = False
        for row in active:
            turn_id = str(row.get("id") or row.get("turn_id") or "")
            if not turn_id:
                continue
            if await self.coordinator.get_lease(turn_id) is not None:
                # Something is genuinely executing it. The conflict is real and
                # the caller should see it.
                continue
            if await self._reap_unowned_live_turn(turn_id):
                reclaimed = True
        return reclaimed

    async def _reap_unowned_live_turn(self, turn_id: str) -> bool:
        """Fail a persisted live turn that has no live lease.

        ``queued``, ``running`` and ``waiting_input`` are all rows
        ``_begin_turn_sync`` counts as active, so any of them left behind by a
        lost worker blocks the whole session, not just the card on screen.

        Mirrors :class:`TurnRecoveryService`'s terminal write (CAS on status
        with the fencing token, ``worker_lost`` failure code, error+done
        events) so whatever raced us wins cleanly and subscribed clients
        stop deterministically. Returns ``True`` when a zombie was reaped.
        """
        store, _runtime = self._resolve()
        try:
            turn = await store.get_turn(turn_id)
        except Exception:
            return False
        if turn is None or str(turn.get("status") or "") not in {
            "queued",
            "running",
            "waiting_input",
        }:
            return False
        error = "The worker executing this turn was lost; resend your answer as a new message"
        metadata = {
            "turn_terminal": True,
            "status": "failed",
            "error_code": "worker_lost",
            "retryable": True,
        }
        reap_event: dict[str, Any] = {
            "type": "error",
            "source": "turn_recovery",
            "stage": "recovery",
            "content": error,
            "metadata": metadata,
            "session_id": turn.get("session_id", ""),
        }
        done_event: dict[str, Any] = {
            "type": "done",
            "source": "turn_recovery",
            "stage": "recovery",
            "content": "",
            "metadata": {"status": "failed", "error_code": "worker_lost", "retryable": True},
            "session_id": turn.get("session_id", ""),
        }
        try:
            transitioned = await store.transition_turn(
                turn_id,
                "failed",
                expected_status=str(turn["status"]),
                fencing_token=int(turn.get("fencing_token") or 0),
                error=error,
                failure_code="worker_lost",
                retryable=True,
            )
            if not transitioned:
                # A recovery pass or a late executor write beat us to it.
                return False
            await store.append_events(
                turn_id,
                [reap_event, done_event],
                fencing_token=int(turn.get("fencing_token") or 0),
            )
        except Exception:
            return False
        for event in (reap_event, done_event):
            with contextlib.suppress(Exception):
                await self.coordinator.publish_event(turn_id, event)
        return True

    async def submit_user_input(
        self,
        turn_id: str,
        content: str,
        *,
        command_id: str | None = None,
        withdraw: str | None = None,
    ) -> bool:
        store, _runtime = self._resolve()
        if (
            await store.get_turn(turn_id) is None
            or await self.coordinator.get_lease(turn_id) is None
        ):
            return False
        await self.coordinator.submit_command(
            turn_id,
            "user_input",
            {"content": content, "withdraw": withdraw},
            command_id=command_id,
        )
        return True

    async def list_sessions(self, limit: int = 50, offset: int = 0) -> list[dict[str, Any]]:
        store, _runtime = self._resolve()
        return await store.list_sessions(limit=limit, offset=offset)

    async def get_session(self, session_id: str) -> dict[str, Any] | None:
        store, _runtime = self._resolve()
        return await store.get_session_with_messages(session_id)

    async def rename_session(self, session_id: str, title: str) -> bool:
        store, _runtime = self._resolve()
        return await store.update_session_title(session_id, title)

    async def delete_session(self, session_id: str) -> bool:
        store, _runtime = self._resolve()
        active = await store.list_active_turns(session_id)
        for turn in active:
            await self.cancel_turn(str(turn["id"]))
        if active:
            return False
        return await store.delete_session(session_id)


__all__ = ["TurnApplicationService"]