Skip to content

Self-improving AI Employees for everyone

5.4 KB
Raw Blame
"""The chat run watchdog: ``ledger.sweep`` every ``runs.watchdog_interval_s``
(config/chat_defaults.json).

main.py starts it once the database is up and stops it at shutdown. The
first sweep runs at once, so a run a crash left open ends once the server
is back; a pending run's wait counts from this process's start, so the
restart itself does not expire it.
"""

from __future__ import annotations

import asyncio
from typing import Any, Optional

from core.logging import get_logger
from services.chat import ledger
from services.chat.config import runs_setting

logger = get_logger(__name__)


async def temporal_workflow_closed(workflow_id: str, run_id: Optional[str]) -> Optional[bool]:
    """Whether the Temporal workflow has closed: True once it is no longer
    running (or Temporal no longer knows it), False while it runs, None when
    Temporal cannot be asked."""
    try:
        from core.container import container

        wrapper = container.temporal_client()
    except Exception:  # noqa: BLE001 - Temporal is optional
        return None
    client = getattr(wrapper, "client", None) if wrapper is not None else None
    if client is None:
        return None
    from temporalio.client import WorkflowExecutionStatus
    from temporalio.service import RPCError, RPCStatusCode

    try:
        description = await client.get_workflow_handle(workflow_id).describe()
        # Continue-as-new closes a run, not the execution chain. Compare the
        # claimant's chain when it differs so a reused ID cannot keep a chat alive.
        current_run = getattr(description, "run_id", None)
        if run_id and current_run and current_run != run_id:
            current_first = getattr(getattr(description, "raw_info", None), "first_run_id", None)
            if current_first != run_id:
                try:
                    original = await client.get_workflow_handle(workflow_id, run_id=run_id).describe()
                except RPCError:
                    # The older run may have aged out while its chain lives.
                    return None
                original_first = getattr(getattr(original, "raw_info", None), "first_run_id", None)
                if not current_first or not original_first:
                    return None
                if current_first != original_first:
                    return True
    except RPCError as exc:
        return True if exc.status == RPCStatusCode.NOT_FOUND else None
    except Exception:  # noqa: BLE001 - unknown is not closed
        return None
    return description.status != WorkflowExecutionStatus.RUNNING


async def cancel_temporal_workflow(workflow_id: str, run_id: Optional[str]) -> None:
    """Ask Temporal to cancel a workflow (a stopped or timed-out run's). Does
    nothing when Temporal cannot be reached; a workflow that already closed is
    fine."""
    try:
        from core.container import container

        wrapper = container.temporal_client()
    except Exception:  # noqa: BLE001 - Temporal is optional
        return
    client = getattr(wrapper, "client", None) if wrapper is not None else None
    if client is None:
        return
    from temporalio.service import RPCError

    try:
        await client.get_workflow_handle(workflow_id, run_id=run_id).cancel()
    except RPCError:
        # Already closed, or unknown: nothing left to cancel.
        return


class ChatRunWatchdog:
    def __init__(self, database: Any, *, interval: Optional[float] = None) -> None:
        self.database = database
        self.interval = float(interval if interval is not None else runs_setting("watchdog_interval_s"))
        self._task: Optional[asyncio.Task] = None

    def start(self) -> None:
        if self._task is None or self._task.done():
            self._task = asyncio.get_running_loop().create_task(self._loop(), name="chat-run-watchdog")

    async def stop(self) -> None:
        task, self._task = self._task, None
        if task is None or task.done():
            return
        task.cancel()
        try:
            await task
        except asyncio.CancelledError:
            pass

    async def _loop(self) -> None:
        while True:
            try:
                await ledger.sweep(
                    self.database,
                    temporal_status=temporal_workflow_closed,
                    temporal_cancel=cancel_temporal_workflow,
                )
            except asyncio.CancelledError:
                raise
            except Exception:  # noqa: BLE001 - the next sweep tries again
                logger.warning("Chat run sweep failed", exc_info=True)
            # Drafts held for the owner end here once past their expiry, an
            # approved send that never started (or waited for Resume) starts
            # once its employee runs, and a turn's wait whose nudge was lost
            # is nudged again (services/approvals/reconcile.py).
            from services.approvals.reconcile import expire_due, recover_sends, renudge_turns

            for step in (expire_due, recover_sends, renudge_turns):
                try:
                    await step(self.database)
                except asyncio.CancelledError:
                    raise
                except Exception:  # noqa: BLE001 - the next round tries again
                    logger.warning("Approvals housekeeping failed", step=step.__name__, exc_info=True)
            await asyncio.sleep(self.interval)


__all__ = ["ChatRunWatchdog", "cancel_temporal_workflow", "temporal_workflow_closed"]