Skip to content

Self-improving AI Employees for everyone

17.7 KB
Raw Blame
"""Authoritative admission for direct Workspace tasks. Never accepts client graphs."""

from __future__ import annotations
import asyncio
import hashlib
import json
from typing import Any
from uuid import UUID, uuid4


# How long a request waits for the Workspace controller's Update (1b.15). A
# submit then answers "still pending"; a reset raises UpdatePending, which
# Reset answers as workflow_control_transition_pending.
SUBMIT_UPDATE_SECONDS = 15
RESET_UPDATE_SECONDS = 240


def invocation_id(principal: str, workflow_id: str, node_id: str, submission_id: str) -> str:
    submission_id = str(UUID(submission_id))
    identity = json.dumps([principal, workflow_id, node_id, submission_id], separators=(",", ":"))
    return "node-invoke-" + hashlib.sha256(identity.encode()).hexdigest()


def temporal_client() -> Any:
    from core.container import container
    from services.plugin import NodeUserError

    wrapper = container.temporal_client()
    if wrapper is None or wrapper.client is None:
        raise NodeUserError("Workflow engine is not ready. Try again after startup completes.")
    return wrapper.client


def controller_id(workflow_id: str) -> str:
    return "workspace-tasks-" + hashlib.sha256(workflow_id.encode()).hexdigest()


async def controller_status(workflow_id: str, *, client=None) -> dict | None:
    from services.temporal.workspace_tasks_workflow import WorkspaceTaskControllerWorkflow
    from temporalio.service import RPCError, RPCStatusCode

    client = client or temporal_client()
    try:
        return await client.get_workflow_handle(controller_id(workflow_id)).query(WorkspaceTaskControllerWorkflow.describe)
    except RPCError as exc:
        if exc.status == RPCStatusCode.NOT_FOUND:
            return None
        raise


async def _controller_update(workflow_id: str, update, argument, *, update_id: str, seconds: float):
    """Send an Update to the Workspace controller, waiting at most ``seconds``
    (``UpdatePending`` past it; the Update may still be applied later)."""
    from core.config import Settings
    from services.temporal.update_calls import bounded_update
    from services.temporal.workspace_tasks_workflow import WorkspaceTaskControllerWorkflow
    from temporalio.client import WithStartWorkflowOperation, WorkflowUpdateFailedError
    from temporalio.common import WorkflowIDConflictPolicy
    from services.plugin import NodeUserError

    try:
        return await bounded_update(temporal_client().execute_update_with_start_workflow(
            update, argument, id=update_id,
            start_workflow_operation=WithStartWorkflowOperation(
                WorkspaceTaskControllerWorkflow.run, {"workflow_id": workflow_id},
                id=controller_id(workflow_id), task_queue=Settings().temporal_task_queue,
                id_conflict_policy=WorkflowIDConflictPolicy.USE_EXISTING,
            ),
        ), seconds)
    except WorkflowUpdateFailedError as exc:
        cause = exc.cause
        messages = {
            "WorkspaceResetInProgress": "Workspace tasks are resetting. Wait for Reset to finish.",
            "WorkspaceAdmissionChanged": "This submission crossed a Reset. Submit a new task.",
            "WorkspaceSubmissionConflict": "Submission ID was already used for a different task",
            "WorkspaceTaskQueueFull": "The Workspace task queue is full",
        }
        for _ in range(8):
            if getattr(cause, "type", None) in messages:
                raise NodeUserError(messages[cause.type]) from None
            if getattr(cause, "cause", None) is None:
                break
            cause = cause.cause
        raise


async def workspace_task_nodes(database, workflow_id: str, control=None) -> tuple[dict, list[dict]]:
    """Include frozen nodes removed from the editable graph while paused."""
    from services.node_registry import get_node_class

    saved = await database.get_workflow(workflow_id)
    graph = (saved.data if hasattr(saved, "data") else saved.get("data", saved)) if saved else {}
    # Filter before merging: replacing an old phone node with a non-device
    # node in the editable graph must not hide its frozen cleanup hook.
    candidates = [*(getattr(control, "graph_snapshot", None) or {}).get("nodes", []), *graph.get("nodes", [])]
    nodes = {(node["id"], node["type"]): node for node in candidates
             if node.get("id") and getattr(get_node_class(node.get("type", "")), "workspace_task", False)}
    return graph, list(nodes.values())


async def reset(workflow_id: str, request_id: str) -> dict:
    """Durably cancel admitted tasks and await plugin cleanup before acknowledging.

    Waits at most ``RESET_UPDATE_SECONDS`` and then raises ``UpdatePending``:
    the cleanup goes on, and the next Reset joins it."""
    from services.temporal.workspace_tasks_workflow import WorkspaceTaskControllerWorkflow

    state = await controller_status(workflow_id)
    # A failed cleanup keeps its admission fence. Resume that operation even
    # when the user's Retry Reset has a new transport request identity.
    request_id = (state or {}).get("reset_request_id") or request_id
    result = await _controller_update(
        workflow_id, WorkspaceTaskControllerWorkflow.reset, request_id,
        update_id="reset:" + uuid4().hex, seconds=RESET_UPDATE_SECONDS,
    )
    if state is not None:
        from services.llm_wait import close_diagnostic_waits
        await close_diagnostic_waits(workflow_id, scope="workspace", workspace_epoch=state.get("epoch", 0))
    return result


async def cancel_legacy_invocations(workflow_id: str) -> int:
    """Migration path for pre-controller roots; Visibility is not new-task admission."""
    from services.temporal.node_invocation import NodeInvocationWorkflow
    from temporalio.service import RPCError, RPCStatusCode

    client = temporal_client()
    targets = []
    async for execution in client.list_workflows(query="WorkflowType='NodeInvocationWorkflow' AND ExecutionStatus='Running'"):
        # New children are cancelled by the durable controller, not a visibility scan.
        if getattr(execution, "parent_id", None):
            continue
        handle = client.get_workflow_handle(execution.id, run_id=execution.run_id)
        try:
            metadata = await handle.query(NodeInvocationWorkflow.describe)
            if metadata.get("workflow_id") == workflow_id:
                targets.append(handle)
        except RPCError as exc:
            if exc.status != RPCStatusCode.NOT_FOUND:
                raise
    await asyncio.gather(*(_cancel_and_wait(handle) for handle in targets))
    return len(targets)


async def _cancel_and_wait(handle) -> None:
    from temporalio.client import WorkflowFailureError
    from temporalio.exceptions import is_cancelled_exception
    from temporalio.service import RPCError, RPCStatusCode

    try:
        await handle.cancel()
        try:
            await asyncio.wait_for(handle.result(), timeout=180)
        except WorkflowFailureError as exc:
            # Any already-terminal failure has closed the activity's workflow.
            # Unexpected RPC/timeouts still propagate; they are not cleanup receipts.
            if not is_cancelled_exception(exc.cause):
                description = await handle.describe()
                from temporalio.client import WorkflowExecutionStatus
                if description.status == WorkflowExecutionStatus.RUNNING:
                    raise
    except RPCError as exc:
        if exc.status != RPCStatusCode.NOT_FOUND:
            raise


async def submit(principal: str, workflow_id: str, node_id: str, prompt: str, submission_id: str) -> dict:
    from core.config import Settings
    from services.authz.workflow_node import resolve_workflow_node
    from services.node_registry import get_node_class
    from services.plugin import NodeUserError
    from services.plugin.deps import get_database
    from services.temporal.node_invocation import NodeInvocationWorkflow
    from services.temporal.update_calls import UpdatePending
    from services.temporal.workspace_tasks_workflow import WorkspaceTaskControllerWorkflow

    # Establish the admission epoch before any saved-graph/authorization await.
    # Reads are harmless; authorization still completes before any mutation.
    # A Reset during graph resolution must reject this original request.
    admission = await controller_status(workflow_id)
    saved, graph, node = await resolve_workflow_node(principal, workflow_id, node_id)
    cls = get_node_class(node["type"])
    if cls is None or not getattr(cls, "workspace_task", False):
        raise NodeUserError("This node does not accept direct Workspace tasks")
    if (node.get("data") or {}).get("disabled"):
        raise NodeUserError("Enable the node before submitting a task")
    if node["type"] == "coding_agent":
        from services.coding_agent_runtime import assert_pi_admissions_enabled
        assert_pi_admissions_enabled()
    if not prompt.strip() or len(prompt) > 20000:
        raise NodeUserError("Task must contain between 1 and 20000 characters")
    database = get_database()
    control = await database.get_latest_workflow_control(workflow_id)
    if (admission or {}).get("resetting") or (control and control.status == "resetting"):
        raise NodeUserError("Workspace tasks are resetting. Wait for Reset to finish.")
    run_id = invocation_id(principal, workflow_id, node_id, submission_id)
    fingerprint = hashlib.sha256(prompt.encode()).hexdigest()
    params = {**(await get_database().get_node_parameters(node_id) or {}), "prompt": prompt}
    settings = Settings()
    context = {
        "node_id": node_id,
        "node_type": node["type"],
        "node_data": params,
        "workflow_id": workflow_id,
        "workflow_slug": getattr(saved, "name", "workflow"),
        "execution_id": run_id,
        "session_id": run_id,
        "user_id": principal,
        "nodes": graph.get("nodes", []),
        "edges": graph.get("edges", []),
        "generation": 0,
        "graphVersion": graph.get("graphVersion", 1),
        "inputs": {},
    }
    payload = {
        "principal": principal,
        "workflow_id": workflow_id,
        "node_id": node_id,
        "fingerprint": fingerprint,
        "activity": f"node.{cls.type}.v{cls.version}",
        "task_queue": cls.task_queue if settings.temporal_worker_pool_enabled else settings.temporal_task_queue,
        "timeout_s": int(cls.start_to_close_timeout.total_seconds()),
        "context": context,
        "admission_epoch": (admission or {}).get("epoch", 0),
    }
    from services.agent_runtime_policy import runtime_behavior_version
    if runtime_behavior_version(params):
        context.update(invocation_id=run_id, workspace_epoch=payload["admission_epoch"])
    if node["type"] == "coding_agent":
        from services.workflow_sanitizer import sanitize_runtime_payload
        snapshot = {saved_node["id"]: await database.get_node_parameters(saved_node["id"]) or {}
                    for saved_node in graph.get("nodes", []) if saved_node.get("id")}
        context.update(pi_harness_version=1, workspace_task_prompt=prompt,
                       native_workspace_version=1, parameter_snapshot=snapshot,
                       temporal_worker_pool_enabled=settings.temporal_worker_pool_enabled)
        from services.coding_agent_runtime import freeze_pi_local_control
        freeze_pi_local_control(context)
        context.update(sanitize_runtime_payload(context))
        payload.update(dispatch_version=1, dispatch_kind="pi_agent")
    if node["type"] == "browser_agent":
        if getattr(settings, "distributed_mode", False) is True:
            from services.credentials.preflight import assert_cluster_credentials
            await assert_cluster_credentials(database, graph, context)
        from services.browser_tasks import browser_tool_for_agent
        from services.browser_owners import routing_for_node
        browser_node = browser_tool_for_agent(graph, node_id)
        binding = await routing_for_node(database, workflow_id, browser_node["id"], principal)
        snapshot = {saved_node["id"]: await database.get_node_parameters(saved_node["id"]) or {}
                    for saved_node in graph.get("nodes", []) if saved_node.get("id")}
        from services.temporal.agent_activities import _strip_credentials
        snapshot = _strip_credentials(snapshot)
        context["node_data"] = _strip_credentials(context["node_data"])
        context["nodes"] = _strip_credentials(context["nodes"])
        context.update(native_workspace_version=1, workspace_task_prompt=prompt,
                       parameter_snapshot=snapshot, temporal_worker_pool_enabled=settings.temporal_worker_pool_enabled,
                       browser_bindings={browser_node["id"]: binding}, _browser_task_id=run_id)
        payload.update(dispatch_version=1, dispatch_kind="native_agent", history_version=1,
                       history_record={"invocation_id": run_id, "submission_id": str(UUID(submission_id)),
                                       "principal": principal, "workflow_id": workflow_id, "node_id": node_id,
                                       "fingerprint": fingerprint, "prompt": prompt,
                                       "browser_owner_ids": [binding["owner_id"]] if binding.get("task_queue") else []})
    client = temporal_client()
    # The controller owns submission idempotence. A new Update ID permits a
    # retry after queue-full/reset rejection; a completed failed Update would
    # otherwise permanently cache that transient admission result. The same
    # submission ID again is safe after "still pending": the controller
    # dedupes it.
    try:
        admitted = await _controller_update(
            workflow_id, WorkspaceTaskControllerWorkflow.submit, payload,
            update_id=f"submit:{run_id}:{uuid4().hex}", seconds=SUBMIT_UPDATE_SECONDS,
        )
    except UpdatePending:
        raise NodeUserError("Submitted; still pending. Check the task list before submitting again.") from None
    if admitted.get("status") == "duplicate":
        from temporalio.service import RPCError, RPCStatusCode
        try:
            existing = await client.get_workflow_handle(run_id).query(NodeInvocationWorkflow.describe)
            existing_fingerprint = existing["fingerprint"]
        except RPCError as exc:
            if exc.status != RPCStatusCode.NOT_FOUND or node["type"] != "browser_agent":
                raise
            from services.workspace_task_history import get_task, TERMINAL
            record = await get_task(database, principal, workflow_id, run_id)
            if record is None or record.status not in TERMINAL:
                raise NodeUserError("This Workspace task was not found") from None
            existing_fingerprint = record.fingerprint
        if existing_fingerprint != fingerprint:
            raise NodeUserError("Submission ID was already used for a different task") from None
    from services.deployment.handlers import _with_runtime_counts
    from services.deployment.control import serialize_control
    from services.status_broadcaster import get_status_broadcaster
    try:
        await get_status_broadcaster().broadcast({
            "type": "workflow_control_status", "workflow_id": workflow_id,
            "data": await _with_runtime_counts(serialize_control(control), workflow_id),
        })
    except Exception:
        # Admission is already committed. Poll/reconnect can recover status;
        # a best-effort UI notification must not report submission failure.
        from core.logging import get_logger
        get_logger(__name__).warning("Workspace admission notification failed", workflow_id=workflow_id, exc_info=True)
    return {"run_id": run_id, "submission_id": submission_id, "status": "accepted"}


async def status(principal: str, workflow_id: str, node_id: str, submission_id: str, *, cancel: bool = False) -> dict:
    from services.authz.workflow_node import resolve_workflow_node
    from services.temporal.node_invocation import NodeInvocationWorkflow
    from services.plugin import NodeUserError
    from temporalio.service import RPCError, RPCStatusCode

    from services.plugin.deps import get_database
    from services.workspace_task_history import get_task
    database = get_database()
    record = await get_task(database, principal, workflow_id, invocation_id(principal, workflow_id, node_id, submission_id))
    if record is None:
        # Legacy device tasks have no history projection and retain saved-node authorization.
        await resolve_workflow_node(principal, workflow_id, node_id)
    handle = temporal_client().get_workflow_handle(invocation_id(principal, workflow_id, node_id, submission_id))
    try:
        if cancel:
            from services.llm_wait import close_diagnostic_waits
            await close_diagnostic_waits(workflow_id, scope="workspace",
                invocation_id=invocation_id(principal, workflow_id, node_id, submission_id))
            await _cancel_and_wait(handle)
        result = await handle.query(NodeInvocationWorkflow.describe)
    except RPCError as exc:
        if exc.status == RPCStatusCode.NOT_FOUND:
            if record is not None and record.status in ("completed", "failed", "cancelled"):
                from services.workspace_task_history import serialize
                return serialize(record)
            raise NodeUserError("This Workspace task was not found") from None
        raise NodeUserError("Workflow engine is temporarily unavailable") from None
    if result.get("status") in ("completed", "failed"):
        try:
            result["result"] = await handle.result()
        except Exception:
            result["status"] = "failed"
    if record is not None:
        from services.workspace_task_history import task_availability, safe_result
        result.update(await task_availability(database, record.browser_owner_ids))
        if "result" in result:
            result["result"] = safe_result(result["result"])
    return result