Skip to content

Self-improving AI Employees for everyone

fix(temporal): app-process Update calls answer pending instead of hanging (1b.15)

trohitg03504492622b
diff --git a/client/src/features/home/__tests__/workspace.test.tsx b/client/src/features/home/__tests__/workspace.test.tsx
index d41fd2a9..e4e89584 100644
--- a/client/src/features/home/__tests__/workspace.test.tsx
+++ b/client/src/features/home/__tests__/workspace.test.tsx
@@ -7,7 +7,7 @@
*/
import { beforeEach, describe, expect, it, vi } from 'vitest';
-import { act, fireEvent, render, screen, within } from '@testing-library/react';
+import { act, fireEvent, render, screen, waitFor, within } from '@testing-library/react';
import userEvent from '@testing-library/user-event';
import { QueryClient, QueryClientProvider } from '@tanstack/react-query';
import type { ReactElement } from 'react';
@@ -40,6 +40,7 @@ vi.mock('@/contexts/WebSocketContext', async (importOriginal) => ({
}));
vi.mock('../../../app/useShellActions', () => ({ enterDev: vi.fn() }));
+vi.mock('../ui/pillToast', () => ({ pillToast: vi.fn() }));
// The browser's Take over handle (components/workspace/surface.ts), and the
// way to give back what a claim took.
const screenControl = vi.hoisted(() => ({ claim: vi.fn(), giveBack: vi.fn() }));
@@ -64,6 +65,7 @@ import { EMPLOYEES_QUERY_KEY } from '../data/employees';
import { parseEmployee, type EmployeeSummary } from '../data/schemas';
import { SPIKE, orbState } from '../orb/orb';
import { WORKSPACE_MIN_WIDTH, loadWorkspacePrefs, useHomeStore } from '../state/homeStore';
+import { pillToast } from '../ui/pillToast';
import { WorkspaceButton } from '../workspace/WorkspaceButton';
import { WorkspaceDock } from '../workspace/WorkspaceDock';
// Loaded up front so the dock's lazy import resolves from the module cache:
@@ -203,6 +205,19 @@ describe('WorkspaceDock', () => {
expect(screen.getByRole('button', { name: 'Stopping…' })).toBeDisabled();
});
+ it('does not say stopped while the stop is still draining', async () => {
+ // The server answers a Stop within 10 s even while work drains (1b.15).
+ vi.mocked(pillToast).mockClear();
+ actions.pauseWorkflow.mockResolvedValue(normalizeWorkflowControlStatus({ generation: 1, state: 'pausing', revision: 5 }, 'w1'));
+ useHomeStore.setState({ workspaceOpen: true, workspaceTab: 'browser' });
+ renderWith([employee({ control: normalizeWorkflowControlStatus({ generation: 1, state: 'running', revision: 4 }, 'w1') })]);
+ fireEvent.click(screen.getByRole('button', { name: 'Stop' }));
+ await waitFor(() => expect(actions.pauseWorkflow).toHaveBeenCalledWith('w1', 4));
+ await act(async () => {});
+ expect(pillToast).not.toHaveBeenCalled();
+ expect(screen.getByRole('button', { name: 'Stopping…' })).toBeDisabled();
+ });
+
it('follows the employee last opened, else the first', () => {
useHomeStore.setState({ workspaceOpen: true, workspaceTab: 'browser' });
renderWith([employee(), employee({ workflow_id: 'w2', name: 'Leo' })]);
diff --git a/client/src/features/home/employee/useEmployeeControl.ts b/client/src/features/home/employee/useEmployeeControl.ts
index 7185bb07..e1542c48 100644
--- a/client/src/features/home/employee/useEmployeeControl.ts
+++ b/client/src/features/home/employee/useEmployeeControl.ts
@@ -90,8 +90,11 @@ export function useEmployeeControl(employee: EmployeeSummary, onConnect: (provid
: kind === 'resume'
? actions.resumeWorkflow(id, current.revision)
: actions.startEmployee(id, current.revision);
- setAwaiting({ kind, target: await run });
- pillToast(`${employee.name} ${CONTROL_DONE[kind]}`);
+ const target = await run;
+ setAwaiting({ kind, target });
+ // A Stop or Resume whose drain is still running answers early: the
+ // label keeps saying so, and no toast claims it is done.
+ if (target.state !== 'pausing' && target.state !== 'resuming') pillToast(`${employee.name} ${CONTROL_DONE[kind]}`);
if (kind !== 'pause') spikeOrb(SPIKE.start);
} catch (error) {
if (error instanceof Error && error.message === 'workflow_control_superseded') return;
diff --git a/docs-internal/TEMPORAL_ARCHITECTURE.md b/docs-internal/TEMPORAL_ARCHITECTURE.md
index 42a12b39..553ec2e6 100644
--- a/docs-internal/TEMPORAL_ARCHITECTURE.md
+++ b/docs-internal/TEMPORAL_ARCHITECTURE.md
@@ -755,7 +755,7 @@ posts to `/api/browser/tasks` (`server/routers/browser_tasks.py`). Both use
independent of deployed Start/Stop/Resume, and have their own Cancel and Reset
cleanup.
-- **Admission.** Each OpenCompany workflow has one long-lived `WorkspaceTaskControllerWorkflow` (`services/temporal/workspace_tasks_workflow.py`, Temporal id from `node_invocations.controller_id`), created on first use through Update-With-Start. Its `submit` Update checks the admission epoch, bounds active tasks, deduplicates the submission UUID and fingerprint, and starts one `NodeInvocationWorkflow` child per task. New Browser payloads carry server-generated `dispatch_version: 1`, `dispatch_kind: native_agent` and `history_version: 1`; their idempotent history admission Activity completes before acceptance is acknowledged. Unversioned payloads retain the original single per-type Activity command path with no retry.
+- **Admission.** Each OpenCompany workflow has one long-lived `WorkspaceTaskControllerWorkflow` (`services/temporal/workspace_tasks_workflow.py`, Temporal id from `node_invocations.controller_id`), created on first use through Update-With-Start. Its `submit` Update checks the admission epoch, bounds active tasks, deduplicates the submission UUID and fingerprint, and starts one `NodeInvocationWorkflow` child per task. New Browser payloads carry server-generated `dispatch_version: 1`, `dispatch_kind: native_agent` and `history_version: 1`; their idempotent history admission Activity completes before acceptance is acknowledged. Unversioned payloads retain the original single per-type Activity command path with no retry. The app waits 15 s for the `submit` Update; past that the submission answers "Submitted; still pending", and resubmitting the same UUID is safe.
- **Native Browser execution.** The invocation starts `AgentWorkflow`, using the existing model/tool Activities. Preparation applies the submitted task after saved configuration and templates while preserving system instructions. `native_workspace_version: 1` permits Continue-As-New with generation-zero task identity, frozen tools, policies, owner bindings and transcript, without enrolling direct tasks in generation control or repeating completed turns. The task does not read/write deployed Context or employee chat. Completion is counted once at the invocation parent, including cancellation/error outcomes; agent children and rollover do not add completions.
- **Browser history.** `WorkspaceTaskRecord` is a projection, not the execution authority. Admission and lifecycle writes are idempotent and terminal states cannot regress. Recent tasks contain prompt, timestamps, bounded safe result/error and authorized relative artifact links, excluding tool transcripts and credentials. Pages default to 20 (maximum 100); terminal records expire after 35 days, while active records and history survive Reset. Workflow authorization protects historical records after node deletion. Retained terminal records also prevent resubmission after controller dedup or Temporal history expiration.
- **Reset.** The controller's `reset` Update first fences new submissions by advancing the epoch. It then cancels every admitted child and waits for each to close, and runs the `workspace_tasks.reset_runtime` activity (`services/temporal/workspace_task_activities.py::reset_workspace_task_runtime`). That activity cancels invocations started before the controller existed and calls each Workspace-task node's `reset_execution_state`. The Update returns only after that cleanup. If cleanup fails, the fence stays in place, and retrying Reset resumes the same request. The workflow toolbar's Reset reaches the controller through `services/deployment/handlers.py::_reset_workspace_tasks`, even for a workflow that was never started; see [temporal-workflow-control.md](./temporal-workflow-control.md#workspace-tasks).
diff --git a/docs-internal/browser_agent_deployment.md b/docs-internal/browser_agent_deployment.md
index 56077f41..d57ffa18 100644
--- a/docs-internal/browser_agent_deployment.md
+++ b/docs-internal/browser_agent_deployment.md
@@ -230,7 +230,9 @@ All routes use application authentication and saved-workflow authorization.
| `DELETE /api/browser/tasks/{submission_id}` | The same authorized identity; requests Cancel and waits for confirmed cleanup |
| `GET /api/browser/tasks/history` | `workflow_id`, optional agent `node_id`, opaque `cursor` and `limit` between 1 and 100 |
-Reuse the submission UUID after a lost acknowledgement. A changed prompt or
+Reuse the submission UUID after a lost acknowledgement, or after the 400
+"Submitted; still pending. Check the task list before submitting again." that
+answers a submission the controller has not acknowledged within 15 s. A changed prompt or
binding under the same identity fails the fingerprint check. Browser tools
must be saved and connected before admission; clients cannot provide graphs,
credential sources, owner URLs or CDP endpoints. Temporal remains authoritative
diff --git a/docs-internal/chat_protocol.md b/docs-internal/chat_protocol.md
index 9fa6913d..3f06d7d3 100644
--- a/docs-internal/chat_protocol.md
+++ b/docs-internal/chat_protocol.md
@@ -831,7 +831,7 @@ and server tests alike:
| `idempotency_key_required` / `expected_revision_required` | controlled stop_chat_run | The generation mutation lacks its request identity or revision. |
| `control_generation_conflict` | controlled stop_chat_run, Resume | The run's owning root differs from the generation currently being controlled. Refresh control; do not target the successor using the old run. |
| `control_revision_conflict` | controlled stop_chat_run, Resume | The generation revision changed; the response/status supplies the authoritative control. This revision is distinct from the thread's revision. |
-| `workflow_control_transition_pending` | controlled stop_chat_run, Resume | A durable lifecycle transition is still in progress. Reconcile control status; the chat run remains live. |
+| `workflow_control_transition_pending` | controlled stop_chat_run, Resume, Reset | A durable lifecycle transition is still in progress. Reconcile control status; the chat run remains live. A version 1 Stop or Resume whose drain outlasts 10 s answers success with `transition_pending: true` instead; Reset answers this code when Workspace cleanup has not answered in 240 s, with the row still `resetting`. |
| `revision_conflict` | edit, regenerate, switch | `expected_revision` is not the thread's revision. |
| `rule_conflict` | set_ask_first | `expected_revision` is stale; the answer carries the current value. |
| `not_editable` | edit, regenerate, decide | Not the owner's text message a run answered, not the latest answer, the editor's `default` chat, or an edited argument that is not editable. |
diff --git a/docs-internal/temporal-workflow-control.md b/docs-internal/temporal-workflow-control.md
index e95485a1..6cfb4525 100644
--- a/docs-internal/temporal-workflow-control.md
+++ b/docs-internal/temporal-workflow-control.md
@@ -85,6 +85,18 @@ for its checkpoint. A disconnected socket cancels only its local waiter; the
supervised transition and durable admission fence remain. Reset retires older
local checkpoint polling and reports the newer control state to those waiters.
+No request waits on the engine without a bound
+(`services/temporal/update_calls.bounded_update`). A version 1 Stop or Resume
+answers within 10 s (`_TRANSITION_ANSWER_SECONDS`): while its drain still
+runs, the answer is `success: true` with `transition_pending: true` and the
+`pausing` or `resuming` state, the drain goes on in its task, and its
+`workflow_control_status` broadcast carries the outcome. Reset waits up to
+240 s for the Workspace task controller's cleanup; past that it answers
+`workflow_control_transition_pending` and the row stays `resetting` until the
+same Reset runs again. Safe Apply waits 30 s for each controller Update and
+otherwise answers `waiting`; `recover_applies` finishes it once its claim's
+lease lapses.
+
## Acknowledged Stop for new generations
New generations atomically store `resource_manifest.execution_control_version=1`.
diff --git a/server/services/deployment/handlers.py b/server/services/deployment/handlers.py
index 8cb63cde..aa8d3df7 100644
--- a/server/services/deployment/handlers.py
+++ b/server/services/deployment/handlers.py
@@ -75,8 +75,9 @@ class ControllerExecutionMissing(RuntimeError):
# Maps workflow_id -> asyncio.Task for parallel workflow deployments.
_deployment_tasks: Dict[str, asyncio.Task] = {}
-# A socket only waits on these tasks. Disconnecting a waiter must not undo a
-# durable Stop, and status polling must never start a second checkpoint drain.
+# A socket only waits on these tasks, and at most _TRANSITION_ANSWER_SECONDS.
+# Disconnecting a waiter must not undo a durable Stop, and status polling must
+# never start a second checkpoint drain.
_control_transition_tasks: Dict[tuple[str, int], asyncio.Task] = {}
_boot_reconciliation_tasks: Dict[tuple[str, int], asyncio.Task] = {}
_control_completion_locks: weakref.WeakValueDictionary = weakref.WeakValueDictionary()
@@ -85,6 +86,9 @@ _control_completion_locks: weakref.WeakValueDictionary = weakref.WeakValueDictio
_CONTROL_OBSERVATION_TIMEOUT_SECONDS = 1.0
_BOOT_REARM_TIMEOUT_SECONDS = 30.0
_BOOT_RETRY_SECONDS = 1.0
+# A Stop or Resume request answers within this, pending while its drain runs
+# on; the transition's broadcast carries the outcome (1b.15).
+_TRANSITION_ANSWER_SECONDS = 10.0
def _control_completion_lock(workflow_id: str) -> asyncio.Lock:
@@ -132,11 +136,21 @@ def _cancel_boot_reconciliation(control) -> None:
async def _await_versioned_transition(service: WorkflowControlService, control, *, task=None) -> Dict[str, Any]:
+ """Wait for the shared transition for at most ``_TRANSITION_ANSWER_SECONDS``.
+ A drain still running answers ``transition_pending``; it goes on in its
+ task, whose broadcast carries the outcome."""
intent_revision, intent_status = control.revision, control.status
intent_identity = (control.id, control.generation)
task = task or _schedule_versioned_transition(service, control)
try:
- current, controller_status = await asyncio.shield(task)
+ # asyncio.wait never cancels the task, and never mistakes a TimeoutError
+ # the transition itself raised for its own bound.
+ await asyncio.wait({task}, timeout=_TRANSITION_ANSWER_SECONDS)
+ if task.done():
+ current, controller_status = task.result()
+ else:
+ current = await service.database.get_latest_workflow_control(control.workflow_id) or control
+ controller_status = None
except asyncio.CancelledError:
# A Reset can retire a scheduled poll before its coroutine starts.
# Cancellation of just this socket waiter leaves task alive instead.
@@ -1434,8 +1448,9 @@ async def _finish_versioned_transition(service: WorkflowControlService, control)
if isinstance(exc, WorkflowUpdateRPCTimeoutOrCancelledError):
if isinstance(exc.__cause__, asyncio.CancelledError):
raise asyncio.CancelledError() from exc
- # A bounded RPC poll can fail without rejecting the durable
- # Update. Keep intent and closed admission; status can retry it.
+ # The SDK ended its wait without rejecting the durable Update (its
+ # own RPC deadline). Keep the intent and the closed admission; a
+ # later status read drives it again.
logger.info("Workflow control acknowledgement remains pending", workflow_id=control.workflow_id,
revision=intent[2], error_type=type(exc).__name__)
return latest, None
@@ -2499,6 +2514,25 @@ async def handle_resume_workflow(data: Dict[str, Any], websocket: WebSocket) ->
@ws_handler("workflow_id")
async def handle_reset_workflow(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
+ """Reset a workflow. When the Workspace controller has not answered its
+ reset in time, the answer is ``workflow_control_transition_pending``; the
+ row stays ``resetting``, and the same Reset again finishes it."""
+ from services.temporal.update_calls import UpdatePending
+
+ try:
+ return await _reset_workflow(data, websocket)
+ except UpdatePending as exc:
+ workflow_id = data["workflow_id"]
+ service = _control_service()
+ logger.warning("Workspace cleanup has not answered; the Reset stays pending",
+ workflow_id=workflow_id, error=str(exc))
+ latest = await service.database.get_latest_workflow_control(workflow_id)
+ payload = (await _control_payload(latest) if latest
+ else await _with_runtime_counts(await service.get_status(workflow_id), workflow_id))
+ return {"success": False, "error": "workflow_control_transition_pending", **payload}
+
+
+async def _reset_workflow(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
workflow_id = data["workflow_id"]
service = _control_service()
current = await service.database.get_latest_workflow_control(workflow_id)
diff --git a/server/services/employees/safe_apply.py b/server/services/employees/safe_apply.py
index ef8fd04a..de4f152a 100644
--- a/server/services/employees/safe_apply.py
+++ b/server/services/employees/safe_apply.py
@@ -7,9 +7,21 @@ from datetime import datetime, timedelta, timezone
import uuid
from typing import Any
from sqlmodel import select
+from core.logging import get_logger
from models.employees import EmployeeApply, EmployeeJob
from models.database import AgentTeam, WorkflowControlExecution
+logger = get_logger(__name__)
+
+# How long each controller Update of an apply waits (1b.15). Past it the
+# apply answers "waiting", and recover_applies finishes it after the lease.
+UPDATE_SECONDS = 30
+
+
+def _send(handle, name: str, argument, update_id: str):
+ from services.temporal.update_calls import bounded_update
+ return bounded_update(handle.execute_update(name, argument, id=update_id), UPDATE_SECONDS)
+
async def _active(database, workflow_id):
async with database.get_session() as session:
@@ -32,7 +44,24 @@ async def _active(database, workflow_id):
async def apply_saved_changes(database: Any, workflow_id: str, *, owner_id: str, key: str, stop_work: bool = False) -> dict:
+ """Apply the saved graph to the running generation. An Update the
+ controller does not answer in ``UPDATE_SECONDS`` leaves the apply
+ ``waiting``: it is not a failure, so nothing is rolled back, and
+ ``recover_applies`` applies it again once its claim's lease lapses."""
+ from services.temporal.update_calls import UpdatePending
+
+ try:
+ return await _apply_saved_changes(database, workflow_id, owner_id=owner_id, key=key, stop_work=stop_work)
+ except UpdatePending as exc:
+ logger.warning("The workflow engine did not answer an apply Update; recovery finishes it",
+ workflow_id=workflow_id, error=str(exc))
+ return {"success": True, "activation_state": "waiting",
+ "message": "The change is saved and will apply as soon as the workflow engine answers."}
+
+
+async def _apply_saved_changes(database: Any, workflow_id: str, *, owner_id: str, key: str, stop_work: bool = False) -> dict:
from core.container import container
+ from services.temporal.update_calls import UpdatePending
from services.deployment.handlers import _controller_handle, _set_cron_pause
from services.workflow_validator import validate_workflow
from services.deployment.control import _graph_hash
@@ -104,7 +133,7 @@ async def apply_saved_changes(database: Any, workflow_id: str, *, owner_id: str,
# approvals, reset Context, or empty the controller's accepted queue.
if handle:
pause_key = identifier + ":pause" + (":" + uuid.uuid4().hex if request.state == "failed" else "")
- await handle.execute_update("pause_admissions", request.resume_after, id=pause_key)
+ await _send(handle, "pause_admissions", request.resume_after, pause_key)
await _set_cron_pause(workflow_id, paused=True, strict=True)
manager = container.workflow_service()._get_deployment_manager()
manager.pause(workflow_id)
@@ -196,7 +225,7 @@ async def apply_saved_changes(database: Any, workflow_id: str, *, owner_id: str,
await client.get_schedule_handle(description.id).update(update_schedule)
if handle:
await verify_claim()
- await handle.execute_update("replace_graph", snapshot, id=identifier + ":graph:" + claim)
+ await _send(handle, "replace_graph", snapshot, identifier + ":graph:" + claim)
if old_state:
await verify_claim()
changes = {"nodes": snapshot["nodes"], "edges": snapshot["edges"]}
@@ -228,7 +257,7 @@ async def apply_saved_changes(database: Any, workflow_id: str, *, owner_id: str,
if request.resume_after:
await verify_claim()
if handle:
- await handle.execute_update("set_control_state", "running", id=identifier + ":resume:" + claim)
+ await _send(handle, "set_control_state", "running", identifier + ":resume:" + claim)
await _set_cron_pause(workflow_id, paused=False, strict=True)
await manager.resume(workflow_id)
await restore_producers()
@@ -243,6 +272,10 @@ async def apply_saved_changes(database: Any, workflow_id: str, *, owner_id: str,
await session.commit()
employee_changed_now(workflow_id)
return {"success": True, "activation_state": "running" if request.resume_after else "paused"}
+ except UpdatePending:
+ # Not a failure: no rollback. The claim lapses with its lease and
+ # recover_applies applies the snapshot again.
+ raise
except Exception:
async with database.get_session() as session:
current_request = await session.get(EmployeeApply, identifier)
@@ -251,8 +284,8 @@ async def apply_saved_changes(database: Any, workflow_id: str, *, owner_id: str,
# Retain the last operational graph if any part of handoff fails.
if handle:
await verify_claim()
- await handle.execute_update("pause_admissions", False, id=identifier + ":rollback-pause:" + claim)
- await handle.execute_update("replace_graph", request.previous, id=identifier + ":rollback:" + claim)
+ await _send(handle, "pause_admissions", False, identifier + ":rollback-pause:" + claim)
+ await _send(handle, "replace_graph", request.previous, identifier + ":rollback:" + claim)
for schedule_id, previous_schedule in old_schedules:
await verify_claim()
await client.get_schedule_handle(schedule_id).update(lambda _input, previous_schedule=previous_schedule: ScheduleUpdate(schedule=previous_schedule))
@@ -261,7 +294,7 @@ async def apply_saved_changes(database: Any, workflow_id: str, *, owner_id: str,
if request.resume_after:
await verify_claim()
if handle:
- await handle.execute_update("set_control_state", "running", id=identifier + ":rollback-resume:" + claim)
+ await _send(handle, "set_control_state", "running", identifier + ":rollback-resume:" + claim)
await _set_cron_pause(workflow_id, paused=False, strict=True)
await manager.resume(workflow_id)
await restore_producers()
diff --git a/server/services/node_invocations.py b/server/services/node_invocations.py
index 2fe279d0..a47a7dc4 100644
--- a/server/services/node_invocations.py
+++ b/server/services/node_invocations.py
@@ -8,6 +8,13 @@ from typing import Any
from uuid import UUID, uuid4
+# How long a request waits for the Workspace controller's Update (1b.15). A
+# submit then answers "still pending"; a reset raises UpdatePending, which
+# Reset answers as workflow_control_transition_pending.
+SUBMIT_UPDATE_SECONDS = 15
+RESET_UPDATE_SECONDS = 240
+
+
def invocation_id(principal: str, workflow_id: str, node_id: str, submission_id: str) -> str:
submission_id = str(UUID(submission_id))
identity = json.dumps([principal, workflow_id, node_id, submission_id], separators=(",", ":"))
@@ -41,22 +48,25 @@ async def controller_status(workflow_id: str, *, client=None) -> dict | None:
raise
-async def _controller_update(workflow_id: str, update, argument, *, update_id: str):
+async def _controller_update(workflow_id: str, update, argument, *, update_id: str, seconds: float):
+ """Send an Update to the Workspace controller, waiting at most ``seconds``
+ (``UpdatePending`` past it; the Update may still be applied later)."""
from core.config import Settings
+ from services.temporal.update_calls import bounded_update
from services.temporal.workspace_tasks_workflow import WorkspaceTaskControllerWorkflow
from temporalio.client import WithStartWorkflowOperation, WorkflowUpdateFailedError
from temporalio.common import WorkflowIDConflictPolicy
from services.plugin import NodeUserError
try:
- return await temporal_client().execute_update_with_start_workflow(
+ return await bounded_update(temporal_client().execute_update_with_start_workflow(
update, argument, id=update_id,
start_workflow_operation=WithStartWorkflowOperation(
WorkspaceTaskControllerWorkflow.run, {"workflow_id": workflow_id},
id=controller_id(workflow_id), task_queue=Settings().temporal_task_queue,
id_conflict_policy=WorkflowIDConflictPolicy.USE_EXISTING,
),
- )
+ ), seconds)
except WorkflowUpdateFailedError as exc:
cause = exc.cause
messages = {
@@ -89,7 +99,10 @@ async def workspace_task_nodes(database, workflow_id: str, control=None) -> tupl
async def reset(workflow_id: str, request_id: str) -> dict:
- """Durably cancel admitted tasks and await plugin cleanup before acknowledging."""
+ """Durably cancel admitted tasks and await plugin cleanup before acknowledging.
+
+ Waits at most ``RESET_UPDATE_SECONDS`` and then raises ``UpdatePending``:
+ the cleanup goes on, and the next Reset joins it."""
from services.temporal.workspace_tasks_workflow import WorkspaceTaskControllerWorkflow
state = await controller_status(workflow_id)
@@ -98,7 +111,7 @@ async def reset(workflow_id: str, request_id: str) -> dict:
request_id = (state or {}).get("reset_request_id") or request_id
result = await _controller_update(
workflow_id, WorkspaceTaskControllerWorkflow.reset, request_id,
- update_id="reset:" + uuid4().hex,
+ update_id="reset:" + uuid4().hex, seconds=RESET_UPDATE_SECONDS,
)
if state is not None:
from services.llm_wait import close_diagnostic_waits
@@ -158,6 +171,7 @@ async def submit(principal: str, workflow_id: str, node_id: str, prompt: str, su
from services.plugin import NodeUserError
from services.plugin.deps import get_database
from services.temporal.node_invocation import NodeInvocationWorkflow
+ from services.temporal.update_calls import UpdatePending
from services.temporal.workspace_tasks_workflow import WorkspaceTaskControllerWorkflow
# Establish the admission epoch before any saved-graph/authorization await.
@@ -248,11 +262,16 @@ async def submit(principal: str, workflow_id: str, node_id: str, prompt: str, su
client = temporal_client()
# The controller owns submission idempotence. A new Update ID permits a
# retry after queue-full/reset rejection; a completed failed Update would
- # otherwise permanently cache that transient admission result.
- admitted = await _controller_update(
- workflow_id, WorkspaceTaskControllerWorkflow.submit, payload,
- update_id=f"submit:{run_id}:{uuid4().hex}",
- )
+ # otherwise permanently cache that transient admission result. The same
+ # submission ID again is safe after "still pending": the controller
+ # dedupes it.
+ try:
+ admitted = await _controller_update(
+ workflow_id, WorkspaceTaskControllerWorkflow.submit, payload,
+ update_id=f"submit:{run_id}:{uuid4().hex}", seconds=SUBMIT_UPDATE_SECONDS,
+ )
+ except UpdatePending:
+ raise NodeUserError("Submitted; still pending. Check the task list before submitting again.") from None
if admitted.get("status") == "duplicate":
from temporalio.service import RPCError, RPCStatusCode
try:
diff --git a/server/services/temporal/update_calls.py b/server/services/temporal/update_calls.py
new file mode 100644
index 00000000..2e31424e
--- /dev/null
+++ b/server/services/temporal/update_calls.py
@@ -0,0 +1,41 @@
+"""Bounded client Update calls (1b.15).
+
+A socket request or a recovery loop must not wait for ever on a workflow that
+does not answer: a stuck run, a busy worker, an engine that is down.
+``bounded_update`` gives the call a deadline and says "pending" when it
+passes. The Update may still be admitted and applied later, so a caller treats
+pending as "not known yet", never as refused.
+"""
+
+import asyncio
+from typing import Awaitable, TypeVar
+
+from temporalio.client import WorkflowUpdateRPCTimeoutOrCancelledError
+
+T = TypeVar("T")
+
+
+class UpdatePending(Exception):
+ """Our deadline passed first. The Update may still be admitted and applied."""
+
+
+async def bounded_update(call: Awaitable[T], seconds: float) -> T:
+ """Await an Update call for at most ``seconds``.
+
+ The SDK turns any interrupted wait into
+ ``WorkflowUpdateRPCTimeoutOrCancelledError``, our deadline included, and
+ its ``rpc_timeout`` bounds only the first RPC, so the deadline is ours. A
+ cancel of the caller's own task comes back as ``CancelledError`` (the SDK
+ cannot tell it from a timeout); an expired deadline raises
+ ``UpdatePending``; anything else passes through unchanged.
+ """
+ try:
+ async with asyncio.timeout(seconds) as deadline:
+ return await call
+ except (TimeoutError, WorkflowUpdateRPCTimeoutOrCancelledError) as exc:
+ task = asyncio.current_task()
+ if task is not None and task.cancelling():
+ raise asyncio.CancelledError() from exc
+ if deadline.expired():
+ raise UpdatePending(f"no answer in {seconds:g} s") from exc
+ raise
diff --git a/server/tests/services/employees/test_conversion.py b/server/tests/services/employees/test_conversion.py
index 3a8713da..35d704ef 100644
--- a/server/tests/services/employees/test_conversion.py
+++ b/server/tests/services/employees/test_conversion.py
@@ -1,6 +1,7 @@
"""Reviewed conversion preserves existing identities and configuration."""
from copy import deepcopy
from types import SimpleNamespace
+from unittest.mock import AsyncMock
import pytest
import nodes as _nodes # noqa: F401 - register shipped plugins
@@ -214,6 +215,21 @@ async def test_apply_and_rollback_keep_configuration_identity_without_reset(real
assert len(calls) == 2
+async def test_a_waiting_apply_leaves_the_review_saved(real_database, monkeypatch):
+ # 1b.15: an apply the engine has not answered is not applied yet.
+ from models.employee_conversion import EmployeeConversion
+ from services.employees import safe_apply, team_runtime
+ await save_existing(real_database, "app_event")
+ monkeypatch.setattr(team_runtime, "team_runtime_error", lambda: None)
+ monkeypatch.setattr(safe_apply, "apply_saved_changes",
+ AsyncMock(return_value={"success": True, "activation_state": "waiting"}))
+ review = await plan_conversion(real_database, "81", "owner", enabled=True)
+ applied = await apply_conversion(real_database, review["review_id"], "owner", enabled=True)
+ assert applied["activation_state"] == "waiting"
+ async with real_database.get_session() as session:
+ assert (await session.get(EmployeeConversion, review["review_id"])).state == "saved"
+
+
async def test_later_owner_changes_require_new_review_before_applying(real_database, monkeypatch):
from services.employees import team_runtime
await save_existing(real_database)
diff --git a/server/tests/services/employees/test_safe_apply.py b/server/tests/services/employees/test_safe_apply.py
index f43601c1..7a44576f 100644
--- a/server/tests/services/employees/test_safe_apply.py
+++ b/server/tests/services/employees/test_safe_apply.py
@@ -297,6 +297,71 @@ async def test_live_claim_serializes_handoffs_and_renews_lease(harness, monkeypa
await first
+def _hang_on(harness, monkeypatch, hangs):
+ """1b.15: the controller never answers the Updates ``hangs`` picks."""
+ import services.deployment.handlers as handlers
+
+ class HangingHandle(harness.Handle):
+ async def execute_update(self, name, argument, **kwargs):
+ result = await super().execute_update(name, argument, **kwargs)
+ if hangs(name, kwargs.get("id") or ""):
+ await asyncio.Event().wait()
+ return result
+
+ monkeypatch.setattr(safe_apply, "UPDATE_SECONDS", 0.05)
+ monkeypatch.setattr(handlers, "_controller_handle", lambda _control: HangingHandle())
+
+
+def _answer_again(harness, monkeypatch):
+ import services.deployment.handlers as handlers
+ monkeypatch.setattr(handlers, "_controller_handle", lambda _control: harness.Handle())
+
+
+async def _request(harness, key):
+ async with harness.database.get_session() as session:
+ return await session.get(EmployeeApply, f"apply:7:{key}")
+
+
+async def test_an_unanswered_graph_update_waits_without_rollback_and_recovery_applies_it(harness, monkeypatch):
+ _hang_on(harness, monkeypatch, lambda name, _id: name == "replace_graph")
+ result = await safe_apply.apply_saved_changes(harness.database, "7", owner_id="owner", key="slow")
+ assert result["success"] is True and result["activation_state"] == "waiting"
+ assert not any(":rollback" in (update_id or "") for _name, _argument, update_id in harness.update_ids)
+ assert (await harness.database.get_latest_workflow_control("7")).graph_snapshot == harness.before
+ assert harness.manager._deployments["7"].nodes == harness.before["nodes"]
+ request = await _request(harness, "slow")
+ assert request.state == "applying" and request.claim_token
+ # The claim lapses with its lease; recovery applies the snapshot again.
+ async with harness.database.reserved_session() as session:
+ (await session.get(EmployeeApply, "apply:7:slow")).lease_until = datetime.now(timezone.utc) - timedelta(minutes=1)
+ await session.commit()
+ _answer_again(harness, monkeypatch)
+ await safe_apply.recover_applies(harness.database)
+ assert (await _request(harness, "slow")).state == "applied"
+ assert (await harness.database.get_latest_workflow_control("7")).graph_snapshot == harness.operational
+
+
+async def test_an_unanswered_admission_pause_keeps_the_apply_waiting_and_recovery_bounded(harness, monkeypatch):
+ _hang_on(harness, monkeypatch, lambda name, _id: name == "pause_admissions")
+ result = await safe_apply.apply_saved_changes(harness.database, "7", owner_id="owner", key="first")
+ assert result["activation_state"] == "waiting"
+ assert (await _request(harness, "first")).state == "waiting"
+ # The recovery loop's own call answers too, instead of stalling the loop.
+ await asyncio.wait_for(safe_apply.recover_applies(harness.database), timeout=5)
+ assert (await _request(harness, "first")).state == "waiting"
+ _answer_again(harness, monkeypatch)
+ await safe_apply.recover_applies(harness.database)
+ assert (await _request(harness, "first")).state == "applied"
+
+
+async def test_an_unanswered_rollback_leaves_the_apply_waiting_not_failed(harness, monkeypatch):
+ harness.fail = True # the forward replace_graph fails once
+ _hang_on(harness, monkeypatch, lambda _name, update_id: ":rollback-pause:" in update_id)
+ result = await safe_apply.apply_saved_changes(harness.database, "7", owner_id="owner", key="undo")
+ assert result["activation_state"] == "waiting"
+ assert (await _request(harness, "undo")).state == "applying"
+
+
async def test_restart_recovers_expired_applying_lease(harness):
async with harness.database.reserved_session() as session:
session.add(EmployeeApply(id="apply:7:expired", workflow_id="7", owner_id="owner", state="applying", resume_after=True,
diff --git a/server/tests/services/test_versioned_workflow_control.py b/server/tests/services/test_versioned_workflow_control.py
index 85e8164c..92275bdd 100644
--- a/server/tests/services/test_versioned_workflow_control.py
+++ b/server/tests/services/test_versioned_workflow_control.py
@@ -234,6 +234,39 @@ async def test_disconnect_cancels_waiter_and_preserves_durable_stop(monkeypatch)
assert finished.status == "paused" and record.row.status == "paused"
+async def test_a_stop_still_draining_answers_pending_and_the_drain_goes_on(monkeypatch):
+ # 1b.15: the socket request answers within the bound; the task carries on.
+ record = lifecycle(monkeypatch)
+ admitted, release = asyncio.Event(), asyncio.Event()
+
+ async def stuck(*args, **kwargs):
+ admitted.set()
+ await release.wait()
+ return {"controller_status": {}, "controlled_executions": 3}
+
+ monkeypatch.setattr(execution_control, "transition_generation", AsyncMock(side_effect=stuck))
+ monkeypatch.setattr(handlers, "_TRANSITION_ANSWER_SECONDS", 0.05)
+ outcome = await asyncio.wait_for(
+ handlers.handle_pause_workflow({"workflow_id": "canvas", "expected_revision": 2}, None), 5)
+ assert outcome["success"] and outcome["transition_pending"] and not outcome["superseded"]
+ assert outcome["state"] == "pausing"
+ transition_task = handlers._control_transition_tasks[(record.row.id, 3)]
+ assert admitted.is_set() and not transition_task.done()
+ release.set()
+ finished, _ = await asyncio.wait_for(asyncio.shield(transition_task), 1)
+ assert finished.status == "paused" and record.row.status == "paused"
+
+
+async def test_a_timeout_raised_by_the_transition_is_not_read_as_pending(monkeypatch):
+ lifecycle(monkeypatch)
+ monkeypatch.setattr(execution_control, "transition_generation",
+ AsyncMock(side_effect=TimeoutError("request timeout; tool still running")))
+ outcome = await handlers.handle_pause_workflow({"workflow_id": "canvas", "expected_revision": 2}, None)
+ assert outcome == {"success": False, "error": "request timeout; tool still running"}
+
+
@pytest.mark.asyncio
@pytest.mark.parametrize("intent", ["pausing", "resuming"])
async def test_late_transition_after_reset_does_not_broadcast_or_reopen(monkeypatch, intent):
diff --git a/server/tests/services/test_workspace_task_reset.py b/server/tests/services/test_workspace_task_reset.py
index 767be52e..0045cf94 100644
--- a/server/tests/services/test_workspace_task_reset.py
+++ b/server/tests/services/test_workspace_task_reset.py
@@ -77,6 +77,63 @@ async def test_direct_reset_barrier_failure_does_not_publish_success(monkeypatch
service.transition.assert_not_awaited()
[email protected]("current", [None, control()])
+async def test_a_pending_workspace_cleanup_answers_pending_and_publishes_nothing(monkeypatch, current):
+ # 1b.15: the Workspace controller's reset waits at most 240 s.
+ from services.temporal.update_calls import UpdatePending
+
+ database = SimpleNamespace(get_latest_workflow_control=AsyncMock(return_value=current))
+ service = SimpleNamespace(database=database, get_status=AsyncMock(return_value={"state": "ready"}), transition=AsyncMock())
+ monkeypatch.setattr(handlers, "_control_service", lambda: service)
+ monkeypatch.setattr(handlers, "_reset_workspace_tasks", AsyncMock(side_effect=UpdatePending("no answer in 240 s")))
+ monkeypatch.setattr(handlers, "_control_payload", AsyncMock(return_value={"state": "ready"}))
+ monkeypatch.setattr(handlers, "_with_runtime_counts", AsyncMock(return_value={"state": "ready"}))
+ notices = broadcaster(monkeypatch)
+ data = {"workflow_id": "wf", "expected_revision": current.revision if current else 0}
+ assert await handlers.handle_reset_workflow(data, None) == {
+ "success": False, "error": "workflow_control_transition_pending", "state": "ready",
+ }
+ notices.broadcast.assert_not_awaited()
+ service.transition.assert_not_awaited()
+
+
+async def test_a_pending_workspace_cleanup_keeps_a_generation_resetting(monkeypatch):
+ from services.deployment import runtime_state
+ from services.temporal.update_calls import UpdatePending
+
+ initial, resetting = control("running", 3), control("resetting", 4)
+ statuses = []
+
+ async def transition(_current, **kwargs):
+ statuses.append(kwargs["status"])
+ return resetting
+
+ database = SimpleNamespace(
+ get_latest_workflow_control=AsyncMock(side_effect=[initial, initial, resetting]),
+ update_workflow_run_data_scope=AsyncMock(),
+ )
+ service = SimpleNamespace(database=database, transition=AsyncMock(side_effect=transition))
+ archive = AsyncMock()
+ monkeypatch.setattr(handlers, "_control_service", lambda: service)
+ monkeypatch.setattr(handlers, "_close_local_admission", Mock())
+ monkeypatch.setattr(handlers, "_broadcast_control", AsyncMock(return_value={"state": "resetting"}))
+ monkeypatch.setattr(handlers, "_signal_controller", AsyncMock())
+ monkeypatch.setattr(handlers, "_delete_cron_schedules", AsyncMock(return_value=0))
+ monkeypatch.setattr(handlers, "handle_cancel_deployment", AsyncMock(return_value={"success": True}))
+ monkeypatch.setattr(handlers, "_terminate_generation_workflows", AsyncMock(return_value=0))
+ monkeypatch.setattr("services.approvals.waiting.cancel_waits", AsyncMock(return_value=0))
+ monkeypatch.setattr(handlers, "_reset_workspace_tasks", AsyncMock(side_effect=UpdatePending("no answer in 240 s")))
+ monkeypatch.setattr(handlers, "_control_payload", AsyncMock(side_effect=lambda row, **_: {"state": row.status}))
+ monkeypatch.setattr(runtime_state, "archive_and_reset_node_state", archive)
+ broadcaster(monkeypatch)
+ result = await handlers.handle_reset_workflow(
+ {"workflow_id": "wf", "expected_revision": 3, "idempotency_key": "reset-generation"}, None)
+ assert result == {"success": False, "error": "workflow_control_transition_pending", "state": "resetting"}
+ assert statuses == ["resetting"]
+ database.update_workflow_run_data_scope.assert_not_awaited()
+ archive.assert_not_awaited()
+
+
@pytest.mark.parametrize("current", [None, control()])
async def test_ordinary_idempotent_reset_has_no_workspace_reset_broadcast(monkeypatch, current):
database = SimpleNamespace(get_latest_workflow_control=AsyncMock(return_value=current))
diff --git a/server/tests/temporal/test_update_calls.py b/server/tests/temporal/test_update_calls.py
new file mode 100644
index 00000000..f58fa72c
--- /dev/null
+++ b/server/tests/temporal/test_update_calls.py
@@ -0,0 +1,62 @@
+"""1b.15: a client Update call answers "pending" instead of hanging."""
+
+from __future__ import annotations
+
+import asyncio
+
+import pytest
+from temporalio.client import WorkflowUpdateFailedError, WorkflowUpdateRPCTimeoutOrCancelledError
+from temporalio.exceptions import ApplicationError
+
+from services.temporal.update_calls import UpdatePending, bounded_update
+
+
+async def _answers():
+ return {"ok": True}
+
+
+async def _hangs():
+ await asyncio.sleep(60)
+
+
+async def _hangs_like_the_sdk():
+ # The SDK recasts the cancel of its RPC wait as its own error.
+ try:
+ await asyncio.sleep(60)
+ except asyncio.CancelledError as exc:
+ raise WorkflowUpdateRPCTimeoutOrCancelledError() from exc
+
+
+async def test_a_call_that_answers_in_time_returns_its_value():
+ assert await bounded_update(_answers(), 1) == {"ok": True}
+
+
[email protected]("call", [_hangs, _hangs_like_the_sdk])
+async def test_a_call_past_its_deadline_is_pending(call):
+ with pytest.raises(UpdatePending):
+ await bounded_update(call(), 0.01)
+
+
[email protected]("call", [_hangs, _hangs_like_the_sdk])
+async def test_a_cancel_of_the_caller_stays_a_cancel(call):
+ task = asyncio.create_task(bounded_update(call(), 30))
+ await asyncio.sleep(0.01)
+ task.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await task
+
+
+async def test_a_timeout_raised_by_the_call_itself_is_not_pending():
+ async def raises_timeout():
+ raise TimeoutError("the transition's own timeout")
+
+ with pytest.raises(TimeoutError, match="own timeout"):
+ await bounded_update(raises_timeout(), 30)
+
+
+async def test_a_failed_update_passes_through():
+ async def fails():
+ raise WorkflowUpdateFailedError(ApplicationError("refused"))
+
+ with pytest.raises(WorkflowUpdateFailedError):
+ await bounded_update(fails(), 30)
diff --git a/server/tests/test_workspace_invocations.py b/server/tests/test_workspace_invocations.py
index 8fab28e8..48062be0 100644
--- a/server/tests/test_workspace_invocations.py
+++ b/server/tests/test_workspace_invocations.py
@@ -74,6 +74,43 @@ async def test_retry_uses_same_child_but_new_update_identity(admission):
assert first.kwargs["update_id"] != second.kwargs["update_id"]
+async def test_a_submit_the_controller_has_not_answered_says_still_pending(admission):
+ # 1b.15: the request waits 15 s; the same submission ID again is safe.
+ from services.temporal.update_calls import UpdatePending
+
+ admission.update.side_effect = UpdatePending("no answer in 15 s")
+ with pytest.raises(NodeUserError, match="still pending"):
+ await service.submit("owner", "wf", "phone", "Read", SUBMISSION)
+ assert admission.update.await_args.kwargs["seconds"] == service.SUBMIT_UPDATE_SECONDS
+ admission.broadcast.assert_not_awaited()
+
+
+async def test_a_reset_the_controller_has_not_answered_stays_pending(admission, monkeypatch):
+ from services import llm_wait
+ from services.temporal.update_calls import UpdatePending
+
+ closed = AsyncMock()
+ monkeypatch.setattr(llm_wait, "close_diagnostic_waits", closed)
+ admission.update.side_effect = UpdatePending("no answer in 240 s")
+ with pytest.raises(UpdatePending):
+ await service.reset("wf", "reset-1")
+ assert admission.update.await_args.kwargs["seconds"] == service.RESET_UPDATE_SECONDS
+ closed.assert_not_awaited() # the cleanup is not known done
+
+
+async def test_the_controller_update_is_bounded(monkeypatch):
+ from core import config
+ from services.temporal.update_calls import UpdatePending
+
+ async def hangs(*_args, **_kwargs):
+ await asyncio.sleep(60)
+
+ monkeypatch.setattr(config, "Settings", lambda: SimpleNamespace(temporal_task_queue="main"))
+ monkeypatch.setattr(service, "temporal_client", lambda: SimpleNamespace(execute_update_with_start_workflow=hangs))
+ with pytest.raises(UpdatePending):
+ await service._controller_update("wf", "submit", {}, update_id="submit:1", seconds=0.01)
+
+
async def test_notification_failure_does_not_undo_committed_admission(admission):
admission.broadcast.side_effect = RuntimeError("socket unavailable")
assert (await service.submit("owner", "wf", "phone", "Read", SUBMISSION))["status"] == "accepted"