Skip to content

Self-improving AI Employees for everyone

fix(control): reconcile stale Start requests after employee Hire

trohitg8b35e6bec970
diff --git a/client/src/contexts/WebSocketContext.tsx b/client/src/contexts/WebSocketContext.tsx
index 658902c4..cb4d8c83 100644
--- a/client/src/contexts/WebSocketContext.tsx
+++ b/client/src/contexts/WebSocketContext.tsx
@@ -2921,6 +2921,19 @@ export const WebSocketProvider: React.FC<{ children: React.ReactNode }> = ({ chi
};
try {
+ if (
+ pending.action === 'start'
+ && previousStatus?.state === 'running'
+ && previousStatus.revision > Number(data.expected_revision ?? 0)
+ ) {
+ // A hire's background Start can finish while an employee summary or
+ // toolbar still carries the earlier revision. Confirm the running
+ // state without submitting another Start. Never rebase stale intent
+ // onto a newer ready generation after Reset.
+ const current = await requestWorkflowControlStatus(workflowId);
+ if (isReconciled(current)) return current;
+ throw new Error('control_revision_conflict');
+ }
return await sendMutationAttempt(data);
} catch (error) {
let reconciled: WorkflowControlStatus | undefined;
diff --git a/client/src/contexts/__tests__/webSocketLifecycle.test.tsx b/client/src/contexts/__tests__/webSocketLifecycle.test.tsx
index d5b02e54..4ac80356 100644
--- a/client/src/contexts/__tests__/webSocketLifecycle.test.tsx
+++ b/client/src/contexts/__tests__/webSocketLifecycle.test.tsx
@@ -92,6 +92,59 @@ describe('WebSocket recovery and ownership', () => {
vi.useRealTimers();
});
+ it.each(['canvas', 'employee'])('reconciles a stale %s Start after Hire without submitting another mutation', async (surface) => {
+ await mount();
+ await open();
+ const running = { workflow_id: 'ravi', state: 'running', generation: 1, revision: 2,
+ can_start: false, can_pause: true, can_reset: true };
+ act(() => latest().message({ type: 'workflow_control_status', data: running }));
+ let result!: Promise<unknown>;
+ act(() => { result = surface === 'canvas'
+ ? actions.startWorkflow('ravi', [], [], 'default', 0)
+ : actions.startEmployee('ravi', 0); });
+ const read = latest().sent.filter(item => item.type === 'get_workflow_control_status').at(-1)!;
+ expect(read).toBeDefined();
+ expect(latest().sent.some(item => ['start_workflow', 'start_employee'].includes(item.type))).toBe(false);
+ await act(async () => { latest().message({ request_id: read.request_id, success: true, ...running }); });
+ await expect(result).resolves.toMatchObject({ state: 'running', revision: 2 });
+ expect(latest().sent.some(item => ['start_workflow', 'start_employee'].includes(item.type))).toBe(false);
+ });
+
+ it('does not rebase a stale Start onto a newer ready generation after Reset', async () => {
+ await mount();
+ await open();
+ act(() => latest().message({ type: 'workflow_control_status', data: {
+ workflow_id: 'ravi', state: 'running', generation: 1, revision: 2, can_start: false,
+ } }));
+ let result!: Promise<unknown>;
+ act(() => { result = actions.startEmployee('ravi', 0).catch(error => error.message); });
+ const read = latest().sent.filter(item => item.type === 'get_workflow_control_status').at(-1)!;
+ const ready = { workflow_id: 'ravi', state: 'ready', generation: 1, revision: 4, can_start: true };
+ await act(async () => { latest().message({ request_id: read.request_id, success: true, ...ready }); });
+ const resync = latest().sent.filter(item => item.type === 'get_workflow_control_status').at(-1)!;
+ await act(async () => { latest().message({ request_id: resync.request_id, success: true, ...ready }); });
+ await expect(result).resolves.toBe('control_revision_conflict');
+ expect(latest().sent.some(item => item.type === 'start_employee')).toBe(false);
+ expect(context.workflowControlStatuses.ravi).toMatchObject({ state: 'ready', revision: 4 });
+ });
+
+ it('reconciles a Start conflict that arrives with the background hire running snapshot', async () => {
+ await mount();
+ await open();
+ let result!: Promise<unknown>;
+ act(() => { result = actions.startEmployee('ravi', 0); });
+ const start = latest().sent.find(item => item.type === 'start_employee')!;
+ expect(start.expected_revision).toBe(0);
+ const running = { workflow_id: 'ravi', state: 'running', generation: 1, revision: 2, can_start: false };
+ await act(async () => { latest().message({ request_id: start.request_id, success: false,
+ error: 'control_revision_conflict', ...running }); });
+ expect(context.workflowControlStatuses.ravi).toMatchObject({ state: 'running', revision: 2 });
+ const resync = latest().sent.filter(item => item.type === 'get_workflow_control_status').at(-1)!;
+ await act(async () => { latest().message({ request_id: resync.request_id, success: true, ...running }); });
+ await expect(result).resolves.toMatchObject({ state: 'running', revision: 2 });
+ expect(latest().sent.filter(item => item.type === 'start_employee')).toHaveLength(1);
+ });
+
it('does not clear phone UI when a failed reset resync finds an unchanged ready workflow', async () => {
await mount();
await open();
diff --git a/docs-internal/temporal-workflow-control.md b/docs-internal/temporal-workflow-control.md
index f275da57..d5ed4fc0 100644
--- a/docs-internal/temporal-workflow-control.md
+++ b/docs-internal/temporal-workflow-control.md
@@ -253,6 +253,18 @@ Send this data with `type: "pause_workflow"`; Resume uses
capabilities before the next mutation: a stable-state CAS also advances the
database revision. Do not infer the next revision from a local counter.
+Hire can finish its background Start while a card or toolbar still holds an
+earlier revision. A stale Start returns `success: false`,
+`error: "control_revision_conflict"` and the current control snapshot, without
+starting another generation. The server logs the requested/current revisions,
+generation and state as an expected warning. The client merges the snapshot
+and reconciles status. When it already knows a newer running revision, it
+confirms that state before sending another Start. It never rebases stale Start
+intent onto a ready generation created by Reset. A conflict at this API
+boundary is distinct from a failed Temporal Workflow Task or Activity; inspect
+the corresponding execution history and pending Activity attempts when
+diagnosing a worker/runtime failure.
+
Controlled `stop_chat_run` takes `run_id`, `expected_revision` and
`idempotency_key`. It authorizes the session, resolves the run's workflow and
root execution identity, and delegates to Stop with an owning-generation
diff --git a/server/services/deployment/handlers.py b/server/services/deployment/handlers.py
index b3e76268..d4fc783f 100644
--- a/server/services/deployment/handlers.py
+++ b/server/services/deployment/handlers.py
@@ -1789,7 +1789,23 @@ async def handle_start_workflow(data: Dict[str, Any], websocket: WebSocket) -> D
raise ValueError("expected_revision_required")
expected_revision = int(data["expected_revision"])
if expected_revision != (latest.revision if latest else 0):
- raise ValueError("control_revision_conflict")
+ # Hire may have started the generation after the caller read its
+ # summary. Keep the revision fence, but return the authoritative
+ # snapshot so the client can reconcile this expected conflict.
+ logger.warning(
+ "Start rejected for stale control revision",
+ workflow_id=workflow_id,
+ expected_revision=expected_revision,
+ current_revision=latest.revision if latest else 0,
+ generation=latest.generation if latest else 0,
+ state=latest.status if latest else "never_started",
+ )
+ payload = (
+ await _control_payload(latest)
+ if latest is not None
+ else await _with_runtime_counts(serialize_control(None), workflow_id)
+ )
+ return {"success": False, "error": "control_revision_conflict", **payload}
if latest is not None and latest.status != "reset":
raise ValueError("workflow_already_started")
diff --git a/server/tests/services/test_workflow_control_handlers.py b/server/tests/services/test_workflow_control_handlers.py
index 7c73244c..c3ca2fbc 100644
--- a/server/tests/services/test_workflow_control_handlers.py
+++ b/server/tests/services/test_workflow_control_handlers.py
@@ -39,6 +39,47 @@ def _control(
)
+ ("latest", "expected_revision", "state", "revision"),
+ [
+ (_control("running", 2), 0, "running", 2),
+ (_control("reset", 4), 3, "ready", 4),
+ (None, 3, "never_started", 0),
+ ],
+)
+async def test_stale_start_returns_current_status_without_starting_or_error_trace(
+ monkeypatch, caplog, latest, expected_revision, state, revision,
+):
+ service = SimpleNamespace(
+ database=SimpleNamespace(
+ get_workflow_control_by_idempotency_key=AsyncMock(return_value=None),
+ get_latest_workflow_control=AsyncMock(return_value=latest),
+ ),
+ begin_generation=AsyncMock(),
+ )
+ start_controller = AsyncMock()
+ deploy = AsyncMock()
+ snapshot = {"workflow_id": "wf", "state": state, "revision": revision}
+ monkeypatch.setattr(handlers, "_control_service", lambda: service)
+ monkeypatch.setattr(handlers, "_control_payload", AsyncMock(return_value=snapshot))
+ monkeypatch.setattr(handlers, "_with_runtime_counts", AsyncMock(return_value=snapshot))
+ monkeypatch.setattr(handlers, "_start_controller", start_controller)
+ monkeypatch.setattr(handlers, "handle_deploy_workflow", deploy)
+
+ result = await handlers.handle_start_workflow(
+ {"workflow_id": "wf", "expected_revision": expected_revision,
+ "idempotency_key": "stale-start"},
+ None,
+ )
+
+ assert result == {"success": False, "error": "control_revision_conflict", **snapshot}
+ service.begin_generation.assert_not_awaited()
+ start_controller.assert_not_awaited()
+ deploy.assert_not_awaited()
+ assert not any(record.levelno >= 40 or record.exc_info for record in caplog.records)
+
+
@pytest.mark.asyncio
async def test_start_waits_for_deployment_setup_before_publishing_running(monkeypatch):
starting = _control("starting", 0)