Skip to content

Self-improving AI Employees for everyone

fix(temporal): let Reset supersede a draining Stop

trohitg64eb4c4375b7
diff --git a/client/src/Dashboard.tsx b/client/src/Dashboard.tsx
index 94f3ba7b..279ec40a 100644
--- a/client/src/Dashboard.tsx
+++ b/client/src/Dashboard.tsx
@@ -736,6 +736,7 @@ const DashboardContent: React.FC = () => {
try {
await pauseWorkflow(workflowId, workflowControl.revision);
} catch (error: any) {
+ if (error?.message === 'workflow_control_superseded') return;
toast.error(error?.message || 'Unable to pause workflow');
}
};
@@ -743,7 +744,10 @@ const DashboardContent: React.FC = () => {
const handleResumeWorkflow = async () => {
if (!currentWorkflow?.id) return;
try { await resumeWorkflow(currentWorkflow.id, workflowControl.revision); }
- catch (error: any) { toast.error(error?.message || 'Unable to resume workflow'); }
+ catch (error: any) {
+ if (error?.message === 'workflow_control_superseded') return;
+ toast.error(error?.message || 'Unable to resume workflow');
+ }
};
const handleResetWorkflow = async () => {
@@ -754,7 +758,10 @@ const DashboardContent: React.FC = () => {
setExecutionResult(null);
setShowResult(false);
}
- catch (error: any) { toast.error(error?.message || 'Unable to reset workflow'); }
+ catch (error: any) {
+ if (error?.message === 'workflow_control_superseded') return;
+ toast.error(error?.message || 'Unable to reset workflow');
+ }
};
// Helper: fetch all node parameters from DB for export
diff --git a/client/src/components/ui/CommandPaletteHost.tsx b/client/src/components/ui/CommandPaletteHost.tsx
index 97f52d06..bda8166e 100644
--- a/client/src/components/ui/CommandPaletteHost.tsx
+++ b/client/src/components/ui/CommandPaletteHost.tsx
@@ -72,6 +72,12 @@ export const buildWorkflowLifecycleCommands = (
'start' | 'pause' | 'resume' | 'reset' | 'workflowControl' | 'workflowControlPending'
>,
): CommandItem[] => {
+ if (handlers.workflowControlPending?.action === 'pause' && handlers.workflowControl.can_reset) {
+ return [{
+ id: 'run.reset', label: 'Reset Workflow Execution', group: 'Run', icon: RotateCcw,
+ hint: 'terminates active work', onRun: handlers.reset,
+ }];
+ }
if (handlers.workflowControlPending) {
return [];
}
diff --git a/client/src/components/ui/TopToolbar.tsx b/client/src/components/ui/TopToolbar.tsx
index 724fb871..5bf135da 100644
--- a/client/src/components/ui/TopToolbar.tsx
+++ b/client/src/components/ui/TopToolbar.tsx
@@ -52,6 +52,7 @@ import {
} from '@/components/ui/alert-dialog';
import {
isWorkflowControlTransitioning,
+ isWorkflowResetBlocked,
type WorkflowControlPendingMutation,
type WorkflowControlStatus,
} from '../../contexts/WebSocketContext';
@@ -136,6 +137,7 @@ const TopToolbar: React.FC<TopToolbarProps> = ({
const [tempName, setTempName] = useState(workflowName);
const { user, logout } = useAuth();
const hasPendingControlMutation = Boolean(workflowControlPending);
+ const resetBlocked = isWorkflowResetBlocked(workflowControlPending);
const isAuthoritativeTransition = isWorkflowControlTransitioning(workflowControl);
const canPauseWorkflow = workflowControl.can_pause || workflowControl.state === 'pausing';
const canResumeWorkflow = workflowControl.can_resume || workflowControl.state === 'resuming';
@@ -492,7 +494,7 @@ const TopToolbar: React.FC<TopToolbarProps> = ({
<ActionButton
intent="stop"
onClick={() => setResetOpen(true)}
- disabled={hasPendingControlMutation}
+ disabled={resetBlocked}
title={isRetryingReset
? 'Retry the interrupted reset transition'
: 'Stop active work and reset this workflow'}
@@ -515,7 +517,7 @@ const TopToolbar: React.FC<TopToolbarProps> = ({
</Badge>
)}
- <AlertDialog open={resetOpen} onOpenChange={setResetOpen}><AlertDialogContent><AlertDialogHeader><AlertDialogTitle>Reset workflow execution?</AlertDialogTitle><AlertDialogDescription>This stops active work, cancels phone tasks, and archives the current generation. Reset waits for phone actions to finish cleaning up. Task history remains available. The workflow will remain stopped until you press Start.</AlertDialogDescription></AlertDialogHeader><AlertDialogFooter><AlertDialogCancel>Keep current execution</AlertDialogCancel><AlertDialogAction disabled={hasPendingControlMutation} onClick={onResetWorkflow}>{isRetryingReset ? 'Retry reset' : 'Reset workflow'}</AlertDialogAction></AlertDialogFooter></AlertDialogContent></AlertDialog>
+ <AlertDialog open={resetOpen} onOpenChange={setResetOpen}><AlertDialogContent><AlertDialogHeader><AlertDialogTitle>Reset workflow execution?</AlertDialogTitle><AlertDialogDescription>This stops active work, cancels phone tasks, and archives the current generation. Reset waits for phone actions to finish cleaning up. Task history remains available. The workflow will remain stopped until you press Start.</AlertDialogDescription></AlertDialogHeader><AlertDialogFooter><AlertDialogCancel>Keep current execution</AlertDialogCancel><AlertDialogAction disabled={resetBlocked} onClick={onResetWorkflow}>{isRetryingReset ? 'Retry reset' : 'Reset workflow'}</AlertDialogAction></AlertDialogFooter></AlertDialogContent></AlertDialog>
<ActionButton
intent="save"
diff --git a/client/src/components/ui/__tests__/CommandPaletteHost.workflowControl.test.ts b/client/src/components/ui/__tests__/CommandPaletteHost.workflowControl.test.ts
index 12347509..d0d7182a 100644
--- a/client/src/components/ui/__tests__/CommandPaletteHost.workflowControl.test.ts
+++ b/client/src/components/ui/__tests__/CommandPaletteHost.workflowControl.test.ts
@@ -53,9 +53,32 @@ describe('command palette workflow lifecycle capabilities', () => {
expect(commands.map((command) => command.id)).not.toContain('run.start');
});
- it('suppresses lifecycle actions while a local mutation is pending', () => {
+ it('offers only Reset while a local Stop is pending', () => {
+ const reset = vi.fn();
const commands = buildWorkflowLifecycleCommands(handlers(
control({ can_pause: true, can_reset: true }),
+ { workflowControlPending: { action: 'pause', state: 'pausing' }, reset },
+ ));
+ expect(commands.map(command => command.id)).toEqual(['run.reset']);
+ commands[0].onRun();
+ expect(reset).toHaveBeenCalledOnce();
+ });
+
+ it.each([
+ { action: 'start' as const, state: 'starting' as const },
+ { action: 'resume' as const, state: 'resuming' as const },
+ { action: 'reset' as const, state: 'resetting' as const },
+ ])('suppresses lifecycle actions while $action is pending', (pending) => {
+ const commands = buildWorkflowLifecycleCommands(handlers(
+ control({ can_pause: true, can_reset: true }),
+ { workflowControlPending: pending },
+ ));
+ expect(commands).toEqual([]);
+ });
+
+ it('does not offer Reset during Stop without the server capability', () => {
+ const commands = buildWorkflowLifecycleCommands(handlers(
+ control({ can_reset: false }),
{ workflowControlPending: { action: 'pause', state: 'pausing' } },
));
expect(commands).toEqual([]);
diff --git a/client/src/contexts/WebSocketContext.tsx b/client/src/contexts/WebSocketContext.tsx
index 9af5b2a0..49b6704b 100644
--- a/client/src/contexts/WebSocketContext.tsx
+++ b/client/src/contexts/WebSocketContext.tsx
@@ -217,6 +217,26 @@ export interface WorkflowControlPendingMutation {
state: WorkflowControlTransitionState;
}
+/** Reset can recover a Stop whose admitted work has not finished draining. */
+export const isWorkflowResetBlocked = (
+ pending: WorkflowControlPendingMutation | undefined,
+): boolean => Boolean(pending && pending.action !== 'pause');
+
+export class WorkflowControlMutationSupersededError extends Error {
+ constructor() {
+ super('workflow_control_superseded');
+ this.name = 'WorkflowControlMutationSupersededError';
+ }
+}
+
+interface WorkflowControlMutationOwner {
+ pending: WorkflowControlPendingMutation;
+ generation?: number;
+ rootExecutionId?: string | null;
+ superseded: boolean;
+ supersede: () => void;
+}
+
const WORKFLOW_CONTROL_STATES = new Set<WorkflowControlState>([
'never_started',
'ready',
@@ -840,7 +860,7 @@ export const WebSocketProvider: React.FC<{ children: React.ReactNode }> = ({ chi
const [workflowControlStatuses, setWorkflowControlStatuses] = useState<Record<string, WorkflowControlStatus>>({});
const workflowControlStatusesRef = useRef<Record<string, WorkflowControlStatus>>({});
const [workflowControlPending, setWorkflowControlPending] = useState<Record<string, WorkflowControlPendingMutation>>({});
- const workflowControlPendingRef = useRef<Map<string, WorkflowControlPendingMutation>>(new Map());
+ const workflowControlPendingRef = useRef<Map<string, WorkflowControlMutationOwner>>(new Map());
const [workflowLock, setWorkflowLock] = useState<WorkflowLock>(defaultWorkflowLock);
// Per-workflow compaction stats: workflow_id -> session_id -> CompactionStats (n8n pattern)
const [allCompactionStats, setAllCompactionStats] = useState<Record<string, Record<string, CompactionStats>>>({});
@@ -2851,10 +2871,12 @@ export const WebSocketProvider: React.FC<{ children: React.ReactNode }> = ({ chi
const requestWorkflowControlStatus = useCallback(async (
workflowId: string,
+ isCurrent?: () => boolean,
): Promise<WorkflowControlStatus> => {
const response = assertWorkflowControlMutationSucceeded(
await sendRequest<any>('get_workflow_control_status', { workflow_id: workflowId }),
);
+ if (isCurrent && !isCurrent()) throw new WorkflowControlMutationSupersededError();
return applyWorkflowControlStatus(workflowId, response);
}, [applyWorkflowControlStatus, sendRequest]);
@@ -2863,25 +2885,60 @@ export const WebSocketProvider: React.FC<{ children: React.ReactNode }> = ({ chi
workflowId: string,
data: Record<string, any>,
): Promise<WorkflowStartResult> => {
- if (workflowControlPendingRef.current.has(workflowId)) {
+ const existingOwner = workflowControlPendingRef.current.get(workflowId);
+ if (existingOwner && (type !== 'reset_workflow' || isWorkflowResetBlocked(existingOwner.pending))) {
throw new Error('A workflow lifecycle change is already in progress');
}
- const pending = WORKFLOW_CONTROL_PENDING_BY_REQUEST[type];
+ const pending = { ...WORKFLOW_CONTROL_PENDING_BY_REQUEST[type] };
const previousStatus = workflowControlStatusesRef.current[workflowId];
+ const matchesGeneration = (status: WorkflowControlStatus) => (
+ (previousStatus?.generation === undefined || status.generation === previousStatus.generation)
+ && (!previousStatus?.root_execution_id || status.root_execution_id === previousStatus.root_execution_id)
+ );
+ if (existingOwner && previousStatus && (
+ (existingOwner.generation !== undefined && previousStatus.generation !== existingOwner.generation)
+ || (existingOwner.rootExecutionId && previousStatus.root_execution_id !== existingOwner.rootExecutionId)
+ )) {
+ throw new Error('control_generation_conflict');
+ }
+ // Stop has already advanced the revision when its pausing broadcast arrives.
+ // Superseding it uses that same generation's current revision, even if the
+ // caller still has the earlier running snapshot.
+ const mutationData = type === 'reset_workflow' ? {
+ ...data,
+ ...(existingOwner && previousStatus ? { expected_revision: previousStatus.revision } : {}),
+ ...(previousStatus?.root_execution_id ? { expected_root_execution_id: previousStatus.root_execution_id } : {}),
+ } : data;
+ let rejectSuperseded!: (error: Error) => void;
+ const superseded = new Promise<never>((_resolve, reject) => { rejectSuperseded = reject; });
+ const owner: WorkflowControlMutationOwner = {
+ pending,
+ generation: previousStatus?.generation,
+ rootExecutionId: previousStatus?.root_execution_id,
+ superseded: false,
+ supersede: () => {
+ owner.superseded = true;
+ rejectSuperseded(new WorkflowControlMutationSupersededError());
+ },
+ };
+ const isCurrent = () => !owner.superseded;
+ const waitForCurrent = <T,>(request: Promise<T>): Promise<T> => Promise.race([request, superseded]);
+ const readStatus = () => waitForCurrent(requestWorkflowControlStatus(workflowId, isCurrent));
const resetRequestIds = new Set<string>();
if (type === 'reset_workflow' && previousStatus?.workspace_resetting
&& typeof previousStatus.workspace_reset_request_id === 'string') {
// Retrying an interrupted reset can finish the original server request.
resetRequestIds.add(previousStatus.workspace_reset_request_id);
}
- let resetExpectedRevision = Number(data.expected_revision ?? previousStatus?.revision ?? 0);
+ let resetExpectedRevision = Number(mutationData.expected_revision ?? previousStatus?.revision ?? 0);
const isReconciled = (status: WorkflowControlStatus) => type === 'reset_workflow'
- ? isWorkflowResetRecoveryConfirmed(status, resetExpectedRevision, resetRequestIds)
+ ? matchesGeneration(status) && isWorkflowResetRecoveryConfirmed(status, resetExpectedRevision, resetRequestIds)
: isWorkflowControlMutationConfirmed(pending.action, status);
- workflowControlPendingRef.current.set(workflowId, pending);
+ workflowControlPendingRef.current.set(workflowId, owner);
setWorkflowControlPending((previous) => ({ ...previous, [workflowId]: pending }));
useWorkflowControlStore.getState().setPending(workflowId, pending);
+ existingOwner?.supersede();
const sendMutationAttempt = async (
attemptData: Record<string, any>,
@@ -2891,13 +2948,14 @@ export const WebSocketProvider: React.FC<{ children: React.ReactNode }> = ({ chi
resetRequestIds.add(idempotencyKey);
resetExpectedRevision = Number(attemptData.expected_revision ?? resetExpectedRevision);
}
- const response = await sendRequest<any>(type, {
+ const response = await waitForCurrent(sendRequest<any>(type, {
...attemptData,
workflow_id: workflowId,
// Every bounded attempt has its own idempotency identity. In
// particular, a conflict retry must not replay the rejected request.
idempotency_key: idempotencyKey,
- }, WORKFLOW_CONTROL_REQUEST_TIMEOUT);
+ }, WORKFLOW_CONTROL_REQUEST_TIMEOUT));
+ if (!isCurrent()) throw new WorkflowControlMutationSupersededError();
// A success:false response can still be the newest authoritative
// snapshot. Merge it before assertion so the UI is correct even if the
@@ -2906,9 +2964,19 @@ export const WebSocketProvider: React.FC<{ children: React.ReactNode }> = ({ chi
const acceptedSnapshot = snapshot
? applyWorkflowControlStatus(workflowId, snapshot)
: undefined;
+ // A Reset from another tab can replace this intent without replacing
+ // this tab's pending owner. Retain its authoritative snapshot, but do
+ // not report the superseded Pause/Resume as a successful operation.
+ if (response?.superseded === true) throw new WorkflowControlMutationSupersededError();
assertWorkflowControlMutationSucceeded(response);
const acceptedStatus = acceptedSnapshot
?? applyWorkflowControlStatus(workflowId, response);
+ if (type === 'reset_workflow' && (!matchesGeneration(acceptedStatus)
+ || !isWorkflowControlMutationConfirmed('reset', acceptedStatus)
+ || acceptedStatus.workspace_resetting === true
+ || acceptedStatus.workspace_available === false)) {
+ throw new Error('workflow_control_transition_pending');
+ }
if (type !== 'start_workflow') return acceptedStatus;
// Lifecycle status is merged independently, but Start also returns the
@@ -2934,19 +3002,22 @@ export const WebSocketProvider: React.FC<{ children: React.ReactNode }> = ({ chi
// toolbar still carries the earlier revision. Confirm the running
// state without submitting another Start. Never rebase stale intent
// onto a newer ready generation after Reset.
- const current = await requestWorkflowControlStatus(workflowId);
+ const current = await readStatus();
if (isReconciled(current)) return current;
throw new Error('control_revision_conflict');
}
- return await sendMutationAttempt(data);
+ return await sendMutationAttempt(mutationData);
} catch (error) {
+ if (error instanceof WorkflowControlMutationSupersededError) throw error;
+ if (!isCurrent()) throw new WorkflowControlMutationSupersededError();
let reconciled: WorkflowControlStatus | undefined;
try {
- reconciled = await requestWorkflowControlStatus(workflowId);
+ reconciled = await readStatus();
if (isReconciled(reconciled)) {
return reconciled;
}
} catch (resyncError) {
+ if (!isCurrent()) throw new WorkflowControlMutationSupersededError();
console.error('[WebSocket] Failed to resync workflow control status:', resyncError);
}
@@ -2957,18 +3028,20 @@ export const WebSocketProvider: React.FC<{ children: React.ReactNode }> = ({ chi
if (
type === 'reset_workflow'
&& reconciled
+ && matchesGeneration(reconciled)
&& shouldRetryResetWorkflowAfterConflict(error, reconciled)
) {
try {
return await sendMutationAttempt({
- ...data,
+ ...mutationData,
expected_revision: reconciled.revision,
});
} catch (retryError) {
+ if (retryError instanceof WorkflowControlMutationSupersededError) throw retryError;
// Preserve timeout/idempotency recovery for the single retry without
// issuing a third mutation attempt.
try {
- const retryReconciled = await requestWorkflowControlStatus(workflowId);
+ const retryReconciled = await readStatus();
if (isReconciled(retryReconciled)) {
return retryReconciled;
}
@@ -2980,14 +3053,16 @@ export const WebSocketProvider: React.FC<{ children: React.ReactNode }> = ({ chi
}
throw error;
} finally {
- workflowControlPendingRef.current.delete(workflowId);
- useWorkflowControlStore.getState().setPending(workflowId, null);
- setWorkflowControlPending((previous) => {
- if (!(workflowId in previous)) return previous;
- const next = { ...previous };
- delete next[workflowId];
- return next;
- });
+ if (workflowControlPendingRef.current.get(workflowId) === owner) {
+ workflowControlPendingRef.current.delete(workflowId);
+ useWorkflowControlStore.getState().setPending(workflowId, null);
+ setWorkflowControlPending((previous) => {
+ if (previous[workflowId] !== pending) return previous;
+ const next = { ...previous };
+ delete next[workflowId];
+ return next;
+ });
+ }
}
}, [applyWorkflowControlStatus, requestWorkflowControlStatus, sendRequest]);
@@ -3032,6 +3107,13 @@ export const WebSocketProvider: React.FC<{ children: React.ReactNode }> = ({ chi
const result = await controlMutation(
'reset_workflow', workflowId, { expected_revision: expectedRevision },
);
+ const current = workflowControlStatusesRef.current[workflowId];
+ if (current && (current.generation !== result.generation
+ || current.root_execution_id !== result.root_execution_id
+ || !isWorkflowControlMutationConfirmed('reset', current)
+ || current.workspace_resetting === true)) {
+ throw new WorkflowControlMutationSupersededError();
+ }
// The broadcast clears every connected tab; repeat locally so the
// initiating tab is correct even if a proxy suppresses self-echo.
useNodeStatusStore.getState().clearWorkflow(workflowId);
diff --git a/client/src/contexts/__tests__/webSocketLifecycle.test.tsx b/client/src/contexts/__tests__/webSocketLifecycle.test.tsx
index 4ac80356..6e72282f 100644
--- a/client/src/contexts/__tests__/webSocketLifecycle.test.tsx
+++ b/client/src/contexts/__tests__/webSocketLifecycle.test.tsx
@@ -7,9 +7,10 @@ const auth = vi.hoisted(() => ({ isAuthenticated: true, isLoading: false }));
vi.mock('../AuthContext', () => ({ useAuth: () => auth }));
vi.mock('../../services/workflowApi', () => ({ workflowApi: {} }));
-import { WebSocketProvider, useWebSocket, useWebSocketActions, type WebSocketActions } from '../WebSocketContext';
+import { WebSocketProvider, WorkflowControlMutationSupersededError, useWebSocket, useWebSocketActions, type WebSocketActions } from '../WebSocketContext';
import { useAppStore } from '../../store/useAppStore';
import { useNodeStatusStore } from '../../stores/nodeStatusStore';
+import { useWorkflowControlStore } from '../../stores/workflowControlStore';
// Exercise the installed PartySocket implementation; only its network peer
// is simulated. Remote code 1000 and native error events really reconnect.
@@ -194,6 +195,139 @@ describe('WebSocket recovery and ownership', () => {
expect(latest().sent.filter(item => item.type === 'stop_chat_run')).toHaveLength(1);
});
+ it.each(['pause_workflow', 'stop_chat_run'])('lets Reset supersede a stuck %s using the admitted generation revision', async (stopType) => {
+ await mount();
+ await open();
+ const clear = vi.spyOn(useNodeStatusStore.getState(), 'clearWorkflow');
+ const notify = vi.fn();
+ actions.addEventListener('workflow_runtime_reset', notify);
+ const running = { workflow_id: 'browser', state: 'running', revision: 3, generation: 1,
+ root_execution_id: 'g1', can_pause: true, can_reset: true, in_flight_count: 1 };
+ act(() => latest().message({ type: 'workflow_control_status', data: running }));
+ let stop!: Promise<unknown>;
+ act(() => { stop = (stopType === 'pause_workflow'
+ ? actions.pauseWorkflow('browser', 3)
+ : actions.stopChatRun('browser', 3, 'r1')).catch(error => error); });
+ const stopRequest = latest().sent.find(item => item.type === stopType)!;
+ act(() => latest().message({ type: 'workflow_control_status', data: { ...running, state: 'pausing', revision: 4 } }));
+ let reset!: Promise<unknown>;
+ await act(async () => { reset = actions.resetWorkflow('browser', 3); });
+ const resetRequest = latest().sent.find(item => item.type === 'reset_workflow')!;
+ expect(resetRequest).toMatchObject({ expected_revision: 4, expected_root_execution_id: 'g1' });
+ await expect(stop).resolves.toBeInstanceOf(WorkflowControlMutationSupersededError);
+ expect(context.workflowControlPending.browser).toEqual({ action: 'reset', state: 'resetting' });
+ expect(useWorkflowControlStore.getState().pending.browser).toEqual({ action: 'reset', state: 'resetting' });
+ expect(clear).not.toHaveBeenCalled();
+ expect(notify).not.toHaveBeenCalled();
+ const resetting = { ...running, state: 'resetting', revision: 6, can_pause: false };
+ await act(async () => {
+ latest().message({ type: 'workflow_control_status', data: resetting });
+ latest().message({ request_id: stopRequest.request_id, success: true,
+ ...running, state: 'paused', revision: 5 });
+ });
+ expect(context.workflowControlStatuses.browser).toMatchObject({ state: 'resetting', revision: 6 });
+ expect(context.workflowControlPending.browser?.action).toBe('reset');
+ expect(latest().sent.some(item => item.type === 'get_workflow_control_status')).toBe(false);
+ await expect(actions.resumeWorkflow('browser', 6)).rejects.toThrow('already in progress');
+ await expect(actions.resetWorkflow('browser', 6)).rejects.toThrow('already in progress');
+ await act(async () => latest().message({ request_id: resetRequest.request_id, success: true,
+ ...running, state: 'ready', revision: 7, can_start: true, can_reset: false, in_flight_count: 0 }));
+ await expect(reset).resolves.toMatchObject({ state: 'ready', revision: 7 });
+ expect(context.workflowControlPending.browser).toBeUndefined();
+ expect(clear).toHaveBeenCalledWith('browser');
+ expect(notify).toHaveBeenCalledWith({ workflow_id: 'browser' });
+ act(() => latest().message({ type: 'workflow_control_status', data: { ...running, state: 'paused', revision: 5 } }));
+ expect(context.workflowControlStatuses.browser).toMatchObject({ state: 'ready', revision: 7 });
+ });
+
+ it('ignores an interrupted Stop status resync once Reset takes ownership', async () => {
+ await mount();
+ await open();
+ const running = { workflow_id: 'browser', state: 'running', revision: 3, generation: 1, root_execution_id: 'g1', can_reset: true };
+ act(() => latest().message({ type: 'workflow_control_status', data: running }));
+ let stop!: Promise<unknown>;
+ act(() => { stop = actions.pauseWorkflow('browser', 3).catch(error => error); });
+ const stopRequest = latest().sent.find(item => item.type === 'pause_workflow')!;
+ const pausing = { ...running, state: 'pausing', revision: 4 };
+ await act(async () => latest().message({ request_id: stopRequest.request_id, success: false,
+ error: 'workflow_control_transition_pending', status: pausing }));
+ const read = latest().sent.find(item => item.type === 'get_workflow_control_status')!;
+ let reset!: Promise<unknown>;
+ await act(async () => { reset = actions.resetWorkflow('browser', 3); });
+ await expect(stop).resolves.toBeInstanceOf(WorkflowControlMutationSupersededError);
+ const resetRequest = latest().sent.find(item => item.type === 'reset_workflow')!;
+ await act(async () => latest().message({ request_id: read.request_id, success: true,
+ ...running, state: 'paused', revision: 5 }));
+ expect(context.workflowControlStatuses.browser).toMatchObject({ state: 'pausing', revision: 4 });
+ expect(context.workflowControlPending.browser?.action).toBe('reset');
+ await act(async () => latest().message({ request_id: resetRequest.request_id, success: true,
+ ...running, state: 'ready', revision: 7 }));
+ await expect(reset).resolves.toMatchObject({ state: 'ready' });
+ });
+
+ it('does not apply pending Stop intent to a newer generation when Reset is pressed', async () => {
+ await mount();
+ await open();
+ const running = { workflow_id: 'browser', state: 'running', revision: 3, generation: 1, root_execution_id: 'g1', can_reset: true };
+ act(() => latest().message({ type: 'workflow_control_status', data: running }));
+ let stop!: Promise<unknown>;
+ act(() => { stop = actions.pauseWorkflow('browser', 3); });
+ const stopRequest = latest().sent.find(item => item.type === 'pause_workflow')!;
+ act(() => latest().message({ type: 'workflow_control_status', data: {
+ ...running, revision: 8, generation: 2, root_execution_id: 'g2',
+ } }));
+ await expect(actions.resetWorkflow('browser', 3)).rejects.toThrow('control_generation_conflict');
+ expect(latest().sent.some(item => item.type === 'reset_workflow')).toBe(false);
+ await act(async () => latest().message({ request_id: stopRequest.request_id, success: true,
+ ...running, state: 'paused', revision: 5 }));
+ await stop;
+ expect(context.workflowControlStatuses.browser).toMatchObject({ generation: 2, revision: 8 });
+ });
+
+ it.each(['pause_workflow', 'stop_chat_run', 'resume_workflow'])('recognizes a Reset from another tab superseding %s without another status request', async (type) => {
+ await mount();
+ await open();
+ const clear = vi.spyOn(useNodeStatusStore.getState(), 'clearWorkflow');
+ const status = { workflow_id: 'browser', state: type === 'resume_workflow' ? 'paused' : 'running',
+ revision: 3, generation: 1, root_execution_id: 'g1', can_reset: true };
+ act(() => latest().message({ type: 'workflow_control_status', data: status }));
+ let result!: Promise<unknown>;
+ act(() => {
+ result = (type === 'resume_workflow'
+ ? actions.resumeWorkflow('browser', 3)
+ : type === 'stop_chat_run'
+ ? actions.stopChatRun('browser', 3, 'r1')
+ : actions.pauseWorkflow('browser', 3)).catch(error => error);
+ });
+ const request = latest().sent.find(item => item.type === type)!;
+ await act(async () => latest().message({ request_id: request.request_id, success: true, superseded: true,
+ ...status, state: 'ready', revision: 7, can_start: true, can_reset: false }));
+ await expect(result).resolves.toBeInstanceOf(WorkflowControlMutationSupersededError);
+ expect(context.workflowControlStatuses.browser).toMatchObject({ state: 'ready', revision: 7 });
+ expect(context.workflowControlPending.browser).toBeUndefined();
+ expect(latest().sent.some(item => item.type === 'get_workflow_control_status')).toBe(false);
+ expect(clear).not.toHaveBeenCalled();
+ });
+
+ it('does not retry a Reset conflict or clear runtime state in a newer generation', async () => {
+ await mount();
+ await open();
+ const clear = vi.spyOn(useNodeStatusStore.getState(), 'clearWorkflow');
+ const running = { workflow_id: 'browser', state: 'running', revision: 3, generation: 1, root_execution_id: 'g1', can_reset: true };
+ act(() => latest().message({ type: 'workflow_control_status', data: running }));
+ let reset!: Promise<unknown>;
+ act(() => { reset = actions.resetWorkflow('browser', 3).catch(error => error.message); });
+ const resetRequest = latest().sent.find(item => item.type === 'reset_workflow')!;
+ const nextGeneration = { ...running, generation: 2, root_execution_id: 'g2', revision: 8 };
+ await act(async () => latest().message({ request_id: resetRequest.request_id, success: false,
+ error: 'control_revision_conflict', status: nextGeneration }));
+ const read = latest().sent.find(item => item.type === 'get_workflow_control_status')!;
+ await act(async () => latest().message({ request_id: read.request_id, success: true, status: nextGeneration }));
+ await expect(reset).resolves.toBe('control_revision_conflict');
+ expect(latest().sent.filter(item => item.type === 'reset_workflow')).toHaveLength(1);
+ expect(clear).not.toHaveBeenCalled();
+ });
+
it.each([false, true])('confirms an exact phone reset ID only after cleanup ends (resetting=%s)', async (workspaceResetting) => {
await mount();
await open();
diff --git a/client/src/features/chat/__tests__/stop.test.tsx b/client/src/features/chat/__tests__/stop.test.tsx
new file mode 100644
index 00000000..fbc93f10
--- /dev/null
+++ b/client/src/features/chat/__tests__/stop.test.tsx
@@ -0,0 +1,68 @@
+import type { ReactNode } from 'react';
+import { QueryClient, QueryClientProvider } from '@tanstack/react-query';
+import { act, renderHook } from '@testing-library/react';
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+import type { WorkflowControlStatus } from '@/contexts/WebSocketContext';
+import { emptyRun, type RunSnapshot } from '@/lib/agui/reduceRun';
+import { useChatRunStore } from '@/stores/chatRunStore';
+import { useStopChatRun } from '../data/stop';
+
+const actions = vi.hoisted(() => ({
+ sendRequest: vi.fn(),
+ getWorkflowControlStatus: vi.fn(),
+ stopChatRun: vi.fn(),
+}));
+vi.mock('@/contexts/WebSocketContext', () => ({ useWebSocketActions: () => actions }));
+
+const control = {
+ workflow_id: 'workflow', revision: 3, execution_control_version: 1,
+} as WorkflowControlStatus;
+
+function mount(onFailed: () => void) {
+ const queryClient = new QueryClient({ defaultOptions: { mutations: { retry: false } } });
+ return renderHook(() => useStopChatRun('session', onFailed, control), {
+ wrapper: ({ children }: { children: ReactNode }) => (
+ <QueryClientProvider client={queryClient}>{children}</QueryClientProvider>
+ ),
+ });
+}
+
+describe('controlled chat Stop recovery', () => {
+ beforeEach(() => {
+ vi.resetAllMocks();
+ useChatRunStore.setState({ sessions: {} });
+ });
+
+ it('does not report a Stop failure or discard partial output when Reset supersedes it', async () => {
+ actions.stopChatRun.mockRejectedValue(new Error('workflow_control_superseded'));
+ const run: RunSnapshot = {
+ ...emptyRun('run', 'session'), workflowId: 'workflow', state: 'running',
+ segments: [{ messageId: 'reply', text: 'Partial browser answer', final: null }],
+ };
+ useChatRunStore.getState().upsertRun(run);
+ const before = useChatRunStore.getState().sessions.session.runs.run;
+ const onFailed = vi.fn();
+ const { result } = mount(onFailed);
+
+ await act(async () => {
+ await expect(result.current.mutateAsync(run)).rejects.toThrow('workflow_control_superseded');
+ });
+
+ expect(onFailed).not.toHaveBeenCalled();
+ expect(useChatRunStore.getState().sessions.session.runs.run).toBe(before);
+ expect(actions.sendRequest).not.toHaveBeenCalled();
+ });
+
+ it('still reports an actual Stop failure', async () => {
+ actions.stopChatRun.mockRejectedValue(new Error('temporal_control_unavailable'));
+ const onFailed = vi.fn();
+ const { result } = mount(onFailed);
+ const run = { ...emptyRun('run', 'session'), workflowId: 'workflow' };
+
+ await act(async () => {
+ await expect(result.current.mutateAsync(run)).rejects.toThrow('temporal_control_unavailable');
+ });
+
+ expect(onFailed).toHaveBeenCalledOnce();
+ });
+});
diff --git a/client/src/features/chat/data/control.ts b/client/src/features/chat/data/control.ts
index 60fffd14..61415cd2 100644
--- a/client/src/features/chat/data/control.ts
+++ b/client/src/features/chat/data/control.ts
@@ -20,6 +20,8 @@ export function useResumeChatGeneration(control: WorkflowControlStatus | undefin
if (current.root_execution_id !== control.root_execution_id) throw new Error('control_generation_conflict');
return resumeWorkflow(control.workflow_id, current.revision);
},
- onError: onFailed,
+ onError: (error) => {
+ if (error.message !== 'workflow_control_superseded') onFailed();
+ },
});
}
diff --git a/client/src/features/chat/data/stop.ts b/client/src/features/chat/data/stop.ts
index 48349512..0e66f1d4 100644
--- a/client/src/features/chat/data/stop.ts
+++ b/client/src/features/chat/data/stop.ts
@@ -46,7 +46,7 @@ export function useStopChatRun(sessionId: string, onFailed?: () => void, control
}
},
onError: (error) => {
- if (error.message !== 'not_stoppable') onFailed?.();
+ if (error.message !== 'not_stoppable' && error.message !== 'workflow_control_superseded') onFailed?.();
},
});
}
diff --git a/client/src/features/home/employee/useEmployeeControl.ts b/client/src/features/home/employee/useEmployeeControl.ts
index dedff727..7185bb07 100644
--- a/client/src/features/home/employee/useEmployeeControl.ts
+++ b/client/src/features/home/employee/useEmployeeControl.ts
@@ -94,6 +94,7 @@ export function useEmployeeControl(employee: EmployeeSummary, onConnect: (provid
pillToast(`${employee.name} ${CONTROL_DONE[kind]}`);
if (kind !== 'pause') spikeOrb(SPIKE.start);
} catch (error) {
+ if (error instanceof Error && error.message === 'workflow_control_superseded') return;
pillToast(controlErrorMessage(error), { tone: 'error' });
if (error instanceof Error && error.message === 'control_revision_conflict') invalidateEmployees(queryClient);
if (error instanceof Error && error.message === 'needs_ai') openConnectAI();
diff --git a/client/src/features/home/workspace/takeover.ts b/client/src/features/home/workspace/takeover.ts
index bcdc36b5..ccdc2f8c 100644
--- a/client/src/features/home/workspace/takeover.ts
+++ b/client/src/features/home/workspace/takeover.ts
@@ -52,6 +52,7 @@ export function useTakeOver(employee: EmployeeSummary, surface: () => SurfaceCon
pillToast(`You have control — ${employee.name} will wait`);
} catch (error) {
giveBack?.();
+ if (error instanceof Error && error.message === 'workflow_control_superseded') return;
pillToast(controlErrorMessage(error), { tone: 'error' });
} finally {
setBusy(false);
diff --git a/docs-internal/temporal-workflow-control.md b/docs-internal/temporal-workflow-control.md
index 360bb949..cb83ac50 100644
--- a/docs-internal/temporal-workflow-control.md
+++ b/docs-internal/temporal-workflow-control.md
@@ -69,13 +69,22 @@ of falsely publishing a stable state.
Every accepted control transition emits `workflow_control_status`. Clients
merge reads, mutation responses, and broadcasts monotonically by generation
then database revision; an older response cannot move another tab backward.
-Only one lifecycle mutation per workflow may be pending in a browser tab.
+Only one lifecycle mutation per workflow may be pending in a browser tab,
+except that Reset may supersede a pending Stop. The superseded Stop cannot
+clear Reset's pending state or apply a late result. Reset uses the current
+revision within the same owning generation and refuses a different generation.
Lifecycle requests have a five-minute acknowledgement window, and a timed-out
request is treated as successful when the immediate authoritative resync
already reports its requested stable state. Transitional Pause, Resume, and
Reset states expose an explicit retry rather than trapping the toolbar in a
permanent spinner.
+Version 1 status reads return transitional state promptly. They start or reuse
+one supervised reconciliation task for the control revision instead of waiting
+for its checkpoint. A disconnected socket cancels only its local waiter; the
+supervised transition and durable admission fence remain. Reset retires older
+local checkpoint polling and reports the newer control state to those waiters.
+
## Acknowledged Stop for new generations
New generations atomically store `resource_manifest.execution_control_version=1`.
@@ -110,6 +119,13 @@ through result bookkeeping; maintenance Activities, permit waits, sleeps and
child-result waits are excluded. Root membership is reconciled until stable.
Visibility scans remain part of destructive Reset, not Stop acknowledgement.
+Generation Reset also retires durable browser ownership held by terminated
+native agents. It verifies each exact task token against the recorded Workflow
+input and generation, then awaits the browser owner's cleanup receipt before
+archiving the generation. Browser operations and shared profile startup are
+settled before releasing ownership. Failed cleanup leaves `resetting` for retry;
+claims belonging to another generation or direct Workspace tasks are preserved.
+
### Stop and Resume acknowledgement sequence
The orchestration lives in
diff --git a/server/nodes/browser/_routing.py b/server/nodes/browser/_routing.py
index 8ba0f805..f74ac745 100644
--- a/server/nodes/browser/_routing.py
+++ b/server/nodes/browser/_routing.py
@@ -108,7 +108,7 @@ async def owner_command(request: Any) -> dict:
binding = await command_binding(principal, data)
if binding is None or binding["owner_id"] != replica_id():
raise NodeUserError("Browser unavailable — waiting for its owner.")
- return await cleanup_browser_task(get_database(), {**binding, "principal_id": principal}, str(data.get("task_id") or ""))
+ return await cleanup_browser_task(get_database(), {**binding, "principal_id": principal}, str(data.get("task_id") or ""), reset=data.get("reset") is True)
if command == "session_file":
import base64
from ._cookies import MAX_UPLOAD_BYTES, parse_session_file
diff --git a/server/nodes/browser/browser/__init__.py b/server/nodes/browser/browser/__init__.py
index ff40e331..42c17907 100644
--- a/server/nodes/browser/browser/__init__.py
+++ b/server/nodes/browser/browser/__init__.py
@@ -411,6 +411,17 @@ class BrowserNode(ToolNode):
# The owning agent's reset hook carries its exact task token.
# A legacy Browser hook must not release an unrelated task.
return {"reset": False, "managed_by_agent": True}
+ running = runtime.running(session.profile_id) if hasattr(runtime, "running") else None
+ if running is not None and controller.lease_session is not None and controller.lease_session.session_id == session.session_id:
+ # A direct Browser Activity may outlive its terminated graph
+ # Workflow. Retire its command before dropping the lease.
+ controller.needs_observation = True
+ await running.cli.interrupt()
+ async with controller._op_lock:
+ if not await running.cli.suspend_for_credentials():
+ await runtime.stop_profile(session.profile_id, reason="workflow reset")
+ if running.chrome.is_running():
+ raise NodeUserError("Browser reset is waiting for its owner to stop the runtime.")
await controller.release_lease(session.session_id)
return {"reset": True}
diff --git a/server/services/browser_owners.py b/server/services/browser_owners.py
index c7a71767..60a34b35 100644
--- a/server/services/browser_owners.py
+++ b/server/services/browser_owners.py
@@ -324,18 +324,42 @@ async def release_browser_task(database: Any, binding: dict, task_id: str) -> di
return {"released": True}
-async def cleanup_browser_task(database: Any, binding: dict, task_id: str) -> dict:
+async def cleanup_browser_task(database: Any, binding: dict, task_id: str, *, reset: bool = False) -> dict:
if getattr(settings(), "distributed_mode", False) is True and binding["owner_id"] != replica_id():
from nodes.browser._routing import forward_command
- return await forward_command(binding, str(binding["principal_id"]), "cleanup_task", {"profile_id": binding["profile_id"], "task_id": task_id})
+ data = {"profile_id": binding["profile_id"], "task_id": task_id}
+ if reset:
+ data["reset"] = True
+ return await forward_command(binding, str(binding["principal_id"]), "cleanup_task", data)
await assert_runtime_epoch(database)
+ if reset and _persistent(database):
+ async with database.get_session() as session:
+ current = await session.get(BrowserProfileOwner, binding["profile_id"])
+ if current is None or current.task_id != task_id or current.owner_id != replica_id():
+ return {"released": False}
from nodes.browser._runtime import get_browser_runtime
runtime = get_browser_runtime()
controller = runtime.controller(binding["profile_id"])
if controller is not None and controller.task_id not in {None, task_id}:
return {"released": False}
+ if reset:
+ # Profile startup is deliberately shared/shielded. A terminated
+ # Activity can stop awaiting it while Chrome is still starting; keep
+ # the claim until that startup settles and its daemon can be retired.
+ starting = getattr(runtime, "_starting", {}).get(binding["profile_id"])
+ if starting is not None and not starting.done():
+ await asyncio.shield(starting)
+ controller = runtime.controller(binding["profile_id"])
+ if controller is not None and controller.task_id not in {None, task_id}:
+ return {"released": False}
running = runtime.running(binding["profile_id"]) if hasattr(runtime, "running") else None
if running is not None and controller is not None and controller.task_id == task_id:
+ if reset:
+ # Termination does not execute the agent's cancellation/finally
+ # path. Interrupt its command before waiting for the operation
+ # lock, rather than waiting indefinitely for a stuck CLI reply.
+ controller.needs_observation = True
+ await running.cli.interrupt()
# Wait for an admitted step to settle before releasing its lifetime claim.
# Suspension confirms the daemon cannot finish an old command later.
async with controller._op_lock:
diff --git a/server/services/browser_reset.py b/server/services/browser_reset.py
new file mode 100644
index 00000000..45b9a413
--- /dev/null
+++ b/server/services/browser_reset.py
@@ -0,0 +1,92 @@
+"""Owner cleanup for native browser claims whose generation was terminated.
+
+Direct Workspace tasks use their own cooperative cancellation barrier. A
+generation Reset instead verifies each durable claim against its exact native
+Workflow's recorded input before asking its owner to retire that task.
+"""
+from __future__ import annotations
+
+import asyncio
+from typing import Any
+
+from sqlalchemy import select
+from temporalio.api.enums.v1 import EventType
+from temporalio.client import WorkflowExecutionStatus
+from temporalio.service import RPCError, RPCStatusCode
+
+from models.browser_owners import BrowserProfileOwner
+from services import browser_owners
+
+_OWNER_CLEANUP_TIMEOUT = 60.0
+_NATIVE_TASK_TYPES = {"AgentWorkflow", "PiCodingAgentWorkflow", "MachinaWorkflow"}
+
+
+async def _recorded_scope(client: Any, task_id: str) -> tuple[Any, dict] | None:
+ """Read one pinned start event, including already-terminated executions."""
+ try:
+ description = await client.get_workflow_handle(task_id).describe()
+ if description.workflow_type not in _NATIVE_TASK_TYPES:
+ return None
+ handle = client.get_workflow_handle(task_id, run_id=description.run_id)
+ async for event in handle.fetch_history_events(page_size=1):
+ if event.event_type != EventType.EVENT_TYPE_WORKFLOW_EXECUTION_STARTED:
+ raise RuntimeError("browser_reset_execution_input_unavailable")
+ values = await client.data_converter.decode(
+ event.workflow_execution_started_event_attributes.input.payloads
+ )
+ if len(values) != 1 or not isinstance(values[0], dict):
+ raise RuntimeError("browser_reset_execution_input_unavailable")
+ return description, values[0]
+ raise RuntimeError("browser_reset_execution_input_unavailable")
+ except RPCError as exc:
+ if exc.status == RPCStatusCode.NOT_FOUND:
+ # Compatibility claims use local invocation tokens, not native
+ # Workflow IDs; their node lifecycle hooks own that cleanup.
+ return None
+ raise
+
+
+async def cleanup_generation_browser_tasks(database: Any, client: Any, control: Any) -> int:
+ """Require owner receipts for this generation's terminated native claims.
+
+ Durable task tokens and profile owners supply the destination. Recorded
+ Workflow input supplies generation proof; editable graphs and token naming
+ conventions never authorize cleanup. Failure propagates so Reset retains
+ its admission fence and can safely retry.
+ """
+ if not browser_owners._persistent(database):
+ return 0
+ async with database.get_session() as session:
+ claims = (await session.execute(select(BrowserProfileOwner).where(
+ BrowserProfileOwner.workflow_id == str(control.workflow_id),
+ BrowserProfileOwner.task_id.is_not(None),
+ ))).scalars().all()
+ if claims and client is None:
+ raise RuntimeError("browser_reset_temporal_unavailable")
+ cleaned = 0
+ for claim in claims:
+ recorded = await _recorded_scope(client, claim.task_id)
+ if recorded is None:
+ continue
+ description, scope = recorded
+ if (str(scope.get("workflow_id")) != str(control.workflow_id)
+ or str(scope.get("generation")) != str(control.generation)
+ or scope.get("_browser_task_id", claim.task_id) != claim.task_id):
+ continue
+ if description.status == WorkflowExecutionStatus.RUNNING:
+ raise RuntimeError("browser_reset_execution_still_running")
+ # Resolve current owner recovery routing for the stored profile, never
+ # the node's editable profile selection. Both local and remote cleanup
+ # check the exact task token again before interrupting anything.
+ binding = await browser_owners.bind_profile(database, claim.profile_id, claim.principal_id)
+ result = await asyncio.wait_for(browser_owners.cleanup_browser_task(
+ database, binding, claim.task_id, reset=True,
+ ), timeout=_OWNER_CLEANUP_TIMEOUT)
+ async with database.get_session() as session:
+ current = await session.get(BrowserProfileOwner, claim.profile_id)
+ if current is not None and current.task_id == claim.task_id:
+ raise RuntimeError("browser_reset_owner_cleanup_unacknowledged")
+ if result.get("released"):
+ cleaned += 1
+ # A concurrent terminal cleanup already released this exact claim.
+ return cleaned
diff --git a/server/services/deployment/execution_control.py b/server/services/deployment/execution_control.py
index f2dca0d4..e317740c 100644
--- a/server/services/deployment/execution_control.py
+++ b/server/services/deployment/execution_control.py
@@ -18,6 +18,10 @@ from core.logging import get_logger
logger = get_logger(__name__)
+class GenerationTransitionSuperseded(RuntimeError):
+ """A newer durable control intent owns this generation."""
+
+
async def count_generation_actions(client: Any, controller_id: str, status: dict) -> int:
"""A read-only UI count; transition verification uses acknowledgements."""
queue = [controller_id, *(status.get("live_roots") or {})]
@@ -38,14 +42,20 @@ async def count_generation_actions(client: Any, controller_id: str, status: dict
return count
-async def transition_generation(client: Any, control: Any, *, paused: bool, on_admission: Any = None) -> dict:
+async def transition_generation(client: Any, control: Any, *, paused: bool, on_admission: Any = None,
+ ensure_current: Any = None) -> dict:
controller = client.get_workflow_handle(control.controller_workflow_id)
revision = int(control.revision)
base = {"generation": int(control.generation), "revision": revision,
"state": "paused" if paused else "running"}
started = time.monotonic()
+ async def current() -> None:
+ if ensure_current is not None:
+ await ensure_current()
+
async def describe(workflow_id: str, chain: str) -> Any:
+ await current()
handle = client.get_workflow_handle(workflow_id)
try:
desc = await handle.describe()
@@ -66,6 +76,7 @@ async def transition_generation(client: Any, control: Any, *, paused: bool, on_a
raise RuntimeError("controller_execution_missing")
controller_chain = controller_description.raw_description.workflow_execution_info.first_run_id
controller_payload = {**base, "first_execution_run_id": controller_chain, "producers_held": True}
+ await current()
status = await controller.execute_update("set_control_state", controller_payload,
id=f"control:{revision}:admission")
if status.get("participant_revision", status.get("revision")) != revision:
@@ -75,6 +86,7 @@ async def transition_generation(client: Any, control: Any, *, paused: bool, on_a
order: list[str] = []
removed: set[tuple[str, str]] = set()
while True:
+ await current()
before = await controller.query("status")
epoch = before["membership_epoch"]
roots = before.get("live_roots", {})
@@ -92,6 +104,7 @@ async def transition_generation(client: Any, control: Any, *, paused: bool, on_a
if desc is None:
targets.pop(workflow_id, None)
if workflow_id in roots and (workflow_id, chain) not in removed:
+ await current()
await controller.execute_update("unregister_execution", {
"workflow_id": workflow_id, "first_execution_run_id": chain,
"generation": control.generation,
@@ -103,6 +116,7 @@ async def transition_generation(client: Any, control: Any, *, paused: bool, on_a
if workflow_id != control.controller_workflow_id:
payload = {**base, "first_execution_run_id": current_chain, "producers_held": True}
try:
+ await current()
ack = await handle.execute_update("set_control_state", payload,
id=f"control:{revision}:admission")
except RPCError as exc:
@@ -125,12 +139,14 @@ async def transition_generation(client: Any, control: Any, *, paused: bool, on_a
revision=revision, latency_seconds=time.monotonic() - started,
controlled_executions=len(targets))
if on_admission is not None:
+ await current()
await on_admission(await controller.query("status"))
if paused:
async def checkpoint(workflow_id: str, chain: str) -> None:
if await describe(workflow_id, chain) is None:
return
try:
+ await current()
result = await client.get_workflow_handle(workflow_id).execute_update(
"wait_for_checkpoint", {**base, "first_execution_run_id": chain},
id=f"control:{revision}:checkpoint")
@@ -141,7 +157,17 @@ async def transition_generation(client: Any, control: Any, *, paused: bool, on_a
if (not result.get("checkpoint") or result.get("state") != base["state"]
or result.get("revision", result.get("participant_revision")) != revision):
raise RuntimeError("temporal_checkpoint_ack_mismatch")
- await asyncio.gather(*(checkpoint(wid, chain) for wid, chain in targets.items()))
+ checkpoints = [asyncio.create_task(checkpoint(wid, chain)) for wid, chain in targets.items()]
+ try:
+ await asyncio.gather(*checkpoints)
+ finally:
+ # These tasks only poll Updates; cancelling a caller does not
+ # cancel the admitted Workflow actions or their durable intent.
+ for task in checkpoints:
+ if not task.done():
+ task.cancel()
+ await asyncio.gather(*checkpoints, return_exceptions=True)
+ await current()
after = await controller.query("status")
if after["membership_epoch"] != epoch:
continue
@@ -151,11 +177,13 @@ async def transition_generation(client: Any, control: Any, *, paused: bool, on_a
continue
chain = targets.get(workflow_id)
if chain and await describe(workflow_id, chain) is not None:
+ await current()
ack = await client.get_workflow_handle(workflow_id).execute_update(
"set_control_state", {**base, "first_execution_run_id": chain, "producers_held": False},
id=f"control:{revision}:release")
if ack.get("revision") != revision or ack.get("state") != "running" or ack.get("producers_held"):
raise RuntimeError("temporal_control_ack_mismatch")
+ await current()
status = await controller.execute_update("set_control_state", {
**controller_payload, "producers_held": False,
"expected_membership_epoch": epoch,
diff --git a/server/services/deployment/handlers.py b/server/services/deployment/handlers.py
index d4fc783f..00d6961b 100644
--- a/server/services/deployment/handlers.py
+++ b/server/services/deployment/handlers.py
@@ -24,6 +24,7 @@ import asyncio
import json
import time
import uuid
+import weakref
from datetime import datetime, timezone
from typing import Any, Dict, Optional
@@ -74,6 +75,79 @@ class ControllerExecutionMissing(RuntimeError):
# Maps workflow_id -> asyncio.Task for parallel workflow deployments.
_deployment_tasks: Dict[str, asyncio.Task] = {}
+# A socket only waits on these tasks. Disconnecting a waiter must not undo a
+# durable Stop, and status polling must never start a second checkpoint drain.
+_control_transition_tasks: Dict[tuple[str, int], asyncio.Task] = {}
+_control_completion_locks: weakref.WeakValueDictionary = weakref.WeakValueDictionary()
+# Counts and Queries are UI observations, never checkpoint acknowledgements.
+# An unavailable worker must not keep Reset behind an admission broadcast.
+_CONTROL_OBSERVATION_TIMEOUT_SECONDS = 1.0
+
+
+def _control_completion_lock(workflow_id: str) -> asyncio.Lock:
+ lock = _control_completion_locks.get(workflow_id)
+ if lock is None:
+ lock = asyncio.Lock()
+ _control_completion_locks[workflow_id] = lock
+ return lock
+
+
+def _schedule_versioned_transition(service: WorkflowControlService, control) -> asyncio.Task:
+ key = (control.id, int(control.revision))
+ task = _control_transition_tasks.get(key)
+ if task is not None and not task.done():
+ return task
+ task = asyncio.create_task(_finish_versioned_transition(service, control),
+ name=f"workflow-control:{control.id}:{control.revision}")
+ _control_transition_tasks[key] = task
+
+ def completed(done: asyncio.Task) -> None:
+ if _control_transition_tasks.get(key) is done:
+ _control_transition_tasks.pop(key, None)
+ if not done.cancelled() and (exc := done.exception()) is not None:
+ logger.warning("Workflow control reconciliation remains pending",
+ workflow_id=control.workflow_id, revision=control.revision,
+ error_type=type(exc).__name__, error=str(exc))
+
+ task.add_done_callback(completed)
+ return task
+
+
+def _supersede_versioned_transitions(control) -> None:
+ for (control_id, revision), task in list(_control_transition_tasks.items()):
+ if control_id == control.id and revision < control.revision and not task.done():
+ # Cancel local polling only. Reset's termination sweep is the
+ # authoritative cancellation of Temporal executions.
+ task.cancel()
+
+
+async def _await_versioned_transition(service: WorkflowControlService, control, *, task=None) -> Dict[str, Any]:
+ intent_revision, intent_status = control.revision, control.status
+ intent_identity = (control.id, control.generation)
+ task = task or _schedule_versioned_transition(service, control)
+ try:
+ current, controller_status = await asyncio.shield(task)
+ except asyncio.CancelledError:
+ # A Reset can retire a scheduled poll before its coroutine starts.
+ # Cancellation of just this socket waiter leaves task alive instead.
+ if not task.cancelled():
+ raise
+ current = await service.database.get_latest_workflow_control(control.workflow_id)
+ if current is None or (current.id, current.generation, current.revision, current.status) == (
+ intent_identity[0], intent_identity[1], intent_revision, intent_status
+ ):
+ raise
+ controller_status = None
+ target_status = "paused" if intent_status == "pausing" else "running"
+ return {
+ "success": True,
+ "superseded": ((current.id, current.generation) != intent_identity
+ or current.status not in {intent_status, target_status}
+ or current.revision not in {intent_revision, intent_revision + 1}),
+ "transition_pending": current.status in {"pausing", "resuming"},
+ **await _control_payload(current, controller_status=controller_status),
+ }
+
@ws_handler()
async def handle_deploy_workflow(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
@@ -643,7 +717,7 @@ async def _query_controller_state(control) -> Optional[Dict[str, Any]]:
if handle is None:
return None
try:
- result = await handle.query("status")
+ result = await asyncio.wait_for(handle.query("status"), timeout=_CONTROL_OBSERVATION_TIMEOUT_SECONDS)
return result if isinstance(result, dict) else None
except Exception as exc:
if _temporal_target_already_gone(exc):
@@ -1037,7 +1111,8 @@ async def _workspace_runtime_status(workflow_id: str) -> Dict[str, Any]:
if wrapper is None or wrapper.client is None:
return {"can_reset": True, "workspace_available": False} if nodes else {}
try:
- state = await controller_status(workflow_id, client=wrapper.client)
+ state = await asyncio.wait_for(controller_status(workflow_id, client=wrapper.client),
+ timeout=_CONTROL_OBSERVATION_TIMEOUT_SECONDS)
except Exception as exc:
logger.warning("Workspace task status unavailable", workflow_id=workflow_id, error_type=type(exc).__name__)
return {"can_reset": True, "workspace_available": False} if nodes else {}
@@ -1098,8 +1173,9 @@ async def _control_payload(
wrapper = container.temporal_client()
if wrapper is not None and wrapper.client is not None:
try:
- payload["generation_in_flight_count"] = await count_generation_actions(
- wrapper.client, control.controller_workflow_id, controller_status)
+ payload["generation_in_flight_count"] = await asyncio.wait_for(count_generation_actions(
+ wrapper.client, control.controller_workflow_id, controller_status),
+ timeout=_CONTROL_OBSERVATION_TIMEOUT_SECONDS)
except Exception as exc:
logger.debug("Generation activity count unavailable", workflow_id=control.workflow_id,
error=str(exc))
@@ -1143,6 +1219,12 @@ async def _reconcile_control(service: WorkflowControlService, control):
if control.status not in ACTIVE_STATES:
return control, None
+ if _versioned_control(control) and control.status in {"pausing", "resuming"}:
+ # A blocked admitted action can keep a checkpoint Update open for
+ # minutes. Status is a snapshot, never a waiter on that Update.
+ _schedule_versioned_transition(service, control)
+ return control, None
+
try:
controller_status = await _query_controller_state(control)
except ControllerExecutionMissing:
@@ -1154,8 +1236,6 @@ async def _reconcile_control(service: WorkflowControlService, control):
}.get(control.status)
if transition_target is None:
return control, controller_status
- if _versioned_control(control):
- return await _finish_versioned_transition(service, control)
requested_state, stable_state = transition_target
if controller_status is None:
@@ -1239,62 +1319,109 @@ async def _reconcile_control(service: WorkflowControlService, control):
async def _finish_versioned_transition(service: WorkflowControlService, control):
- """Finish durable intent without cancelling admitted work on RPC timeout."""
+ """Drain the admitted work while allowing a newer Reset to take over."""
from core.container import container
- from services.deployment.execution_control import transition_generation
-
- paused = control.status == "pausing"
- runtime = container.workflow_service()
- runtime.pause_deployment(control.workflow_id)
- if paused:
- schedules = await _set_cron_pause(control.workflow_id, paused=True, strict=True)
- triggers = await runtime.update_trigger_pause_status(control.workflow_id, paused=True)
- wrapper = container.temporal_client()
- if wrapper is None or wrapper.client is None:
- if container.settings().temporal_enabled:
- raise TemporalControlUnavailable("temporal_control_unavailable")
- result = {"controller_status": None, "controlled_executions": 0}
- else:
- async def admitted(status):
- try:
- await _broadcast_control(control, controller_status=status)
- except Exception as exc:
- logger.warning("Control admission status broadcast failed", workflow_id=control.workflow_id,
- error=str(exc))
- result = await transition_generation(wrapper.client, control, paused=paused, on_admission=admitted)
- details = {"controlled_executions": result["controlled_executions"]}
- values: Dict[str, Any] = {}
- if paused:
- details.update(paused_schedules=schedules, paused_triggers=triggers)
- else:
- # The existing execution continuations are released before producers.
- details["resumed_schedules"] = await _set_cron_pause(control.workflow_id, paused=False, strict=True)
- details["resumed_queued_events"] = await runtime.resume_deployment(control.workflow_id)
- details["resumed_triggers"] = await runtime.update_trigger_pause_status(control.workflow_id, paused=False)
- values.update(CLEAR_PAUSE_REASON)
- values["resource_manifest"] = {**(control.resource_manifest or {}),
- "last_resumed_at": datetime.now(timezone.utc).isoformat()}
- try:
- control = await service.transition(control, expected_revision=control.revision,
- from_statuses={control.status}, status="paused" if paused else "running",
- values=values or None)
- except ValueError as exc:
- if str(exc) != "control_revision_conflict":
- raise
+ from temporalio.client import WorkflowUpdateRPCTimeoutOrCancelledError
+ from services.deployment.execution_control import GenerationTransitionSuperseded, transition_generation
+
+ intent = (control.id, control.generation, control.revision, control.status)
+ paused = intent[3] == "pausing"
+
+ async def latest_control():
latest = await service.database.get_latest_workflow_control(control.workflow_id)
- if latest is None or latest.generation != control.generation:
- raise
- return latest, result["controller_status"]
- if not paused:
- await _clear_failure_streak(service.database, control)
- payload = await _broadcast_control(control, controller_status=result["controller_status"], extra=details)
- from services.status_broadcaster import get_status_broadcaster
- broadcaster = get_status_broadcaster()
- await broadcaster.update_workflow_status(executing=bool(not paused and payload.get("in_flight_count")),
- current_node=None, progress=0, workflow_id=control.workflow_id)
- await broadcaster.update_deployment_status(is_running=True, status=control.status,
- active_runs=payload.get("generation_in_flight_count", 0), workflow_id=control.workflow_id)
- return control, result["controller_status"]
+ if latest is None:
+ raise RuntimeError("workflow_control_missing")
+ return latest
+
+ def owns(latest) -> bool:
+ return (latest.id, latest.generation, latest.revision, latest.status) == intent
+
+ async def ensure_current() -> None:
+ if not owns(await latest_control()):
+ raise GenerationTransitionSuperseded()
+
+ try:
+ runtime = container.workflow_service()
+ # Reset shares this short completion lock. It never waits for the
+ # activity drain; an older transition cannot gate or reopen a later
+ # generation's producers after Reset has claimed the durable row.
+ async with _control_completion_lock(control.workflow_id):
+ await ensure_current()
+ runtime.pause_deployment(control.workflow_id)
+ if paused:
+ schedules = await _set_cron_pause(control.workflow_id, paused=True, strict=True)
+ await ensure_current()
+ triggers = await runtime.update_trigger_pause_status(control.workflow_id, paused=True)
+ wrapper = container.temporal_client()
+ if wrapper is None or wrapper.client is None:
+ if container.settings().temporal_enabled:
+ raise TemporalControlUnavailable("temporal_control_unavailable")
+ result = {"controller_status": None, "controlled_executions": 0}
+ else:
+ async def admitted(status):
+ async with _control_completion_lock(control.workflow_id):
+ await ensure_current()
+ try:
+ await _broadcast_control(control, controller_status=status)
+ except Exception as exc:
+ logger.warning("Control admission status broadcast failed", workflow_id=control.workflow_id,
+ error=str(exc))
+ result = await transition_generation(wrapper.client, control, paused=paused,
+ on_admission=admitted, ensure_current=ensure_current)
+ async with _control_completion_lock(control.workflow_id):
+ await ensure_current()
+ details = {"controlled_executions": result["controlled_executions"]}
+ values: Dict[str, Any] = {}
+ if paused:
+ details.update(paused_schedules=schedules, paused_triggers=triggers)
+ else:
+ # Existing continuations are released before producers, and
+ # Reset cannot interleave with these producer mutations.
+ details["resumed_schedules"] = await _set_cron_pause(control.workflow_id, paused=False, strict=True)
+ await ensure_current()
+ details["resumed_queued_events"] = await runtime.resume_deployment(control.workflow_id)
+ await ensure_current()
+ details["resumed_triggers"] = await runtime.update_trigger_pause_status(control.workflow_id, paused=False)
+ await ensure_current()
+ values.update(CLEAR_PAUSE_REASON)
+ values["resource_manifest"] = {**(control.resource_manifest or {}),
+ "last_resumed_at": datetime.now(timezone.utc).isoformat()}
+ control = await service.transition(control, expected_revision=intent[2],
+ from_statuses={intent[3]}, status="paused" if paused else "running", values=values or None)
+ if not paused:
+ await _clear_failure_streak(service.database, control)
+ payload = await _broadcast_control(control, controller_status=result["controller_status"], extra=details)
+ from services.status_broadcaster import get_status_broadcaster
+ broadcaster = get_status_broadcaster()
+ await broadcaster.update_workflow_status(executing=bool(not paused and payload.get("in_flight_count")),
+ current_node=None, progress=0, workflow_id=control.workflow_id)
+ await broadcaster.update_deployment_status(is_running=True, status=control.status,
+ active_runs=payload.get("generation_in_flight_count", 0), workflow_id=control.workflow_id)
+ return control, result["controller_status"]
+ except (Exception, asyncio.CancelledError) as exc:
+ latest = await latest_control()
+ if not owns(latest):
+ # Reset may terminate the execution or cancel this local poll
+ # before an Update produces an outcome. That is supersession,
+ # never a failed Reset or a successful pause acknowledgement.
+ logger.debug("Workflow control transition superseded", workflow_id=control.workflow_id,
+ revision=intent[2], current_revision=latest.revision, state=latest.status)
+ return latest, None
+ if isinstance(exc, WorkflowUpdateRPCTimeoutOrCancelledError):
+ if isinstance(exc.__cause__, asyncio.CancelledError):
+ raise asyncio.CancelledError() from exc
+ # A bounded RPC poll can fail without rejecting the durable
+ # Update. Keep intent and closed admission; status can retry it.
+ logger.info("Workflow control acknowledgement remains pending", workflow_id=control.workflow_id,
+ revision=intent[2], error_type=type(exc).__name__)
+ return latest, None
+ if str(exc) == "controller_execution_missing":
+ async with _control_completion_lock(control.workflow_id):
+ latest = await latest_control()
+ if not owns(latest):
+ return latest, None
+ return await _fail_missing_controller(service, latest), None
+ raise
# ---------------------------------------------------------------------------
@@ -1357,6 +1484,11 @@ async def mark_clean_shutdown() -> None:
"""Stamp the clean-shutdown marker (lifespan teardown hook)."""
from core.container import container
+ tasks = list(_control_transition_tasks.values())
+ for task in tasks:
+ task.cancel()
+ if tasks:
+ await asyncio.gather(*tasks, return_exceptions=True)
try:
await container.database().set_cache_entry(_SHUTDOWN_STATE_CACHE_KEY, "clean")
except Exception as exc: # noqa: BLE001 — best-effort during teardown
@@ -2041,6 +2173,8 @@ async def handle_pause_workflow(data: Dict[str, Any], websocket: WebSocket) -> D
**await _control_payload(control, controller_status=controller_status),
}
if control.status == "pausing":
+ if _versioned_control(control):
+ return await _await_versioned_transition(service, control)
return {
"success": False,
"error": "workflow_control_transition_pending",
@@ -2050,18 +2184,19 @@ async def handle_pause_workflow(data: Dict[str, Any], websocket: WebSocket) -> D
# client must not be able to dress its own pause up as an automatic one.
reason_values = _automatic_pause_values(data) if websocket is None else None
transition_kwargs: Dict[str, Any] = {"values": reason_values} if reason_values else {}
- control = await service.transition(
- control,
- expected_revision=_expected_revision(data, control),
- from_statuses={"running"},
- status="pausing",
- **transition_kwargs,
- )
- await _broadcast_control(control)
- container.workflow_service().pause_deployment(workflow_id)
+ async with _control_completion_lock(workflow_id):
+ control = await service.transition(
+ control,
+ expected_revision=_expected_revision(data, control),
+ from_statuses={"running"},
+ status="pausing",
+ **transition_kwargs,
+ )
+ transition_task = _schedule_versioned_transition(service, control) if _versioned_control(control) else None
+ await _broadcast_control(control)
+ container.workflow_service().pause_deployment(workflow_id)
if _versioned_control(control):
- control, controller_status = await _finish_versioned_transition(service, control)
- return {"success": True, **await _control_payload(control, controller_status=controller_status)}
+ return await _await_versioned_transition(service, control, task=transition_task)
try:
controller_status = await _update_controller_state(
control,
@@ -2137,18 +2272,21 @@ async def handle_resume_workflow(data: Dict[str, Any], websocket: WebSocket) ->
**await _control_payload(control, controller_status=controller_status),
}
if control.status == "resuming":
+ if _versioned_control(control):
+ return await _await_versioned_transition(service, control)
return {
"success": False,
"error": "workflow_control_transition_pending",
**await _control_payload(control, controller_status=controller_status),
}
- control = await service.transition(
- control, expected_revision=_expected_revision(data, control), from_statuses={"paused"}, status="resuming"
- )
- await _broadcast_control(control)
+ async with _control_completion_lock(workflow_id):
+ control = await service.transition(
+ control, expected_revision=_expected_revision(data, control), from_statuses={"paused"}, status="resuming"
+ )
+ transition_task = _schedule_versioned_transition(service, control) if _versioned_control(control) else None
+ await _broadcast_control(control)
if _versioned_control(control):
- control, controller_status = await _finish_versioned_transition(service, control)
- return {"success": True, **await _control_payload(control, controller_status=controller_status)}
+ return await _await_versioned_transition(service, control, task=transition_task)
resume_update_key = data.get("idempotency_key") or uuid.uuid4().hex
async def _apply_resume_update(target):
@@ -2243,6 +2381,10 @@ async def handle_reset_workflow(data: Dict[str, Any], websocket: WebSocket) -> D
workflow_id = data["workflow_id"]
service = _control_service()
current = await service.database.get_latest_workflow_control(workflow_id)
+ if data.get("expected_root_execution_id") and (
+ current is None or data["expected_root_execution_id"] != current.root_execution_id
+ ):
+ raise ValueError("control_generation_conflict")
if current is None or current.status == "reset":
# Workspace tasks also exist before Start and after a generation was
# reset. Their own Temporal controller supplies the cleanup barrier.
@@ -2266,18 +2408,29 @@ async def handle_reset_workflow(data: Dict[str, Any], websocket: WebSocket) -> D
# The completed-generation branch above only resets direct Workspace
# tasks. Generation-wide cleanup below must not sweep a later Start.
- if current.status != "resetting":
- current = await service.transition(
- current, expected_revision=_expected_revision(data, current),
- from_statuses={"starting", "running", "pausing", "paused", "resuming", "failed"}, status="resetting",
- values={"terminal_reason": "workflow_reset", "completed_at": datetime.now(timezone.utc)},
- )
+ async with _control_completion_lock(workflow_id):
+ # Completion can win while Reset waits for this short lock. Reload
+ # before CAS and never let a delayed Reset sweep a later generation.
+ latest = await service.database.get_latest_workflow_control(workflow_id)
+ if latest is None or latest.id != current.id or latest.generation != current.generation:
+ raise ValueError("control_generation_conflict")
+ current = latest
+ if current.status != "resetting":
+ # A generation-scoped Reset supersedes Stop/Resume even when
+ # their stable acknowledgement advanced the revision in flight.
+ # Older clients keep their strict revision-only CAS contract.
+ expected_revision = current.revision if data.get("expected_root_execution_id") else _expected_revision(data, current)
+ current = await service.transition(
+ current, expected_revision=expected_revision,
+ from_statuses={"starting", "running", "pausing", "paused", "resuming", "failed"}, status="resetting",
+ values={"terminal_reason": "workflow_reset", "completed_at": datetime.now(timezone.utc)},
+ )
- # Quiesce every producer before the final execution sweep. The local gate
- # is synchronous, so no callback can be admitted while broadcasts or
- # Temporal cleanup yield control.
- _close_local_admission(workflow_id)
- await _broadcast_control(current)
+ # The durable Reset revision supersedes socket-independent Stop
+ # polling. Admission closes before any cleanup/broadcast yields.
+ _close_local_admission(workflow_id)
+ _supersede_versioned_transitions(current)
+ await _broadcast_control(current)
try:
await _signal_controller(current, "reset")
@@ -2321,6 +2474,12 @@ async def handle_reset_workflow(data: Dict[str, Any], websocket: WebSocket) -> D
# Controller, cron, and legacy local admission paths are now closed. This
# final strict sweep therefore observes a fixed set of generation runs.
terminated = await _terminate_generation_workflows(current, strict=True)
+ from core.container import container
+ from services.browser_reset import cleanup_generation_browser_tasks
+ wrapper = container.temporal_client()
+ cleaned_browser_tasks = await cleanup_generation_browser_tasks(
+ service.database, wrapper.client if wrapper is not None else None, current,
+ )
workspace = await _reset_workspace_tasks(workflow_id, service.database, current, data)
archived = await service.database.update_workflow_run_data_scope(
@@ -2353,6 +2512,7 @@ async def handle_reset_workflow(data: Dict[str, Any], websocket: WebSocket) -> D
payload = await _broadcast_control(current, extra={
"terminated_executions": terminated,
"deleted_schedules": deleted_schedules,
+ "cleaned_browser_tasks": cleaned_browser_tasks,
"local_cleanup_completed": local_cleanup_completed,
"archived_nodes": node_state["archived_nodes"],
"reset_nodes": node_state["reset_nodes"],
@@ -2362,6 +2522,7 @@ async def handle_reset_workflow(data: Dict[str, Any], websocket: WebSocket) -> D
"idempotent": False,
"terminated_executions": terminated,
"deleted_schedules": deleted_schedules,
+ "cleaned_browser_tasks": cleaned_browser_tasks,
"local_cleanup_completed": local_cleanup_completed,
"archived_nodes": node_state["archived_nodes"],
"reset_nodes": node_state["reset_nodes"],
diff --git a/server/tests/nodes/browser/test_browser_owners.py b/server/tests/nodes/browser/test_browser_owners.py
index e89b0a2b..7bd557d5 100644
--- a/server/tests/nodes/browser/test_browser_owners.py
+++ b/server/tests/nodes/browser/test_browser_owners.py
@@ -16,6 +16,26 @@ from services.plugin.base import NodeUserError
from nodes.browser._session import ProfileController
+async def native_reset_client(scope, *, status=None, workflow_type="AgentWorkflow"):
+ from temporalio.api.enums.v1 import EventType
+ from temporalio.api.history.v1 import HistoryEvent, WorkflowExecutionStartedEventAttributes
+ from temporalio.api.common.v1 import Payloads
+ from temporalio.client import WorkflowExecutionStatus
+ from temporalio.converter import DataConverter
+
+ converter = DataConverter.default
+ opening = HistoryEvent(event_id=1, event_type=EventType.EVENT_TYPE_WORKFLOW_EXECUTION_STARTED,
+ workflow_execution_started_event_attributes=WorkflowExecutionStartedEventAttributes(
+ input=Payloads(payloads=await converter.encode([scope]))))
+ async def history(**kwargs):
+ assert kwargs == {"page_size": 1}
+ yield opening
+ handle = SimpleNamespace(describe=AsyncMock(return_value=SimpleNamespace(
+ workflow_type=workflow_type, run_id="terminated-run",
+ status=status or WorkflowExecutionStatus.TERMINATED)), fetch_history_events=history)
+ return SimpleNamespace(get_workflow_handle=Mock(return_value=handle), data_converter=converter)
+
+
@pytest.fixture
async def database():
engine = create_async_engine("sqlite+aiosqlite:///:memory:")
@@ -50,6 +70,177 @@ async def test_task_claim_blocks_other_tasks_and_cleanup_is_token_matched(databa
assert controller.task_id == "second"
[email protected]("workflow_type", ["AgentWorkflow", "MachinaWorkflow"])
+async def test_generation_reset_retires_terminated_native_claim_and_interrupts_stuck_step(database, workflow_type):
+ from services.browser_reset import cleanup_generation_browser_tasks
+ from nodes.browser.browser import BrowserNode
+
+ db, controller = database
+ binding = {**await owners.bind_profile(db, "profile", "owner"), "workflow_id": "wf", "node_id": "browser"}
+ task_id = "native-agent-workflow"
+ await owners.claim_browser_task(db, binding, "owner", task_id)
+ session = SimpleNamespace(profile_id="profile", session_id="session")
+ runtime = SimpleNamespace(controller=lambda _: controller, find_session=lambda *_: session,
+ running=lambda _: running)
+ interrupted = asyncio.Event()
+ async def interrupt():
+ interrupted.set()
+ cli = SimpleNamespace(interrupt=AsyncMock(side_effect=interrupt), suspend_for_credentials=AsyncMock(return_value=True))
+ running = SimpleNamespace(cli=cli)
+ async def stuck_step():
+ async with controller._op_lock:
+ await interrupted.wait()
+ step = asyncio.create_task(stuck_step())
+ await asyncio.sleep(0)
+ client = await native_reset_client({"workflow_id": "wf", "generation": 7}, workflow_type=workflow_type)
+ control = SimpleNamespace(workflow_id="wf", generation=7, graph_snapshot={"nodes": []})
+ with patch("nodes.browser._runtime.get_browser_runtime", return_value=runtime), \
+ patch("nodes.browser._runtime.peek_browser_runtime", return_value=runtime):
+ assert await BrowserNode.reset_execution_state(node_id="browser", workflow_id="wf",
+ execution_id="deployment", generation=7, graph={}, database=db) == {"reset": False, "managed_by_agent": True}
+ assert await cleanup_generation_browser_tasks(db, client, control) == 1
+ assert await cleanup_generation_browser_tasks(db, client, control) == 0
+ await step
+ cli.interrupt.assert_awaited_once()
+ cli.suspend_for_credentials.assert_awaited_once()
+ client.get_workflow_handle.assert_any_call(task_id, run_id="terminated-run")
+ assert controller.task_id is None and controller.needs_observation
+ async with db.get_session() as db_session:
+ assert (await db_session.get(BrowserProfileOwner, "profile")).task_id is None
+
+
+ {"workflow_id": "wf", "generation": 0}, # Independent Workspace task.
+ {"workflow_id": "wf", "generation": 8},
+ {"workflow_id": "other", "generation": 7},
+ {"workflow_id": "wf", "generation": 7, "_browser_task_id": "unrelated-token"},
+])
+async def test_generation_reset_preserves_claims_outside_recorded_scope(database, scope):
+ from services.browser_reset import cleanup_generation_browser_tasks
+
+ db, controller = database
+ binding = {**await owners.bind_profile(db, "profile", "owner"), "workflow_id": "wf"}
+ await owners.claim_browser_task(db, binding, "owner", "task")
+ client = await native_reset_client(scope)
+ with patch.object(owners, "cleanup_browser_task", AsyncMock()) as cleanup:
+ assert await cleanup_generation_browser_tasks(db, client, SimpleNamespace(workflow_id="wf", generation=7)) == 0
+ cleanup.assert_not_awaited()
+ assert controller.task_id == "task"
+
+
+async def test_generation_reset_keeps_claim_when_native_execution_remains_running(database):
+ from services.browser_reset import cleanup_generation_browser_tasks
+ from temporalio.client import WorkflowExecutionStatus
+
+ db, controller = database
+ binding = {**await owners.bind_profile(db, "profile", "owner"), "workflow_id": "wf"}
+ await owners.claim_browser_task(db, binding, "owner", "task")
+ client = await native_reset_client({"workflow_id": "wf", "generation": 7}, status=WorkflowExecutionStatus.RUNNING)
+ with patch.object(owners, "cleanup_browser_task", AsyncMock()) as cleanup, \
+ pytest.raises(RuntimeError, match="still_running"):
+ await cleanup_generation_browser_tasks(db, client, SimpleNamespace(workflow_id="wf", generation=7))
+ cleanup.assert_not_awaited()
+ assert controller.task_id == "task"
+
+
[email protected]("receipt", [{"released": False}, {"released": True}])
+async def test_generation_reset_requires_durable_owner_cleanup_receipt(database, receipt):
+ from services.browser_reset import cleanup_generation_browser_tasks
+
+ db, controller = database
+ binding = {**await owners.bind_profile(db, "profile", "owner"), "workflow_id": "wf"}
+ await owners.claim_browser_task(db, binding, "owner", "task")
+ client = await native_reset_client({"workflow_id": "wf", "generation": 7})
+ with patch.object(owners, "cleanup_browser_task", AsyncMock(return_value=receipt)), \
+ pytest.raises(RuntimeError, match="unacknowledged"):
+ await cleanup_generation_browser_tasks(db, client, SimpleNamespace(workflow_id="wf", generation=7))
+ assert controller.task_id == "task"
+
+
+async def test_reset_owner_cleanup_never_interrupts_replacement_task(database):
+ db, controller = database
+ binding = {**await owners.bind_profile(db, "profile", "owner"), "workflow_id": "wf"}
+ await owners.claim_browser_task(db, binding, "owner", "replacement")
+ cli = SimpleNamespace(interrupt=AsyncMock(), suspend_for_credentials=AsyncMock(return_value=True))
+ runtime = SimpleNamespace(controller=lambda _: controller, running=lambda _: SimpleNamespace(cli=cli))
+ with patch("nodes.browser._runtime.get_browser_runtime", return_value=runtime):
+ assert await owners.cleanup_browser_task(db, binding, "terminated-task", reset=True) == {"released": False}
+ cli.interrupt.assert_not_awaited()
+ assert controller.task_id == "replacement"
+
+
+async def test_reset_owner_cleanup_waits_for_shielded_browser_start(database):
+ db, controller = database
+ binding = {**await owners.bind_profile(db, "profile", "owner"), "workflow_id": "wf"}
+ await owners.claim_browser_task(db, binding, "owner", "task")
+ starting = asyncio.get_running_loop().create_future()
+ cli = SimpleNamespace(interrupt=AsyncMock(), suspend_for_credentials=AsyncMock(return_value=True))
+ runtime = SimpleNamespace(controller=lambda _: controller, _starting={"profile": starting},
+ running=lambda _: SimpleNamespace(cli=cli) if starting.done() else None)
+ with patch("nodes.browser._runtime.get_browser_runtime", return_value=runtime):
+ cleanup = asyncio.create_task(owners.cleanup_browser_task(db, binding, "task", reset=True))
+ await asyncio.sleep(0.01)
+ assert not cleanup.done() and controller.task_id == "task"
+ starting.set_result(None)
+ assert await cleanup == {"released": True}
+ cli.interrupt.assert_awaited_once()
+ cli.suspend_for_credentials.assert_awaited_once()
+
+
+async def test_remote_reset_cleanup_forwards_exact_task_and_reset_intent(database):
+ db, _ = database
+ binding = await owners.bind_profile(db, "profile", "owner")
+ forward = AsyncMock(return_value={"released": True})
+ with patch.object(owners, "replica_id", return_value="other-replica"), \
+ patch("nodes.browser._routing.forward_command", forward):
+ assert await owners.cleanup_browser_task(db, binding, "native-task", reset=True) == {"released": True}
+ forward.assert_awaited_once_with(binding, "owner", "cleanup_task",
+ {"profile_id": "profile", "task_id": "native-task", "reset": True})
+
+
+async def test_owner_command_preserves_authenticated_reset_cleanup_intent(database):
+ import json
+ from nodes.browser import _routing
+
+ db, _ = database
+ binding = await owners.bind_profile(db, "profile", "owner")
+ body = json.dumps({"command": "cleanup_task", "data": {
+ "profile_id": "profile", "task_id": "native-task", "reset": True}}).encode()
+ request = SimpleNamespace(body=AsyncMock(return_value=body), headers={
+ "X-Browser-Forward": owners.sign_forward("owner", "POST", _routing.COMMAND_PATH, body)})
+ cleanup = AsyncMock(return_value={"released": True})
+ with patch.object(_routing, "get_database", return_value=db), \
+ patch.object(_routing, "command_binding", AsyncMock(return_value=binding)), \
+ patch.object(owners, "cleanup_browser_task", cleanup):
+ assert await _routing.owner_command(request) == {"released": True}
+ cleanup.assert_awaited_once_with(db, {**binding, "principal_id": "owner"}, "native-task", reset=True)
+
+
+async def test_direct_browser_reset_interrupts_unclaimed_terminated_activity_before_releasing_lease(database):
+ from nodes.browser.browser import BrowserNode
+
+ db, controller = database
+ session = SimpleNamespace(profile_id="profile", session_id="session")
+ controller.lease_session = session
+ interrupted = asyncio.Event()
+ cli = SimpleNamespace(interrupt=AsyncMock(side_effect=lambda: interrupted.set()),
+ suspend_for_credentials=AsyncMock(return_value=True))
+ runtime = SimpleNamespace(controller=lambda _: controller, find_session=lambda *_: session,
+ running=lambda _: SimpleNamespace(cli=cli))
+ async def step():
+ async with controller._op_lock:
+ await interrupted.wait()
+ activity = asyncio.create_task(step())
+ await asyncio.sleep(0)
+ with patch("nodes.browser._runtime.peek_browser_runtime", return_value=runtime):
+ assert await BrowserNode.reset_execution_state(node_id="browser", workflow_id="wf",
+ execution_id="deployment", generation=7, graph={}, database=db) == {"reset": True}
+ await activity
+ assert controller.lease_session is None
+ cli.interrupt.assert_awaited_once()
+ cli.suspend_for_credentials.assert_awaited_once()
+
+
async def test_unavailable_owner_is_never_reassigned(database):
db, _ = database
async with db.get_session() as session:
diff --git a/server/tests/services/test_versioned_workflow_control.py b/server/tests/services/test_versioned_workflow_control.py
index 2c91fc41..9be8d6f6 100644
--- a/server/tests/services/test_versioned_workflow_control.py
+++ b/server/tests/services/test_versioned_workflow_control.py
@@ -1,17 +1,32 @@
"""Durable transition intent and compatibility of generation-level control."""
+import asyncio
+from copy import copy
from types import SimpleNamespace
from unittest.mock import AsyncMock
import pytest
+import pytest_asyncio
from services.deployment import handlers
from services.deployment import execution_control
+@pytest_asyncio.fixture(autouse=True)
+async def drain_supervised_transitions():
+ yield
+ tasks = list(handlers._control_transition_tasks.values())
+ for task in tasks:
+ task.cancel()
+ if tasks:
+ await asyncio.gather(*tasks, return_exceptions=True)
+ handlers._control_transition_tasks.clear()
+
+
def control(state="pausing"):
return SimpleNamespace(id="control:1", workflow_id="canvas", generation=1, revision=3,
root_execution_id="generation-root", controller_workflow_id="controller", controller_run_id="first",
- status=state, resource_manifest={"execution_control_version": 1})
+ execution_id="generation-root", data_scope_id="generation-root", created_at=None, updated_at=None,
+ terminal_reason=None, status=state, resource_manifest={"execution_control_version": 1})
@pytest.mark.asyncio
@@ -27,7 +42,8 @@ async def test_stop_timeout_keeps_intent_and_closed_producers(monkeypatch):
monkeypatch.setattr(handlers, "_set_cron_pause", schedules)
transition = AsyncMock(side_effect=TimeoutError("request timeout; tool still running"))
monkeypatch.setattr(execution_control, "transition_generation", transition)
- service = SimpleNamespace(transition=AsyncMock())
+ service = SimpleNamespace(transition=AsyncMock(), database=SimpleNamespace(
+ get_latest_workflow_control=AsyncMock(return_value=row)))
with pytest.raises(TimeoutError):
await handlers._finish_versioned_transition(service, row)
assert row.status == "pausing"
@@ -66,7 +82,8 @@ async def test_resume_releases_continuations_before_producers_and_saves_timeout_
monkeypatch.setattr(handlers, "_broadcast_control", AsyncMock(return_value={}))
from services import status_broadcaster
monkeypatch.setattr(status_broadcaster, "get_status_broadcaster", lambda: AsyncMock())
- result, _ = await handlers._finish_versioned_transition(SimpleNamespace(transition=stable, database=object()), row)
+ database = SimpleNamespace(get_latest_workflow_control=AsyncMock(return_value=row))
+ result, _ = await handlers._finish_versioned_transition(SimpleNamespace(transition=stable, database=database), row)
assert result.status == "running"
assert order == ["continuations", "schedules", "local", "running"]
@@ -89,3 +106,404 @@ async def test_chat_generation_guard_precedes_reconciliation(monkeypatch):
result = await handlers.handle_pause_workflow({"workflow_id": "canvas", "expected_root_execution_id": "other"}, None)
assert result["error"] == "control_generation_conflict"
reconcile.assert_not_awaited()
+
+
+def lifecycle(monkeypatch, state="running"):
+ """An immutable-snapshot control store and observable lifecycle effects."""
+ from core.container import container
+ from services import status_broadcaster, browser_reset
+ from services.deployment import runtime_state
+
+ row = control(state)
+ row.revision = 2 if state == "running" else 3
+ row.execution_id = row.data_scope_id = "generation-root"
+ record = SimpleNamespace(row=row, broadcasts=[], effects=[])
+ database = SimpleNamespace(
+ get_latest_workflow_control=AsyncMock(side_effect=lambda _: copy(record.row)),
+ update_workflow_run_data_scope=AsyncMock(return_value=True),
+ )
+
+ async def transition(target, *, expected_revision, from_statuses, status, values=None):
+ if record.row.revision != expected_revision or record.row.status not in from_statuses:
+ raise ValueError("control_revision_conflict")
+ record.row = copy(record.row)
+ record.row.revision += 1
+ record.row.status = status
+ for key, value in (values or {}).items():
+ setattr(record.row, key, value)
+ return copy(record.row)
+
+ service = SimpleNamespace(database=database, transition=AsyncMock(side_effect=transition))
+
+ async def payload(target, **kwargs):
+ return {"state": "ready" if target.status == "reset" else target.status,
+ "revision": target.revision, "generation": target.generation}
+
+ async def broadcast(target, **kwargs):
+ record.broadcasts.append((target.status, target.revision))
+ return await payload(target)
+
+ runtime = SimpleNamespace(
+ pause_deployment=lambda _: record.effects.append("close_local"),
+ update_trigger_pause_status=AsyncMock(return_value=1), resume_deployment=AsyncMock(return_value=0),
+ )
+ record.service, record.runtime = service, runtime
+ monkeypatch.setattr(handlers, "_control_service", lambda: service)
+ monkeypatch.setattr(container, "workflow_service", lambda: runtime)
+ monkeypatch.setattr(container, "temporal_client", lambda: SimpleNamespace(client=object()))
+ monkeypatch.setattr(handlers, "_control_payload", payload)
+ monkeypatch.setattr(handlers, "_broadcast_control", broadcast)
+ monkeypatch.setattr(handlers, "_query_controller_state", AsyncMock(return_value={"state": state}))
+ monkeypatch.setattr(handlers, "_set_cron_pause", AsyncMock(return_value=1))
+ monkeypatch.setattr(handlers, "_delete_cron_schedules", AsyncMock(return_value=1))
+ monkeypatch.setattr(handlers, "_signal_controller", AsyncMock())
+ monkeypatch.setattr(handlers, "_terminate_generation_workflows", AsyncMock(return_value=3))
+ monkeypatch.setattr(handlers, "_reset_workspace_tasks", AsyncMock(return_value={"cancelled": 0}))
+ monkeypatch.setattr(handlers, "handle_cancel_deployment", AsyncMock(return_value={"success": True}))
+ monkeypatch.setattr(handlers, "_clear_failure_streak", AsyncMock())
+ monkeypatch.setattr(browser_reset, "cleanup_generation_browser_tasks", AsyncMock(return_value=1))
+ monkeypatch.setattr(runtime_state, "archive_and_reset_node_state",
+ AsyncMock(return_value={"archived_nodes": [], "reset_nodes": []}))
+ monkeypatch.setattr(status_broadcaster, "get_status_broadcaster", lambda: AsyncMock())
+ return record
+
+
+async def test_blocked_stop_status_returns_and_reset_supersedes_poll(monkeypatch):
+ from temporalio.client import WorkflowUpdateRPCTimeoutOrCancelledError
+ record = lifecycle(monkeypatch)
+ admitted, blocked = asyncio.Event(), asyncio.Event()
+
+ async def stuck(*args, on_admission, **kwargs):
+ await on_admission({"state": "paused"})
+ admitted.set()
+ try:
+ await blocked.wait()
+ except asyncio.CancelledError as exc:
+ # The actual SDK wraps local Update polling cancellation this way.
+ raise WorkflowUpdateRPCTimeoutOrCancelledError() from exc
+
+ temporal = AsyncMock(side_effect=stuck)
+ monkeypatch.setattr(execution_control, "transition_generation", temporal)
+ stop = asyncio.create_task(handlers.handle_pause_workflow(
+ {"workflow_id": "canvas", "expected_revision": 2}, None))
+ await asyncio.wait_for(admitted.wait(), 1)
+ for _ in range(3):
+ snapshot = await asyncio.wait_for(handlers.handle_get_workflow_control_status({"workflow_id": "canvas"}, None), 0.2)
+ assert snapshot["state"] == "pausing"
+ assert not stop.done()
+ assert temporal.await_count == 1
+ handlers._query_controller_state.assert_awaited_once()
+
+ reset = await asyncio.wait_for(handlers.handle_reset_workflow(
+ {"workflow_id": "canvas", "expected_revision": 3,
+ "expected_root_execution_id": "generation-root"}, None), 1)
+ outcome = await asyncio.wait_for(stop, 1)
+ assert reset["success"] and reset["state"] == "ready"
+ assert outcome["success"] and outcome["superseded"]
+ assert outcome["state"] in {"resetting", "ready"}
+ assert record.row.status == "reset"
+ assert record.broadcasts == [("pausing", 3), ("pausing", 3), ("resetting", 4), ("reset", 5)]
+ record.runtime.resume_deployment.assert_not_awaited()
+
+
+async def test_disconnect_cancels_waiter_and_preserves_durable_stop(monkeypatch):
+ record = lifecycle(monkeypatch)
+ admitted, release = asyncio.Event(), asyncio.Event()
+
+ async def stuck(*args, **kwargs):
+ admitted.set()
+ await release.wait()
+ return {"controller_status": {}, "controlled_executions": 3}
+
+ monkeypatch.setattr(execution_control, "transition_generation", AsyncMock(side_effect=stuck))
+ waiter = asyncio.create_task(handlers.handle_pause_workflow({"workflow_id": "canvas", "expected_revision": 2}, None))
+ await asyncio.wait_for(admitted.wait(), 1)
+ transition_task = handlers._control_transition_tasks[(record.row.id, 3)]
+ waiter.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await waiter
+ assert not transition_task.done()
+ assert record.row.status == "pausing"
+ record.runtime.resume_deployment.assert_not_awaited()
+ release.set()
+ finished, _ = await asyncio.wait_for(asyncio.shield(transition_task), 1)
+ assert finished.status == "paused" and record.row.status == "paused"
+
+
[email protected]("intent", ["pausing", "resuming"])
+async def test_late_transition_after_reset_does_not_broadcast_or_reopen(monkeypatch, intent):
+ record = lifecycle(monkeypatch, intent)
+ started, release = asyncio.Event(), asyncio.Event()
+
+ async def late(*args, on_admission, **kwargs):
+ started.set()
+ await release.wait()
+ # An admission response can arrive after another request wins Reset.
+ await on_admission({"state": "paused" if intent == "pausing" else "running"})
+ return {"controller_status": {}, "controlled_executions": 3}
+
+ monkeypatch.setattr(execution_control, "transition_generation", AsyncMock(side_effect=late))
+ transition_task = asyncio.create_task(handlers._finish_versioned_transition(record.service, copy(record.row)))
+ await asyncio.wait_for(started.wait(), 1)
+ record.row = copy(record.row)
+ record.row.status, record.row.revision = "reset", 5
+ release.set()
+ finished, status = await asyncio.wait_for(transition_task, 1)
+ assert finished.status == "reset" and status is None
+ assert record.broadcasts == []
+ record.runtime.resume_deployment.assert_not_awaited()
+ assert all(call.kwargs["paused"] for call in handlers._set_cron_pause.await_args_list)
+
+
+async def test_transition_failure_after_reset_reports_current_state(monkeypatch):
+ record = lifecycle(monkeypatch, "pausing")
+
+ async def failed(*args, **kwargs):
+ record.row = copy(record.row)
+ record.row.status, record.row.revision = "resetting", 4
+ raise RuntimeError("workflow execution was terminated by Reset")
+
+ monkeypatch.setattr(execution_control, "transition_generation", AsyncMock(side_effect=failed))
+ finished, status = await handlers._finish_versioned_transition(record.service, copy(record.row))
+ assert finished.status == "resetting" and status is None
+ assert record.broadcasts == []
+ record.service.transition.assert_not_awaited()
+
+
+async def test_browser_cleanup_failure_keeps_resetting_and_scope_unarchived(monkeypatch):
+ from services import browser_reset
+ record = lifecycle(monkeypatch)
+ cleanup = AsyncMock(side_effect=RuntimeError("browser_reset_owner_cleanup_unacknowledged"))
+ monkeypatch.setattr(browser_reset, "cleanup_generation_browser_tasks", cleanup)
+ result = await handlers.handle_reset_workflow({"workflow_id": "canvas", "expected_revision": 2}, None)
+ assert result == {"success": False, "error": "browser_reset_owner_cleanup_unacknowledged"}
+ assert record.row.status == "resetting"
+ handlers._terminate_generation_workflows.assert_awaited_once()
+ cleanup.assert_awaited_once()
+ handlers._reset_workspace_tasks.assert_not_awaited()
+ record.service.database.update_workflow_run_data_scope.assert_not_awaited()
+
+
[email protected]("state", ["running", "reset"])
+async def test_stale_reset_generation_cannot_cleanup_current_generation(monkeypatch, state):
+ record = lifecycle(monkeypatch, state)
+ result = await handlers.handle_reset_workflow(
+ {"workflow_id": "canvas", "expected_root_execution_id": "older-generation"}, None)
+ assert result["error"] == "control_generation_conflict"
+ record.service.transition.assert_not_awaited()
+ handlers._reset_workspace_tasks.assert_not_awaited()
+ handlers._terminate_generation_workflows.assert_not_awaited()
+
+
+async def test_generation_scoped_reset_wins_if_stop_completed_in_flight(monkeypatch):
+ record = lifecycle(monkeypatch, "paused")
+ record.row.revision = 4 # Stop completed after the client observed pausing:3.
+ result = await handlers.handle_reset_workflow(
+ {"workflow_id": "canvas", "expected_revision": 3,
+ "expected_root_execution_id": "generation-root"}, None)
+ assert result["success"] and result["state"] == "ready"
+ assert record.service.transition.await_args_list[0].kwargs["expected_revision"] == 4
+
+
+async def test_revision_only_reset_keeps_legacy_conflict_contract(monkeypatch):
+ record = lifecycle(monkeypatch, "paused")
+ record.row.revision = 4
+ result = await handlers.handle_reset_workflow({"workflow_id": "canvas", "expected_revision": 3}, None)
+ assert result == {"success": False, "error": "control_revision_conflict"}
+ handlers._terminate_generation_workflows.assert_not_awaited()
+
+
+async def test_sdk_rpc_timeout_returns_pending_without_undoing_intent(monkeypatch):
+ from temporalio.client import WorkflowUpdateRPCTimeoutOrCancelledError
+ record = lifecycle(monkeypatch, "pausing")
+ monkeypatch.setattr(execution_control, "transition_generation",
+ AsyncMock(side_effect=WorkflowUpdateRPCTimeoutOrCancelledError()))
+ result = await handlers._await_versioned_transition(record.service, copy(record.row))
+ assert result["success"] and result["transition_pending"]
+ assert result["state"] == "pausing" and not result["superseded"]
+ record.service.transition.assert_not_awaited()
+ record.runtime.resume_deployment.assert_not_awaited()
+
+
+async def test_sdk_local_cancellation_is_not_recast_as_control_failure(monkeypatch):
+ from temporalio.client import WorkflowUpdateRPCTimeoutOrCancelledError
+ record = lifecycle(monkeypatch, "pausing")
+ cancelled = WorkflowUpdateRPCTimeoutOrCancelledError()
+ cancelled.__cause__ = asyncio.CancelledError()
+ monkeypatch.setattr(execution_control, "transition_generation", AsyncMock(side_effect=cancelled))
+ with pytest.raises(asyncio.CancelledError):
+ await handlers._finish_versioned_transition(record.service, copy(record.row))
+ assert record.row.status == "pausing"
+ record.service.transition.assert_not_awaited()
+ record.runtime.resume_deployment.assert_not_awaited()
+
+
+def temporal_tree(state="paused"):
+ from temporalio.client import WorkflowExecutionStatus
+
+ def description(first):
+ return SimpleNamespace(status=WorkflowExecutionStatus.RUNNING, raw_description=SimpleNamespace(
+ workflow_execution_info=SimpleNamespace(first_run_id=first), pending_children=[]))
+
+ status = {"participant_revision": 3, "participant_state": state, "state": state,
+ "revision": 3, "membership_epoch": 0, "producers_held": True, "checkpoint": True,
+ "live_roots": {"root": {"workflow_id": "root", "first_execution_run_id": "root-first"}}}
+ controller = SimpleNamespace(describe=AsyncMock(return_value=description("first")),
+ query=AsyncMock(return_value=status), execute_update=AsyncMock(return_value=status))
+ root = SimpleNamespace(describe=AsyncMock(return_value=description("root-first")),
+ execute_update=AsyncMock(return_value=status))
+ client = SimpleNamespace(get_workflow_handle=lambda wid: {"controller": controller, "root": root}[wid])
+ return client, controller, root, status
+
+
+async def test_generation_resume_supersession_prevents_temporal_release():
+ client, controller, root, _ = temporal_tree("running")
+ current = True
+
+ async def ensure_current():
+ if not current:
+ raise execution_control.GenerationTransitionSuperseded()
+
+ async def reset_wins(status):
+ nonlocal current
+ current = False
+
+ with pytest.raises(execution_control.GenerationTransitionSuperseded):
+ await execution_control.transition_generation(client, control("resuming"), paused=False,
+ on_admission=reset_wins, ensure_current=ensure_current)
+ assert len(controller.execute_update.await_args_list) == 1
+ assert len(root.execute_update.await_args_list) == 1
+ assert all(call.args[1]["producers_held"] for call in
+ [*controller.execute_update.await_args_list, *root.execute_update.await_args_list])
+
+
+async def test_checkpoint_failure_retires_sibling_rpc_pollers():
+ client, controller, root, status = temporal_tree()
+ polling, retired = asyncio.Event(), asyncio.Event()
+
+ async def root_update(name, payload, **kwargs):
+ if name == "wait_for_checkpoint":
+ polling.set()
+ try:
+ await asyncio.Event().wait()
+ finally:
+ retired.set()
+ return status
+
+ async def controller_update(name, payload, **kwargs):
+ if name == "wait_for_checkpoint":
+ await polling.wait()
+ raise RuntimeError("checkpoint transport failure")
+ return status
+
+ root.execute_update.side_effect = root_update
+ controller.execute_update.side_effect = controller_update
+ with pytest.raises(RuntimeError, match="checkpoint transport failure"):
+ await asyncio.wait_for(execution_control.transition_generation(client, control(), paused=True), 1)
+ assert retired.is_set()
+
+
[email protected]("intent, handler", [
+ ("pausing", handlers.handle_pause_workflow), ("resuming", handlers.handle_resume_workflow),
+])
+async def test_explicit_retry_joins_existing_transition(monkeypatch, intent, handler):
+ record = lifecycle(monkeypatch, intent)
+ started, release = asyncio.Event(), asyncio.Event()
+
+ async def pending(*args, **kwargs):
+ started.set()
+ await release.wait()
+ return {"controller_status": {}, "controlled_executions": 3}
+
+ temporal = AsyncMock(side_effect=pending)
+ monkeypatch.setattr(execution_control, "transition_generation", temporal)
+ background = handlers._schedule_versioned_transition(record.service, copy(record.row))
+ await asyncio.wait_for(started.wait(), 1)
+ retry = asyncio.create_task(handler({"workflow_id": "canvas", "expected_revision": 3}, None))
+ snapshot = await asyncio.wait_for(handlers.handle_get_workflow_control_status({"workflow_id": "canvas"}, None), 0.2)
+ assert snapshot["state"] == intent
+ await asyncio.sleep(0)
+ assert not retry.done()
+ release.set()
+ result = await asyncio.wait_for(retry, 1)
+ assert result["success"] and not result["transition_pending"]
+ assert result["state"] == ("paused" if intent == "pausing" else "running")
+ await background
+ assert temporal.await_count == 1
+
+
+async def test_unavailable_generation_observation_cannot_hold_reset_lock(monkeypatch):
+ real_payload, real_broadcast = handlers._control_payload, handlers._broadcast_control
+ record = lifecycle(monkeypatch)
+ record.runtime.get_deployment_status = lambda _: {"active_runs": 2}
+ monkeypatch.setattr(handlers, "_control_payload", real_payload)
+ monkeypatch.setattr(handlers, "_broadcast_control", real_broadcast)
+ monkeypatch.setattr(handlers, "_workspace_runtime_status", AsyncMock(return_value={}))
+ monkeypatch.setattr(handlers, "_CONTROL_OBSERVATION_TIMEOUT_SECONDS", 0.02)
+ observing, retired = asyncio.Event(), asyncio.Event()
+
+ async def count(*args):
+ observing.set()
+ try:
+ await asyncio.Event().wait()
+ finally:
+ retired.set()
+
+ async def pending(*args, on_admission, **kwargs):
+ await on_admission({"state": "paused", "revision": 3})
+ await asyncio.Event().wait()
+
+ monkeypatch.setattr(execution_control, "count_generation_actions", count)
+ monkeypatch.setattr(execution_control, "transition_generation", AsyncMock(side_effect=pending))
+ stop = asyncio.create_task(handlers.handle_pause_workflow({"workflow_id": "canvas", "expected_revision": 2}, None))
+ await asyncio.wait_for(observing.wait(), 1)
+ reset = await asyncio.wait_for(handlers.handle_reset_workflow(
+ {"workflow_id": "canvas", "expected_revision": 3, "expected_root_execution_id": "generation-root"}, None), 0.5)
+ assert reset["success"] and reset["state"] == "ready"
+ assert retired.is_set()
+ assert (await asyncio.wait_for(stop, 0.5))["superseded"]
+ # Runtime fallback is explicitly observational. Never report an invented
+ # zero generation count or use it as evidence of a checkpoint.
+ snapshot = await real_payload(record.row, controller_status={"state": "reset"})
+ assert "generation_in_flight_count" not in snapshot
+ assert snapshot["in_flight_count"] == 2
+
+
+async def test_workspace_observation_timeout_omits_unavailable_counts(monkeypatch):
+ from core.container import container
+ from services import node_invocations
+ record = lifecycle(monkeypatch, "pausing")
+ record.runtime.get_deployment_status = lambda _: {"active_runs": 2}
+ monkeypatch.setattr(container, "database", lambda: record.service.database)
+ monkeypatch.setattr(node_invocations, "workspace_task_nodes", AsyncMock(return_value=(None, ["browser"])))
+ monkeypatch.setattr(handlers, "_CONTROL_OBSERVATION_TIMEOUT_SECONDS", 0.02)
+ retired = asyncio.Event()
+
+ async def stalled(*args, **kwargs):
+ try:
+ await asyncio.Event().wait()
+ finally:
+ retired.set()
+
+ monkeypatch.setattr(node_invocations, "controller_status", stalled)
+ snapshot = await asyncio.wait_for(handlers._with_runtime_counts({"state": "pausing"}, "canvas"), 0.5)
+ assert retired.is_set()
+ assert snapshot["workspace_available"] is False
+ assert "workspace_active_count" not in snapshot
+ assert "generation_in_flight_count" not in snapshot
+ assert snapshot["in_flight_count"] == 2
diff --git a/server/tests/temporal/test_generation_execution_control_integration.py b/server/tests/temporal/test_generation_execution_control_integration.py
index d316390a..baa0c3d7 100644
--- a/server/tests/temporal/test_generation_execution_control_integration.py
+++ b/server/tests/temporal/test_generation_execution_control_integration.py
@@ -4,6 +4,7 @@ from __future__ import annotations
import asyncio
import importlib.util
+import os
import subprocess
import sys
from datetime import timedelta
@@ -40,6 +41,15 @@ _contexts: dict[str, dict]
_deliveries: list[dict]
+async def _test_environment():
+ # The time-skipping test server is unavailable on Windows. An explicitly
+ # supplied cached CLI gives this same SDK/replay gate a local dev server.
+ cli = os.environ.get("TEMPORAL_TEST_CLI")
+ if cli:
+ return await WorkflowEnvironment.start_local(dev_server_existing_path=cli)
+ return await WorkflowEnvironment.start_time_skipping()
+
+
@activity.defn(name="execute_node_activity")
async def _execute_node(context: dict) -> dict:
node_id = context["node_id"]
@@ -155,7 +165,7 @@ async def _run_gate():
_calls, _contexts = [], {}
queue = "generation-control-" + uuid4().hex
definitions = [MachinaWorkflow, WorkflowControlWorkflow, ControlledChildProbe]
- async with await WorkflowEnvironment.start_time_skipping() as environment:
+ async with await _test_environment() as environment:
client = environment.client
activities = [_execute_node, *ExecutionControlActivities(client).activities()]
async with Worker(client, task_queue=queue, workflows=definitions, activities=activities, max_cached_workflows=0):
@@ -240,7 +250,7 @@ async def _run_cron_job_gate():
original_pressure = WorkflowControlWorkflow._history_pressure
WorkflowControlWorkflow._history_pressure = _cron_job_history_pressure
try:
- async with await WorkflowEnvironment.start_time_skipping() as environment:
+ async with await _test_environment() as environment:
client = environment.client
activities = [_execute_node, _store_output, _record_completion, _deliver, _fail_job,
*ExecutionControlActivities(client).activities()]