Skip to content

Self-improving AI Employees for everyone

fix(workflow-control): make Start/Pause/Resume/Reset authoritative over Temporal

trohitg77dc7db98ba2
diff --git a/client/src/Dashboard.tsx b/client/src/Dashboard.tsx
index f74d111c..8d987487 100644
--- a/client/src/Dashboard.tsx
+++ b/client/src/Dashboard.tsx
@@ -221,7 +221,7 @@ const DashboardContent: React.FC = () => {
} = useWorkflowManagement();
const { collapsedSections, searchQuery, setSearchQuery, toggleSection } = useComponentPalette();
- const { saveNodeParameters, getAllNodeParameters, executeWorkflow, deployWorkflow, getWorkflowStatus, nodeStatuses, deploymentStatus, workflowControlStatuses, startWorkflow, pauseWorkflow, resumeWorkflow, resetWorkflow, getWorkflowControlStatus, workflowLock, isConnected, isReady, sendRequest, clearNodeStatus } = useWebSocket();
+ const { saveNodeParameters, getAllNodeParameters, executeWorkflow, getWorkflowStatus, nodeStatuses, deploymentStatus, workflowControlStatuses, workflowControlPending, startWorkflow, pauseWorkflow, resumeWorkflow, resetWorkflow, getWorkflowControlStatus, workflowLock, isConnected, isReady, sendRequest, clearNodeStatus } = useWebSocket();
const applyUIDefaults = useAppStore((state) => state.applyUIDefaults);
// Workflows list: server-owned data, cached by TanStack Query.
@@ -232,18 +232,21 @@ const DashboardContent: React.FC = () => {
// Only show as "running" or "locked" if it applies to the currently viewed workflow
const isCurrentWorkflowDeployed = deploymentStatus.isRunning &&
deploymentStatus.workflow_id === currentWorkflow?.id;
- // Backend is the source of truth: per-workflow `isExecuting` is set by
- // the `workflow_status` handler whenever ANY active run exists for the
- // current workflow (ad-hoc node, whole-workflow run, deployed trigger
- // run, or deployment registration). The toolbar Start/Stop button
- // tracks this unified signal so it stays Stop while nodes glow.
+ // Durable lifecycle capabilities are the only source of truth for toolbar
+ // and command-palette actions. Ad-hoc `isExecuting` remains separate and
+ // only drives canvas execution visuals.
const workflowControl = currentWorkflow?.id
? workflowControlStatuses[currentWorkflow.id] || {
workflow_id: currentWorkflow.id, state: 'never_started' as const, revision: 0,
active_count: 0, in_flight_count: 0, queued_count: 0,
- can_start: true, can_pause: false, can_resume: false, can_reset: false,
+ // Status is unresolved until the backend snapshot arrives. Keep every
+ // lifecycle action disabled rather than briefly exposing a stale Start.
+ can_start: false, can_pause: false, can_resume: false, can_reset: false,
}
: { state: 'never_started' as const, revision: 0, active_count: 0, in_flight_count: 0, queued_count: 0, can_start: false, can_pause: false, can_resume: false, can_reset: false };
+ const workflowControlPendingMutation = currentWorkflow?.id
+ ? workflowControlPending[currentWorkflow.id]
+ : undefined;
const isCurrentWorkflowLocked = workflowLock.locked &&
workflowLock.workflow_id === currentWorkflow?.id;
const [globalModelDefaults, setGlobalModelDefaults] = React.useState<{ provider: string; model: string } | null>(null);
@@ -798,18 +801,20 @@ const DashboardContent: React.FC = () => {
const workflow = await openExampleAndChat('AI Assistant');
if (!workflow) return;
- const alreadyDeployed = deploymentStatus.isRunning && deploymentStatus.workflow_id === workflow.id;
- if (alreadyDeployed) return;
-
try {
- const result = await deployWorkflow(workflow.id, workflow.nodes, workflow.edges, 'default');
- if (!result.success) {
- toast.error('Could not start the example — press Start on the toolbar.');
- }
+ const status = await getWorkflowControlStatus(workflow.id);
+ if (!status.can_start) return;
+ await startWorkflow(
+ workflow.id,
+ workflow.nodes,
+ workflow.edges,
+ 'default',
+ status.revision,
+ );
} catch {
toast.error('Could not start the example — press Start on the toolbar.');
}
- }, [openExampleAndChat, deployWorkflow, deploymentStatus.isRunning, deploymentStatus.workflow_id]);
+ }, [getWorkflowControlStatus, openExampleAndChat, startWorkflow]);
const handlePauseWorkflow = async () => {
const workflowId = currentWorkflow?.id;
@@ -1264,8 +1269,8 @@ const DashboardContent: React.FC = () => {
onNew={handleNew}
onOpen={handleOpen}
onRun={withSound('run', handleRun)}
- isRunning={isExecuting}
workflowControl={workflowControl}
+ workflowControlPending={workflowControlPendingMutation}
onStartWorkflow={withSound('run', handleDeploy)}
onPauseWorkflow={handlePauseWorkflow}
onResumeWorkflow={withSound('run', handleResumeWorkflow)}
@@ -1434,6 +1439,7 @@ const DashboardContent: React.FC = () => {
resume: withSound('run', handleResumeWorkflow),
reset: handleResetWorkflow,
workflowControl,
+ workflowControlPending: workflowControlPendingMutation,
exportFile: handleExportFile,
importJSON: handleImportJSON,
openSettings: () => setSettingsOpen(true),
diff --git a/client/src/components/ui/CommandPaletteHost.tsx b/client/src/components/ui/CommandPaletteHost.tsx
index a73649d1..da763025 100644
--- a/client/src/components/ui/CommandPaletteHost.tsx
+++ b/client/src/components/ui/CommandPaletteHost.tsx
@@ -1,3 +1,4 @@
+/* eslint-disable react-refresh/only-export-components -- lifecycle command builder is exported for focused capability tests. */
/**
* CommandPaletteHost — Dashboard-level command list registration.
*
@@ -31,9 +32,12 @@ import {
} from 'lucide-react';
import { CommandPalette, type CommandItem } from './CommandPalette';
import { AVAILABLE_THEMES, useTheme, type ThemeName } from '../../contexts/ThemeContext';
-import type { WorkflowControlStatus } from '../../contexts/WebSocketContext';
+import {
+ type WorkflowControlPendingMutation,
+ type WorkflowControlStatus,
+} from '../../contexts/WebSocketContext';
-interface Handlers {
+export interface CommandPaletteHandlers {
save: () => void;
newWorkflow: () => void;
open: () => void;
@@ -42,6 +46,7 @@ interface Handlers {
resume: () => void;
reset: () => void;
workflowControl: WorkflowControlStatus;
+ workflowControlPending?: WorkflowControlPendingMutation;
exportFile: () => void;
importJSON: () => void;
openSettings: () => void;
@@ -69,9 +74,63 @@ const THEME_LABEL: Record<ThemeName, string> = {
interface Props {
open: boolean;
onOpenChange: (open: boolean) => void;
- handlers: Handlers;
+ handlers: CommandPaletteHandlers;
}
+export const buildWorkflowLifecycleCommands = (
+ handlers: Pick<
+ CommandPaletteHandlers,
+ 'start' | 'pause' | 'resume' | 'reset' | 'workflowControl' | 'workflowControlPending'
+ >,
+): CommandItem[] => {
+ if (handlers.workflowControlPending) {
+ return [];
+ }
+
+ const { state } = handlers.workflowControl;
+ const commands: CommandItem[] = [];
+ if (handlers.workflowControl.can_start) {
+ commands.push({
+ id: 'run.start',
+ label: 'Start Workflow',
+ group: 'Run',
+ icon: Play,
+ onRun: handlers.start,
+ });
+ }
+ if (handlers.workflowControl.can_pause || state === 'pausing') {
+ commands.push({
+ id: 'run.pause',
+ label: state === 'pausing' ? 'Retry Pause Workflow' : 'Pause Workflow',
+ group: 'Run',
+ icon: Pause,
+ onRun: handlers.pause,
+ });
+ }
+ if (handlers.workflowControl.can_resume || state === 'resuming') {
+ commands.push({
+ id: 'run.resume',
+ label: state === 'resuming' ? 'Retry Resume Workflow' : 'Resume Workflow',
+ group: 'Run',
+ icon: Play,
+ onRun: handlers.resume,
+ });
+ }
+ if (handlers.workflowControl.can_reset) {
+ commands.push({
+ id: 'run.reset',
+ label: state === 'resetting'
+ ? 'Retry Reset Workflow Execution'
+ : 'Reset Workflow Execution',
+ group: 'Run',
+ icon: RotateCcw,
+ hint: 'terminates active work',
+ onRun: handlers.reset,
+ });
+ }
+ return commands;
+};
+
export const CommandPaletteHost: React.FC<Props> = ({ open, onOpenChange, handlers }) => {
const { theme, setTheme } = useTheme();
@@ -116,27 +175,7 @@ export const CommandPaletteHost: React.FC<Props> = ({ open, onOpenChange, handle
},
// ── Run ────────────────────────────────────────────────────────
- handlers.workflowControl.state === 'running'
- ? {
- id: 'run.pause',
- label: 'Pause Workflow',
- group: 'Run',
- icon: Pause,
- onRun: handlers.pause,
- }
- : handlers.workflowControl.state === 'paused'
- ? { id: 'run.resume', label: 'Resume Workflow', group: 'Run', icon: Play, onRun: handlers.resume }
- : {
- id: 'run.start',
- label: 'Start Workflow',
- group: 'Run',
- icon: Play,
- onRun: handlers.start,
- },
- ...(handlers.workflowControl.can_reset ? [{
- id: 'run.reset', label: 'Reset Workflow Execution', group: 'Run', icon: RotateCcw,
- hint: 'terminates active work', onRun: handlers.reset,
- } satisfies CommandItem] : []),
+ ...buildWorkflowLifecycleCommands(handlers),
// ── Open panels ────────────────────────────────────────────────
{
diff --git a/client/src/components/ui/TopToolbar.tsx b/client/src/components/ui/TopToolbar.tsx
index 381ea9f0..7eb84551 100644
--- a/client/src/components/ui/TopToolbar.tsx
+++ b/client/src/components/ui/TopToolbar.tsx
@@ -47,7 +47,11 @@ import {
AlertDialog, AlertDialogAction, AlertDialogCancel, AlertDialogContent,
AlertDialogDescription, AlertDialogFooter, AlertDialogHeader, AlertDialogTitle,
} from '@/components/ui/alert-dialog';
-import type { WorkflowControlStatus } from '../../contexts/WebSocketContext';
+import {
+ isWorkflowControlTransitioning,
+ type WorkflowControlPendingMutation,
+ type WorkflowControlStatus,
+} from '../../contexts/WebSocketContext';
import { ThemeSwitcher } from '@/components/ui/ThemeSwitcher';
import { cn } from '@/lib/utils';
import { useAuth } from '../../contexts/AuthContext';
@@ -67,8 +71,8 @@ interface TopToolbarProps {
onNew: () => void;
onOpen: () => void;
onRun: () => void;
- isRunning?: boolean;
workflowControl: WorkflowControlStatus;
+ workflowControlPending?: WorkflowControlPendingMutation;
onStartWorkflow: () => void;
onPauseWorkflow: () => void;
onResumeWorkflow: () => void;
@@ -96,8 +100,8 @@ const TopToolbar: React.FC<TopToolbarProps> = ({
onNew,
onOpen,
onRun: _onRun,
- isRunning = false,
workflowControl,
+ workflowControlPending,
onStartWorkflow,
onPauseWorkflow,
onResumeWorkflow,
@@ -121,6 +125,14 @@ const TopToolbar: React.FC<TopToolbarProps> = ({
const [resetOpen, setResetOpen] = useState(false);
const [tempName, setTempName] = useState(workflowName);
const { user, logout } = useAuth();
+ const hasPendingControlMutation = Boolean(workflowControlPending);
+ const isAuthoritativeTransition = isWorkflowControlTransitioning(workflowControl);
+ const canPauseWorkflow = workflowControl.can_pause || workflowControl.state === 'pausing';
+ const canResumeWorkflow = workflowControl.can_resume || workflowControl.state === 'resuming';
+ const isRetryingReset = workflowControl.state === 'resetting';
+ const controlTransitionLabel = (
+ workflowControlPending?.state ?? workflowControl.state
+ ).replace('_', ' ');
// Global Model Selector state
const { getValidatedAiProviders, saveGlobalModel, isConnected: apiKeysConnected } = useApiKeys();
@@ -404,37 +416,76 @@ const TopToolbar: React.FC<TopToolbarProps> = ({
<Divider />
{/* Durable workflow lifecycle */}
- {workflowControl.can_start ? (
+ {hasPendingControlMutation ? (
+ <ActionButton
+ intent="run"
+ disabled
+ title={`Workflow is ${controlTransitionLabel}`}
+ >
+ <LoaderCircle className="h-3 w-3 animate-spin" />
+ {controlTransitionLabel}
+ </ActionButton>
+ ) : workflowControl.can_start ? (
<ActionButton
intent="run"
onClick={onStartWorkflow}
- disabled={isRunning}
title="Start workflow"
>
<Play className="h-3 w-3 fill-current" />
Start
</ActionButton>
- ) : workflowControl.state === 'paused' ? (
- <ActionButton intent="run" onClick={onResumeWorkflow} title="Resume this workflow execution">
- <Play className="h-3 w-3 fill-current" /> Resume
+ ) : canResumeWorkflow ? (
+ <ActionButton
+ intent="run"
+ onClick={onResumeWorkflow}
+ title={workflowControl.state === 'resuming'
+ ? 'Retry the interrupted resume transition'
+ : 'Resume this workflow execution'}
+ >
+ <Play className="h-3 w-3 fill-current" />
+ {workflowControl.state === 'resuming' ? 'Retry Resume' : 'Resume'}
</ActionButton>
- ) : workflowControl.state === 'running' ? (
+ ) : canPauseWorkflow ? (
<ActionButton
intent="stop"
onClick={onPauseWorkflow}
- title="Pause new workflow scheduling after in-flight work finishes"
+ title={workflowControl.state === 'pausing'
+ ? 'Retry the interrupted pause transition'
+ : 'Pause new workflow scheduling after in-flight work finishes'}
>
- <Pause className="h-3 w-3 fill-current" /> Pause
+ <Pause className="h-3 w-3 fill-current" />
+ {workflowControl.state === 'pausing' ? 'Retry Pause' : 'Pause'}
+ </ActionButton>
+ ) : isAuthoritativeTransition ? (
+ <ActionButton
+ intent="run"
+ disabled
+ title={`Workflow is ${controlTransitionLabel}`}
+ >
+ <LoaderCircle className="h-3 w-3 animate-spin" />
+ {controlTransitionLabel}
</ActionButton>
) : (
<ActionButton intent="run" disabled title={`Workflow is ${workflowControl.state}`}>
- <LoaderCircle className="h-3 w-3 animate-spin" /> {workflowControl.state.replace('_', ' ')}
+ {workflowControl.state.replace('_', ' ')}
</ActionButton>
)}
- {workflowControl.can_reset && <ActionButton intent="stop" onClick={() => setResetOpen(true)} title="Terminate and archive this execution"><RotateCcw className="h-3 w-3" /> Reset</ActionButton>}
+ {workflowControl.can_reset && (
+ <ActionButton
+ intent="stop"
+ onClick={() => setResetOpen(true)}
+ disabled={hasPendingControlMutation}
+ title={isRetryingReset
+ ? 'Retry the interrupted reset transition'
+ : 'Terminate and archive this execution'}
+ >
+ <RotateCcw className="h-3 w-3" />
+ {isRetryingReset ? 'Retry Reset' : 'Reset'}
+ </ActionButton>
+ )}
- <AlertDialog open={resetOpen} onOpenChange={setResetOpen}><AlertDialogContent><AlertDialogHeader><AlertDialogTitle>Reset workflow execution?</AlertDialogTitle><AlertDialogDescription>This immediately terminates active work and archives the current generation. Task history remains available. The workflow will remain stopped until you press Start.</AlertDialogDescription></AlertDialogHeader><AlertDialogFooter><AlertDialogCancel>Keep current execution</AlertDialogCancel><AlertDialogAction onClick={onResetWorkflow}>Reset workflow</AlertDialogAction></AlertDialogFooter></AlertDialogContent></AlertDialog>
+ <AlertDialog open={resetOpen} onOpenChange={setResetOpen}><AlertDialogContent><AlertDialogHeader><AlertDialogTitle>Reset workflow execution?</AlertDialogTitle><AlertDialogDescription>This immediately terminates active work and archives the current generation. Task history remains available. The workflow will remain stopped until you press Start.</AlertDialogDescription></AlertDialogHeader><AlertDialogFooter><AlertDialogCancel>Keep current execution</AlertDialogCancel><AlertDialogAction disabled={hasPendingControlMutation} onClick={onResetWorkflow}>{isRetryingReset ? 'Retry reset' : 'Reset workflow'}</AlertDialogAction></AlertDialogFooter></AlertDialogContent></AlertDialog>
<ActionButton
intent="save"
diff --git a/client/src/components/ui/__tests__/CommandPaletteHost.workflowControl.test.ts b/client/src/components/ui/__tests__/CommandPaletteHost.workflowControl.test.ts
new file mode 100644
index 00000000..74582e20
--- /dev/null
+++ b/client/src/components/ui/__tests__/CommandPaletteHost.workflowControl.test.ts
@@ -0,0 +1,84 @@
+import { describe, expect, it, vi } from 'vitest';
+import type { WorkflowControlStatus } from '../../../contexts/WebSocketContext';
+import {
+ buildWorkflowLifecycleCommands,
+ type CommandPaletteHandlers,
+} from '../CommandPaletteHost';
+
+const control = (
+ overrides: Partial<WorkflowControlStatus> = {},
+): WorkflowControlStatus => ({
+ workflow_id: 'workflow-1',
+ generation: 1,
+ state: 'running',
+ revision: 1,
+ active_count: 0,
+ in_flight_count: 0,
+ queued_count: 0,
+ can_start: false,
+ can_pause: false,
+ can_resume: false,
+ can_reset: false,
+ ...overrides,
+});
+
+const handlers = (
+ workflowControl: WorkflowControlStatus,
+ overrides: Partial<CommandPaletteHandlers> = {},
+) => ({
+ start: vi.fn(),
+ pause: vi.fn(),
+ resume: vi.fn(),
+ reset: vi.fn(),
+ workflowControl,
+ ...overrides,
+});
+
+describe('command palette workflow lifecycle capabilities', () => {
+ it.each([
+ ['run.start', { state: 'failed' as const, can_start: true }],
+ ['run.pause', { state: 'paused' as const, can_pause: true }],
+ ['run.resume', { state: 'running' as const, can_resume: true }],
+ ])('exposes %s from its capability rather than the generic state', (id, overrides) => {
+ const commands = buildWorkflowLifecycleCommands(handlers(control(overrides)));
+ expect(commands.map((command) => command.id)).toContain(id);
+ });
+
+ it('does not infer a start action when no lifecycle capability allows one', () => {
+ const commands = buildWorkflowLifecycleCommands(handlers(control({
+ state: 'never_started',
+ can_start: false,
+ })));
+ expect(commands.map((command) => command.id)).not.toContain('run.start');
+ });
+
+ it('suppresses lifecycle actions while a local mutation is pending', () => {
+ const commands = buildWorkflowLifecycleCommands(handlers(
+ control({ can_pause: true, can_reset: true }),
+ { workflowControlPending: { action: 'pause', state: 'pausing' } },
+ ));
+ expect(commands).toEqual([]);
+ });
+
+ it.each([
+ ['pausing' as const, 'run.pause', 'Retry Pause Workflow'],
+ ['resuming' as const, 'run.resume', 'Retry Resume Workflow'],
+ ['resetting' as const, 'run.reset', 'Retry Reset Workflow Execution'],
+ ])('makes an authoritative %s transition retryable', (state, id, label) => {
+ const commands = buildWorkflowLifecycleCommands(handlers(control({
+ state,
+ can_reset: state === 'resetting',
+ })));
+ expect(commands).toEqual(expect.arrayContaining([
+ expect.objectContaining({ id, label }),
+ ]));
+ });
+
+ it('allows Reset to recover an authoritative Start transition when permitted', () => {
+ const commands = buildWorkflowLifecycleCommands(handlers(control({
+ state: 'starting',
+ can_reset: true,
+ })));
+ expect(commands.map((command) => command.id)).toEqual(['run.reset']);
+ });
+});
diff --git a/client/src/contexts/WebSocketContext.tsx b/client/src/contexts/WebSocketContext.tsx
index c2be66ec..d700abc9 100644
--- a/client/src/contexts/WebSocketContext.tsx
+++ b/client/src/contexts/WebSocketContext.tsx
@@ -63,6 +63,11 @@ interface QueuedSend {
// Request timeout (30 seconds)
const REQUEST_TIMEOUT = 30000;
+// Lifecycle handlers wait for durable deployment setup, Temporal acknowledgements,
+// and generation teardown. Keep their request alive longer than ordinary reads
+// while retaining a finite bound so a lost response cannot lock controls forever.
+export const WORKFLOW_CONTROL_REQUEST_TIMEOUT = 5 * 60 * 1000;
+
// Maximum queued sends before backpressure kicks in (FIFO eviction of oldest)
const QUEUE_MAX_SIZE = 200;
@@ -142,6 +147,55 @@ export interface WorkflowControlStatus {
error?: string | null;
}
+export type WorkflowControlMutationAction = 'start' | 'pause' | 'resume' | 'reset';
+export type WorkflowControlTransitionState = Extract<
+ WorkflowControlState,
+ 'starting' | 'pausing' | 'resuming' | 'resetting'
+>;
+
+export interface WorkflowControlPendingMutation {
+ action: WorkflowControlMutationAction;
+ state: WorkflowControlTransitionState;
+}
+
+const WORKFLOW_CONTROL_STATES = new Set<WorkflowControlState>([
+ 'never_started',
+ 'ready',
+ 'starting',
+ 'running',
+ 'pausing',
+ 'paused',
+ 'resuming',
+ 'resetting',
+ 'failed',
+]);
+
+const WORKFLOW_CONTROL_TRANSITION_STATES = new Set<WorkflowControlState>([
+ 'starting',
+ 'pausing',
+ 'resuming',
+ 'resetting',
+]);
+
+export const isWorkflowControlTransitioning = (
+ status: Pick<WorkflowControlStatus, 'state'>,
+): boolean => WORKFLOW_CONTROL_TRANSITION_STATES.has(status.state);
+
+const WORKFLOW_CONTROL_STABLE_STATE_BY_ACTION: Record<
+ WorkflowControlMutationAction,
+ WorkflowControlState
+> = {
+ start: 'running',
+ pause: 'paused',
+ resume: 'running',
+ reset: 'ready',
+};
+
+export const isWorkflowControlMutationConfirmed = (
+ action: WorkflowControlMutationAction,
+ status: Pick<WorkflowControlStatus, 'state'>,
+): boolean => status.state === WORKFLOW_CONTROL_STABLE_STATE_BY_ACTION[action];
+
export interface TeamTaskTraceEvent {
event_id: string | number;
event_type: string;
@@ -361,6 +415,7 @@ interface WebSocketContextValue {
workflowStatus: WorkflowStatus;
deploymentStatus: DeploymentStatus;
workflowControlStatuses: Record<string, WorkflowControlStatus>;
+ workflowControlPending: Record<string, WorkflowControlPendingMutation>;
workflowLock: WorkflowLock;
compactionStats: Record<string, CompactionStats>; // session_id -> stats (current workflow)
@@ -476,8 +531,9 @@ const defaultDeploymentStatus: DeploymentStatus = {
status: 'idle'
};
-const emptyWorkflowControlStatus = (workflowId?: string): WorkflowControlStatus => ({
+export const emptyWorkflowControlStatus = (workflowId?: string): WorkflowControlStatus => ({
workflow_id: workflowId || null,
+ generation: 0,
state: 'never_started',
revision: 0,
active_count: 0,
@@ -489,7 +545,7 @@ const emptyWorkflowControlStatus = (workflowId?: string): WorkflowControlStatus
can_reset: false,
});
-const normalizeWorkflowControlStatus = (value: any, workflowId?: string): WorkflowControlStatus => {
+export const normalizeWorkflowControlStatus = (value: any, workflowId?: string): WorkflowControlStatus => {
const source = value?.status && typeof value.status === 'object' ? value.status : value || {};
const state = source.state || source.control_state || (source.is_running ? 'running' : 'never_started');
return {
@@ -497,7 +553,8 @@ const normalizeWorkflowControlStatus = (value: any, workflowId?: string): Workfl
...source,
workflow_id: source.workflow_id ?? workflowId ?? null,
state,
- revision: Number(source.revision || 0),
+ generation: Number(source.generation ?? 0),
+ revision: Number(source.revision ?? 0),
active_count: Number(source.active_count ?? source.active_runs ?? 0),
in_flight_count: Number(source.in_flight_count ?? source.active_count ?? source.active_runs ?? 0),
queued_count: Number(source.queued_count || 0),
@@ -508,6 +565,83 @@ const normalizeWorkflowControlStatus = (value: any, workflowId?: string): Workfl
};
};
+/**
+ * Mutation failures may still carry the newest authoritative control-plane
+ * snapshot (for example, an in-progress retry). Only recognize payloads with
+ * an explicit valid lifecycle state so a generic error cannot be projected as
+ * a synthetic `never_started` status.
+ */
+export const extractWorkflowControlStatusSnapshot = (
+ value: unknown,
+ workflowId?: string,
+): WorkflowControlStatus | undefined => {
+ if (!value || typeof value !== 'object') return undefined;
+ const candidate = value as Record<string, any>;
+ const source = candidate.status && typeof candidate.status === 'object'
+ ? candidate.status as Record<string, any>
+ : candidate;
+ const state = source.state ?? source.control_state;
+ if (typeof state !== 'string' || !WORKFLOW_CONTROL_STATES.has(state as WorkflowControlState)) {
+ return undefined;
+ }
+ return normalizeWorkflowControlStatus(value, workflowId);
+};
+
+/**
+ * Resolve two snapshots from the workflow control plane without allowing a
+ * delayed response or broadcast to move the UI to an older generation or
+ * revision. Equal versions are accepted because runtime counts can change
+ * without a control-plane transition.
+ */
+export const mergeWorkflowControlStatus = (
+ current: WorkflowControlStatus | undefined,
+ incoming: WorkflowControlStatus,
+): WorkflowControlStatus => {
+ if (!current) return incoming;
+
+ const currentGeneration = Number(current.generation ?? 0);
+ const incomingGeneration = Number(incoming.generation ?? 0);
+ if (incomingGeneration < currentGeneration) return current;
+ if (incomingGeneration > currentGeneration) return incoming;
+
+ const currentRevision = Number(current.revision ?? 0);
+ const incomingRevision = Number(incoming.revision ?? 0);
+ return incomingRevision < currentRevision ? current : incoming;
+};
+
+export const assertWorkflowControlMutationSucceeded = <T,>(response: T): T => {
+ if ((response as any)?.success === false) {
+ const message = (response as any)?.error || (response as any)?.message || 'Workflow lifecycle mutation failed';
+ throw new Error(String(message));
+ }
+ return response;
+};
+
+export const shouldRetryResetWorkflowAfterConflict = (
+ error: unknown,
+ status: Pick<WorkflowControlStatus, 'can_reset'>,
+): boolean => (
+ status.can_reset
+ && error instanceof Error
+ && error.message === 'control_revision_conflict'
+);
+
+type WorkflowControlMutationRequest =
+ | 'start_workflow'
+ | 'pause_workflow'
+ | 'resume_workflow'
+ | 'reset_workflow';
+
+const WORKFLOW_CONTROL_PENDING_BY_REQUEST: Record<
+ WorkflowControlMutationRequest,
+ WorkflowControlPendingMutation
+> = {
+ start_workflow: { action: 'start', state: 'starting' },
+ pause_workflow: { action: 'pause', state: 'pausing' },
+ resume_workflow: { action: 'resume', state: 'resuming' },
+ reset_workflow: { action: 'reset', state: 'resetting' },
+};
+
const defaultWorkflowLock: WorkflowLock = {
locked: false,
workflow_id: null,
@@ -604,6 +738,8 @@ export const WebSocketProvider: React.FC<{ children: React.ReactNode }> = ({ chi
const [deploymentStatus, setDeploymentStatus] = useState<DeploymentStatus>(defaultDeploymentStatus);
const [workflowControlStatuses, setWorkflowControlStatuses] = useState<Record<string, WorkflowControlStatus>>({});
const workflowControlStatusesRef = useRef<Record<string, WorkflowControlStatus>>({});
+ const [workflowControlPending, setWorkflowControlPending] = useState<Record<string, WorkflowControlPendingMutation>>({});
+ const workflowControlPendingRef = useRef<Map<string, WorkflowControlPendingMutation>>(new Map());
const [workflowLock, setWorkflowLock] = useState<WorkflowLock>(defaultWorkflowLock);
// Per-workflow compaction stats: workflow_id -> session_id -> CompactionStats (n8n pattern)
const [allCompactionStats, setAllCompactionStats] = useState<Record<string, Record<string, CompactionStats>>>({});
@@ -636,9 +772,25 @@ export const WebSocketProvider: React.FC<{ children: React.ReactNode }> = ({ chi
// do not read from elsewhere.
const previousWorkflowIdForSwitchRef = useRef<string | undefined>(currentWorkflowId);
- useEffect(() => {
- workflowControlStatusesRef.current = workflowControlStatuses;
- }, [workflowControlStatuses]);
+ const applyWorkflowControlStatus = useCallback((
+ workflowId: string,
+ value: any,
+ ): WorkflowControlStatus => {
+ const incoming = normalizeWorkflowControlStatus(value, workflowId);
+ const current = workflowControlStatusesRef.current[workflowId];
+ const merged = mergeWorkflowControlStatus(current, incoming);
+ if (current && merged === current) return current;
+
+ const next = {
+ ...workflowControlStatusesRef.current,
+ [workflowId]: merged,
+ };
+ // Update the ref synchronously so multiple responses/broadcasts received
+ // in one React batch still compare against the latest accepted version.
+ workflowControlStatusesRef.current = next;
+ setWorkflowControlStatuses(next);
+ return merged;
+ }, []);
// Detect workflow switches and refresh deployment status from the backend.
// The single source of truth for currentWorkflowId is `useAppStore`; the
@@ -1384,10 +1536,7 @@ export const WebSocketProvider: React.FC<{ children: React.ReactNode }> = ({ chi
const payload = message.data || message;
const workflowId = payload.workflow_id;
if (workflowId) {
- setWorkflowControlStatuses((previous) => ({
- ...previous,
- [workflowId]: normalizeWorkflowControlStatus(payload, workflowId),
- }));
+ applyWorkflowControlStatus(workflowId, payload);
}
break;
}
@@ -1686,7 +1835,7 @@ export const WebSocketProvider: React.FC<{ children: React.ReactNode }> = ({ chi
} catch (error) {
console.error('[WebSocket] Failed to parse message:', error);
}
- }, []); // Empty deps - reads workflow id via useAppStore.getState() escape hatch
+ }, [applyWorkflowControlStatus]);
// Drain queued sends after a successful reconnect. Each queued send gets a
// fresh request_id and a reset timeout budget; responses correlate via the
@@ -2551,20 +2700,102 @@ export const WebSocketProvider: React.FC<{ children: React.ReactNode }> = ({ chi
}
}, [sendRequest]);
+ const requestWorkflowControlStatus = useCallback(async (
+ workflowId: string,
+ ): Promise<WorkflowControlStatus> => {
+ const response = assertWorkflowControlMutationSucceeded(
+ await sendRequest<any>('get_workflow_control_status', { workflow_id: workflowId }),
+ );
+ return applyWorkflowControlStatus(workflowId, response);
+ }, [applyWorkflowControlStatus, sendRequest]);
+
const controlMutation = useCallback(async (
- type: 'start_workflow' | 'pause_workflow' | 'resume_workflow' | 'reset_workflow',
+ type: WorkflowControlMutationRequest,
workflowId: string,
data: Record<string, any>,
): Promise<WorkflowControlStatus> => {
- const response = await sendRequest<any>(type, {
- workflow_id: workflowId,
- idempotency_key: crypto.randomUUID(),
- ...data,
- });
- const normalized = normalizeWorkflowControlStatus(response, workflowId);
- setWorkflowControlStatuses((previous) => ({ ...previous, [workflowId]: normalized }));
- return normalized;
- }, [sendRequest]);
+ if (workflowControlPendingRef.current.has(workflowId)) {
+ throw new Error('A workflow lifecycle change is already in progress');
+ }
+
+ const pending = WORKFLOW_CONTROL_PENDING_BY_REQUEST[type];
+ workflowControlPendingRef.current.set(workflowId, pending);
+ setWorkflowControlPending((previous) => ({ ...previous, [workflowId]: pending }));
+
+ const sendMutationAttempt = async (
+ attemptData: Record<string, any>,
+ ): Promise<WorkflowControlStatus> => {
+ const response = await sendRequest<any>(type, {
+ ...attemptData,
+ workflow_id: workflowId,
+ // Every bounded attempt has its own idempotency identity. In
+ // particular, a conflict retry must not replay the rejected request.
+ idempotency_key: crypto.randomUUID(),
+ }, WORKFLOW_CONTROL_REQUEST_TIMEOUT);
+
+ // A success:false response can still be the newest authoritative
+ // snapshot. Merge it before assertion so the UI is correct even if the
+ // subsequent reconciliation request is unavailable.
+ const snapshot = extractWorkflowControlStatusSnapshot(response, workflowId);
+ const acceptedSnapshot = snapshot
+ ? applyWorkflowControlStatus(workflowId, snapshot)
+ : undefined;
+ assertWorkflowControlMutationSucceeded(response);
+ return acceptedSnapshot ?? applyWorkflowControlStatus(workflowId, response);
+ };
+
+ try {
+ return await sendMutationAttempt(data);
+ } catch (error) {
+ let reconciled: WorkflowControlStatus | undefined;
+ try {
+ reconciled = await requestWorkflowControlStatus(workflowId);
+ if (isWorkflowControlMutationConfirmed(pending.action, reconciled)) {
+ return reconciled;
+ }
+ } catch (resyncError) {
+ console.error('[WebSocket] Failed to resync workflow control status:', resyncError);
+ }
+
+ // Start can legitimately advance `starting -> starting` while attaching
+ // its controller run id, which increments the revision. If Reset raced
+ // that CAS, retry exactly once with the newly read revision. Other
+ // mutations and other errors never retry automatically.
+ if (
+ type === 'reset_workflow'
+ && reconciled
+ && shouldRetryResetWorkflowAfterConflict(error, reconciled)
+ ) {
+ try {
+ return await sendMutationAttempt({
+ ...data,
+ expected_revision: reconciled.revision,
+ });
+ } catch (retryError) {
+ // Preserve timeout/idempotency recovery for the single retry without
+ // issuing a third mutation attempt.
+ try {
+ const retryReconciled = await requestWorkflowControlStatus(workflowId);
+ if (isWorkflowControlMutationConfirmed(pending.action, retryReconciled)) {
+ return retryReconciled;
+ }
+ } catch (resyncError) {
+ console.error('[WebSocket] Failed to resync workflow control status after reset retry:', resyncError);
+ }
+ throw retryError;
+ }
+ }
+ throw error;
+ } finally {
+ workflowControlPendingRef.current.delete(workflowId);
+ setWorkflowControlPending((previous) => {
+ if (!(workflowId in previous)) return previous;
+ const next = { ...previous };
+ delete next[workflowId];
+ return next;
+ });
+ }
+ }, [applyWorkflowControlStatus, requestWorkflowControlStatus, sendRequest]);
const graphEnvelope = (nodes: any[], edges: any[]) => ({
nodes: nodes.map((node) => ({ id: node.id, type: node.type || '', data: node.data || {} })),
@@ -2613,15 +2844,12 @@ export const WebSocketProvider: React.FC<{ children: React.ReactNode }> = ({ chi
const getWorkflowControlStatusAsync = useCallback(async (workflowId: string) => {
try {
- const response = await sendRequest<any>('get_workflow_control_status', { workflow_id: workflowId });
- const normalized = normalizeWorkflowControlStatus(response, workflowId);
- setWorkflowControlStatuses((previous) => ({ ...previous, [workflowId]: normalized }));
- return normalized;
+ return await requestWorkflowControlStatus(workflowId);
} catch (error) {
console.error('[WebSocket] Failed to get workflow control status:', error);
- return emptyWorkflowControlStatus(workflowId);
+ throw error;
}
- }, [sendRequest]);
+ }, [requestWorkflowControlStatus]);
const getTeamTaskTraceAsync = useCallback(async (scope: Record<string, any>): Promise<TeamTaskTraceResponse> => {
const response = await sendRequest<any>('get_team_task_trace', scope);
@@ -3249,6 +3477,7 @@ export const WebSocketProvider: React.FC<{ children: React.ReactNode }> = ({ chi
workflowStatus,
deploymentStatus,
workflowControlStatuses,
+ workflowControlPending,
workflowLock,
// Compaction stats (real-time via broadcasts)
@@ -3358,7 +3587,7 @@ export const WebSocketProvider: React.FC<{ children: React.ReactNode }> = ({ chi
apiKeyStatuses,
consoleLogs, terminalLogs, chatMessages,
nodeStatuses, nodeParameters,
- variables, workflowStatus, deploymentStatus, workflowControlStatuses, workflowLock,
+ variables, workflowStatus, deploymentStatus, workflowControlStatuses, workflowControlPending, workflowLock,
compactionStats, updateCompactionStats,
getNodeStatus, getApiKeyStatus, getVariable,
requestStatus, clearNodeStatus,
diff --git a/client/src/contexts/__tests__/workflowControlStatus.test.ts b/client/src/contexts/__tests__/workflowControlStatus.test.ts
new file mode 100644
index 00000000..a63c222b
--- /dev/null
+++ b/client/src/contexts/__tests__/workflowControlStatus.test.ts
@@ -0,0 +1,180 @@
+import { describe, expect, it } from 'vitest';
+import {
+ WORKFLOW_CONTROL_REQUEST_TIMEOUT,
+ assertWorkflowControlMutationSucceeded,
+ extractWorkflowControlStatusSnapshot,
+ isWorkflowControlMutationConfirmed,
+ mergeWorkflowControlStatus,
+ normalizeWorkflowControlStatus,
+ shouldRetryResetWorkflowAfterConflict,
+ type WorkflowControlMutationAction,
+ type WorkflowControlState,
+ type WorkflowControlStatus,
+} from '../WebSocketContext';
+
+const status = (
+ overrides: Partial<WorkflowControlStatus> = {},
+): WorkflowControlStatus => ({
+ workflow_id: 'workflow-1',
+ generation: 1,
+ state: 'running',
+ revision: 1,
+ active_count: 0,
+ in_flight_count: 0,
+ queued_count: 0,
+ can_start: false,
+ can_pause: true,
+ can_resume: false,
+ can_reset: true,
+ ...overrides,
+});
+
+describe('workflow control status ordering', () => {
+ it('rejects an older generation even when it carries a larger revision', () => {
+ const current = status({ generation: 3, revision: 2, state: 'running' });
+ const stale = status({ generation: 2, revision: 99, state: 'paused' });
+
+ expect(mergeWorkflowControlStatus(current, stale)).toBe(current);
+ });
+
+ it('accepts a newer pre-migration generation whose revision restarted', () => {
+ const current = status({ generation: 2, revision: 8, state: 'ready' });
+ const incoming = status({ generation: 3, revision: 1, state: 'starting' });
+
+ expect(mergeWorkflowControlStatus(current, incoming)).toBe(incoming);
+ });
+
+ it('rejects a lower revision in the same generation', () => {
+ const current = status({ generation: 4, revision: 7, state: 'paused' });
+ const stale = status({ generation: 4, revision: 6, state: 'running' });
+
+ expect(mergeWorkflowControlStatus(current, stale)).toBe(current);
+ });
+
+ it('accepts equal-version runtime count refreshes', () => {
+ const current = status({ generation: 4, revision: 7, active_count: 1 });
+ const incoming = status({ generation: 4, revision: 7, active_count: 2 });
+
+ expect(mergeWorkflowControlStatus(current, incoming)).toBe(incoming);
+ });
+
+ it('normalizes nested snapshots and numeric version strings', () => {
+ const normalized = normalizeWorkflowControlStatus({
+ status: {
+ workflow_id: 'workflow-1',
+ generation: '5',
+ revision: '11',
+ state: 'paused',
+ active_runs: '2',
+ },
+ });
+
+ expect(normalized).toMatchObject({
+ workflow_id: 'workflow-1',
+ generation: 5,
+ revision: 11,
+ state: 'paused',
+ active_count: 2,
+ can_resume: true,
+ });
+ });
+});
+
+describe('workflow control mutation responses', () => {
+ it('uses a lifecycle timeout long enough for durable setup and teardown', () => {
+ expect(WORKFLOW_CONTROL_REQUEST_TIMEOUT).toBe(5 * 60 * 1000);
+ });
+
+ it('rejects explicit success:false responses with the backend error', () => {
+ expect(() => assertWorkflowControlMutationSucceeded({
+ success: false,
+ error: 'control_revision_conflict',
+ state: 'never_started',
+ })).toThrow('control_revision_conflict');
+ });
+
+ it('returns successful status payloads unchanged', () => {
+ const response = { success: true, generation: 2, revision: 3 };
+ expect(assertWorkflowControlMutationSucceeded(response)).toBe(response);
+ });
+
+ it('extracts the authoritative snapshot carried by a success:false response', () => {
+ const current = status({ state: 'running', revision: 3 });
+ const response = {
+ success: false,
+ error: 'workflow_control_transition_pending',
+ workflow_id: 'workflow-1',
+ generation: 1,
+ state: 'pausing',
+ revision: 4,
+ can_start: false,
+ can_pause: false,
+ can_resume: false,
+ can_reset: true,
+ };
+
+ const snapshot = extractWorkflowControlStatusSnapshot(response);
+
+ expect(snapshot).toMatchObject({
+ workflow_id: 'workflow-1',
+ state: 'pausing',
+ revision: 4,
+ can_reset: true,
+ });
+ expect(mergeWorkflowControlStatus(current, snapshot!)).toBe(snapshot);
+ expect(() => assertWorkflowControlMutationSucceeded(response))
+ .toThrow('workflow_control_transition_pending');
+ });
+
+ it('does not mistake a generic mutation error for a control snapshot', () => {
+ expect(extractWorkflowControlStatusSnapshot({
+ success: false,
+ error: 'control_revision_conflict',
+ workflow_id: 'workflow-1',
+ })).toBeUndefined();
+ });
+
+ it('only retries Reset conflicts when the resynced snapshot still allows Reset', () => {
+ const conflict = new Error('control_revision_conflict');
+ expect(shouldRetryResetWorkflowAfterConflict(
+ conflict,
+ status({ state: 'starting', revision: 2, can_reset: true }),
+ )).toBe(true);
+ expect(shouldRetryResetWorkflowAfterConflict(
+ conflict,
+ status({ state: 'ready', revision: 3, can_reset: false }),
+ )).toBe(false);
+ expect(shouldRetryResetWorkflowAfterConflict(
+ new Error('workflow_control_transition_pending'),
+ status({ state: 'starting', revision: 2, can_reset: true }),
+ )).toBe(false);
+ });
+
+ it.each([
+ ['start', 'running'],
+ ['pause', 'paused'],
+ ['resume', 'running'],
+ ['reset', 'ready'],
+ ] satisfies Array<[WorkflowControlMutationAction, WorkflowControlState]>)(
+ 'accepts an authoritative %s resync only after it reaches %s',
+ (action, stableState) => {
+ expect(isWorkflowControlMutationConfirmed(action, status({
+ state: stableState,
+ }))).toBe(true);
+ },
+ );
+
+ it.each([
+ ['start', 'starting'],
+ ['pause', 'pausing'],
+ ['resume', 'resuming'],
+ ['reset', 'resetting'],
+ ] satisfies Array<[WorkflowControlMutationAction, WorkflowControlState]>)(
+ 'does not treat the transitional %s state as confirmed',
+ (action, transitionalState) => {
+ expect(isWorkflowControlMutationConfirmed(action, status({
+ state: transitionalState,
+ }))).toBe(false);
+ },
+ );
+});
diff --git a/docs-internal/frontend_architecture.md b/docs-internal/frontend_architecture.md
index 796462ae..6b62b4af 100644
--- a/docs-internal/frontend_architecture.md
+++ b/docs-internal/frontend_architecture.md
@@ -393,6 +393,25 @@ See [ui_migration_plan.md](./ui_migration_plan.md) Phase 6.
persisted generation record is reconciled with the Temporal controller rather
than inferred from process-local tasks. `workflow_control_status` broadcasts
update transitions in real time; reconnect performs an authoritative read.
+ Reads, broadcasts, and mutation responses merge monotonically by generation
+ and database revision, while per-workflow pending mutation state disables
+ duplicate clicks. A `{success: false}` response carrying a recognizable
+ control snapshot merges that snapshot before surfacing the error; every
+ failure is followed by an authoritative resync, and a generic error is never
+ normalized into `never_started`.
+ Lifecycle mutations use a five-minute acknowledgement timeout because Start
+ and Reset may wait on durable setup/cleanup. If a timeout races a successful
+ operation, the resync-confirmed stable state is returned as success.
+ Authoritative `pausing`, `resuming`, and `resetting` states remain retryable
+ after the local request finishes. The UI treats stable `paused`/`running` as
+ confirmation that strict cron-Schedule and running-execution fan-out
+ completed; a fan-out failure must remain transitional after resync.
+ Likewise, `ready` means Reset first quiesced controller, cron, and local
+ producers and then completed its final execution sweep. A duplicate Reset in
+ `ready` is a read-only idempotent response, so an old request cannot clean up
+ resources belonging to a later Start. Standalone cron and detached child
+ executions carry `EventWorkflowId`, allowing the server to uphold that
+ boundary through Temporal Visibility.
The older `deployment_snapshot` and binary deployment status remain migration
adapters for legacy deployments and must not override a resolved controller
generation. See [Temporal Execution Engine RFC](temporal-execution-engine-rfc.md).
diff --git a/docs-internal/temporal-execution-engine-rfc.md b/docs-internal/temporal-execution-engine-rfc.md
index 15cec5cd..f3c1e471 100644
--- a/docs-internal/temporal-execution-engine-rfc.md
+++ b/docs-internal/temporal-execution-engine-rfc.md
@@ -51,8 +51,8 @@ path.
### 2.2 Non-goals
-- Native Temporal Workflow Pause. The engine uses cooperative Signals and
- deterministic wait conditions.
+- Native Temporal Workflow Pause. The engine uses an acknowledged controller
+ Update, cooperative descendant Signals, and deterministic wait conditions.
- Temporal's history-rewind Reset. Product Reset terminates and archives the
current application generation.
- Shared parent/child LLM transcripts. Delegated agents start with isolated
@@ -243,20 +243,29 @@ sequenceDiagram
User->>UI: Pause
UI->>API: pause_workflow(expected_revision)
API->>DB: running -> pausing (CAS)
- API->>C: pause signal
- API->>T: pause cron schedules
- API->>DM: gate legacy admission + disarm trigger UI
+ API-->>UI: workflow_control_status(pausing)
+ API->>DM: close local admission
+ API->>C: set_control_state(paused) Update
+ C-->>API: acknowledged paused state
+ API->>T: strictly pause cron schedules
+ API->>T: strictly signal running descendants
+ T-->>API: schedule + descendant fan-out complete
+ API->>DM: disarm trigger UI
API->>DB: pausing -> paused
- API-->>UI: paused
+ API-->>UI: workflow_control_status(paused)
User->>UI: Resume
UI->>API: resume_workflow(expected_revision)
API->>DB: paused -> resuming (CAS)
- API->>C: resume signal
- API->>T: unpause cron schedules
+ API-->>UI: workflow_control_status(resuming)
+ API->>C: set_control_state(running) Update
+ C-->>API: acknowledged running state
+ API->>T: strictly unpause cron schedules
+ API->>T: strictly signal running descendants
+ T-->>API: schedule + descendant fan-out complete
API->>DM: drain buffered legacy events FIFO + rearm UI
API->>DB: resuming -> running
- API-->>UI: running
+ API-->>UI: workflow_control_status(running)
```
### 6.4 Cooperative pause semantics
@@ -268,19 +277,35 @@ Pause is an admission barrier, not suspension of operating-system threads:
- Poll cycles do not start new graph runs; a poll activity already in flight may
finish and its events wait for Resume.
- Temporal cron schedules are explicitly paused.
-- `MachinaWorkflow`, `AgentWorkflow`, and `DelegatedTaskWorkflow` gate new
- scheduling at deterministic wait conditions.
+- `WorkflowControlWorkflow`, `TriggerListenerWorkflow`,
+ `PollingTriggerWorkflow`, `CronTriggerWorkflow`, `MachinaWorkflow`,
+ `AgentWorkflow`, and `DelegatedTaskWorkflow` gate new scheduling at
+ deterministic, replay-patched wait conditions. Admission is rechecked after
+ awaited trigger broadcasts, polling cycles, LLM, preflight, permit, and
+ delegation operations, immediately before the next activity or child
+ workflow is scheduled.
- Activities already started are allowed to finish. Their results remain in
history and are consumed after the gate opens.
- Trigger nodes broadcast an idle/paused projection, stop their listening
animation, and rearm on Resume.
+- Stable `paused`/`running` is published only after strict cron-Schedule and
+ running-execution fan-out completes. Visibility or per-target failures leave
+ the generation transitional and retryable.
### 6.5 Reset semantics
-Reset transitions the generation to `resetting`, rejects new admission,
-signals the controller to close, terminates visible descendants carrying the
-application workflow search attribute, cancels local compatibility resources,
-and archives the record as `reset`. It does **not** start the next generation.
+Reset transitions the generation to `resetting` and quiesces producers before
+its final execution sweep: it synchronously closes local admission, signals the
+controller to close, deletes generation cron schedules, and cancels local
+compatibility resources. It then strictly terminates controller descendants and
+standalone roots carrying the application workflow search attribute before
+archiving the data scope. `EventWorkflowId` is propagated to Schedule action
+executions and detached/abandoned graph children so those standalone runs remain
+discoverable. Runtime/node cleanup completes before the database publishes
+`reset`; a failed step leaves `resetting` retryable. A duplicate command against
+an already-reset generation returns idempotently without repeating cleanup,
+which prevents an old retry from affecting a later generation. Reset does
+**not** start the next generation.
Task history and historical Temporal histories are preserved. Existing
historical rows continue to appear in Temporal Web until namespace retention or
diff --git a/docs-internal/temporal-workflow-control.md b/docs-internal/temporal-workflow-control.md
index e1ca9def..618f1a2f 100644
--- a/docs-internal/temporal-workflow-control.md
+++ b/docs-internal/temporal-workflow-control.md
@@ -16,15 +16,54 @@ is authoritative for execution history.
- **Resume** signals the same running Temporal executions and drains buffered
trigger events in FIFO order, unpauses cron schedules, and rearms trigger
nodes.
-- **Reset** revision-guards the old generation, terminates visible executions
- carrying the workflow search attribute, cancels local deployment resources,
- and archives the old generation. It leaves the control state `ready`; the
- user must press **Start** to create the next generation.
-
-All mutations require an expected revision and support an idempotency key.
+- **Reset** revision-guards the old generation, closes controller and local
+ admission, removes Temporal cron schedules, cancels local compatibility
+ resources, and then performs a final strict execution sweep before archiving
+ the old generation. It leaves the control state `ready`; the user must press
+ **Start** to create the next generation.
+
+Clients send an expected revision and idempotency key with every mutation.
+New state transitions compare-and-swap the expected revision; Start persists
+its request key, controller Updates receive unique request identities, and
+stable/retry states are idempotent from their durable state.
+Revisions increase across generation boundaries instead of restarting at zero,
+so a delayed request from an archived generation cannot satisfy a later
+generation's compare-and-swap check.
The toolbar and command palette derive available actions from the server's
`can_start`, `can_pause`, `can_resume`, and `can_reset` fields.
+Start does not publish `running` merely because deployment setup was placed on
+an asyncio task. The control handler waits for trigger/listener setup to finish;
+setup failure moves the generation to `failed` and closes its controller.
+Once the `running` compare-and-swap commits, a status-broadcast/projection
+failure does not tear down that live generation; the failed request is recovered
+by the client's authoritative status resync.
+
+Pause and Resume use the controller's acknowledged `set_control_state` Temporal
+Update. The database first publishes `pausing`/`resuming`, then publishes the
+stable state only after the Update result confirms `paused`/`running`. If the
+Update is known to have been rejected before admission, the database and local
+admission gate return to the prior stable state. An unknown outcome, such as a
+transport timeout after the server may have accepted the Update, remains
+transitional. Status reads and explicit UI retries reconcile that state by
+idempotently retrying the desired Update, reapplying schedule/local admission
+gates, and completing the database CAS.
+Legacy pause/resume Signals remain registered for history compatibility and
+fan out to already-running descendant workflows. That fan-out and every cron
+Schedule mutation are strict lifecycle barriers: a visibility or per-target
+failure leaves the database in `pausing`/`resuming` for reconciliation instead
+of falsely publishing a stable state.
+
+Every accepted control transition emits `workflow_control_status`. Clients
+merge reads, mutation responses, and broadcasts monotonically by generation
+then database revision; an older response cannot move another tab backward.
+Only one lifecycle mutation per workflow may be pending in a browser tab.
+Lifecycle requests have a five-minute acknowledgement window, and a timed-out
+request is treated as successful when the immediate authoritative resync
+already reports its requested stable state. Transitional Pause, Resume, and
+Reset states expose an explicit retry rather than trapping the toolbar in a
+permanent spinner.
+
## Generation-scoped workflow data
The editable `workflows` row is the stable canvas definition; it is not runtime
@@ -39,13 +78,32 @@ state. Every successful **Start** atomically creates a
- remains immutable as an execution snapshot while runtime records accumulate
under its scope ID.
-**Reset** terminates the generation and marks its data scope `archived`; it does
-not delete outputs, tasks, traces, or the graph snapshot. The toolbar returns to
+**Reset** reaches `ready` only after durable executions and schedules are gone,
+the generation data scope is archived, and node/runtime state has been reset.
+If any cleanup step fails, the control remains `resetting`; retrying Reset
+resumes cleanup instead of falsely completing or skipping it. Reset does not
+delete outputs, tasks, traces, or the graph snapshot. The toolbar returns to
**Start**. The next Start creates a new generation, execution ID, Temporal
controller, and empty runtime namespace from the then-current saved canvas.
Historical scopes therefore remain queryable without leaking state into the
new run.
+Reset quiesces every producer before its final strict Visibility sweep: local
+admission closes synchronously, the controller is told to close, cron Schedules
+are deleted, and legacy local resources are cancelled. `EventWorkflowId` is
+propagated to cron action executions and detached/abandoned graph children, so
+the first Visibility pass identifies both the controller tree and standalone
+execution roots. Because Temporal does not inherit custom Search Attributes
+onto child workflows, a second batched `RootWorkflowId` pass expands every
+tagged root to its active Agent and delegated descendants before signal or
+termination fan-out.
+Idempotent cron redeploys refresh the Schedule's frozen action metadata while
+preserving its current paused state; an ownership check prevents a same-label
+Schedule from being overwritten by a different application workflow.
+Once `reset` is persisted, duplicate Reset requests return idempotently without
+repeating generation-wide cleanup; this prevents an old retry from racing and
+terminating a newly started generation.
+
The editor's live projection follows the same boundary. Reset broadcasts
`workflow_runtime_reset`, clears node statuses, variables, and console/chat
projections, and remounts the parameter panel so local output
diff --git a/server/core/database.py b/server/core/database.py
index 30c5bcf5..63464bdd 100644
--- a/server/core/database.py
+++ b/server/core/database.py
@@ -3,7 +3,7 @@
import json
import inspect
from copy import deepcopy
-from datetime import datetime, timezone
+from datetime import datetime, timedelta, timezone
from typing import Awaitable, Callable, Dict, Any, List, Optional, Tuple, Union
from pydantic_core import to_jsonable_python
from sqlmodel import SQLModel, select
@@ -3078,7 +3078,7 @@ class Database:
update(TeamTask)
.where(
TeamTask.id == task_id,
- TeamTask.status == "queued",
+ TeamTask.status.in_(["queued", "pending"]),
or_(TeamTask.assigned_to.is_(None), TeamTask.assigned_to == agent_node_id),
)
.values(
@@ -3230,6 +3230,128 @@ class Database:
logger.error(f"Failed to fail task: {e}")
return False
+ async def cancel_team_task(
+ self,
+ team_id: str,
+ task_id: str,
+ reason: str,
+ ) -> Optional[Dict[str, Any]]:
+ """Terminally cancel active delegated work without applying retry policy.
+
+ The state guard makes retries idempotent and prevents cancellation
+ from overwriting a result that won the race to a terminal state.
+ Active attempts receive the same terminal reason as the task row;
+ queued work gets an attempt record as well so every cancellation has
+ a durable terminal audit entry.
+ """
+ active_statuses = ["blocked", "queued", "pending", "running"]
+ terminal_statuses = {
+ "submitted",
+ "accepted",
+ "failed",
+ "cancelled",
+ "skipped",
+ "completed",
+ }
+ cancellation_reason = reason or "Delegated agent cancelled"
+ now = datetime.now(timezone.utc)
+
+ async def _cancel(session: AsyncSession) -> Dict[str, Any]:
+ changed = await session.execute(
+ update(TeamTask)
+ .where(
+ TeamTask.id == task_id,
+ TeamTask.team_id == team_id,
+ TeamTask.status.in_(active_statuses),
+ )
+ .values(
+ status="cancelled",
+ cancellation_requested=True,
+ cancellation_reason=cancellation_reason,
+ completed_at=now,
+ revision=TeamTask.revision + 1,
+ )
+ )
+ task = (
+ await session.execute(
+ select(TeamTask).where(
+ TeamTask.id == task_id,
+ TeamTask.team_id == team_id,
+ )
+ )
+ ).scalar_one_or_none()
+ if task is None:
+ return {"found": False}
+
+ cancellation_applied = changed.rowcount == 1
+ if cancellation_applied:
+ attempt_id = f"{task_id}:attempt:{task.current_attempt}"
+ attempt = (
+ await session.execute(
+ select(TeamTaskAttempt).where(
+ TeamTaskAttempt.id == attempt_id
+ )
+ )
+ ).scalar_one_or_none()
+ if attempt is None:
+ session.add(
+ TeamTaskAttempt(
+ id=attempt_id,
+ task_id=task_id,
+ team_id=team_id,
+ attempt_number=task.current_attempt,
+ assignee_node_id=task.assigned_to,
+ status="cancelled",
+ child_workflow_id=task.child_workflow_id,
+ child_run_id=task.child_run_id,
+ runner_workflow_id=task.runner_workflow_id,
+ runner_run_id=task.runner_run_id,
+ parent_workflow_id=task.parent_workflow_id,
+ parent_run_id=task.parent_run_id,
+ error=cancellation_reason,
+ created_at=task.created_at or now,
+ started_at=task.started_at,
+ completed_at=now,
+ )
+ )
+ else:
+ attempt.status = "cancelled"
+ attempt.error = cancellation_reason
+ attempt.completed_at = now
+
+ if not cancellation_applied and task.status not in terminal_statuses:
+ return {
+ "found": True,
+ "status": task.status,
+ "cancellation_applied": False,
+ "valid_state": False,
+ }
+
+ snapshot = self._team_task_dict(task)
+ snapshot["found"] = True
+ snapshot["cancellation_applied"] = cancellation_applied
+ snapshot["valid_state"] = True
+ return snapshot
+
+ try:
+ result, _ = await self.run_runtime_mutation(
+ resource_type="team_task",
+ resource_id=task_id,
+ operation="cancel",
+ mutate=_cancel,
+ )
+ if not result.get("found") or not result.get("valid_state", True):
+ return None
+ return result
+ except Exception as e:
+ logger.error(
+ "Failed to cancel team task",
+ team_id=team_id,
+ task_id=task_id,
+ error=str(e),
+ )
+ return None
+
async def get_claimable_tasks(self, team_id: str) -> List[Dict[str, Any]]:
"""Get pending tasks with resolved dependencies."""
try:
@@ -3407,76 +3529,160 @@ class Database:
return {"error": str(e)}
async def acquire_subagent_permit(self, root_execution_id: str, permit_id: str, limit: int = 3) -> bool:
- """Atomically acquire a root-global permit; retries by the owner succeed."""
+ """Atomically acquire a root-global permit; retries by the owner succeed.
+
+ Active leases older than the maximum delegated-workflow duration are
+ opportunistically reconciled in the same write transaction. This is a
+ last-resort capacity repair for workers that died before compensation.
+ """
if limit < 1:
return False
+ now = datetime.now(timezone.utc)
+ stale_before = now - timedelta(hours=3)
+
try:
- async with self.get_session() as session:
+ async def _acquire(session: AsyncSession) -> Dict[str, Any]:
+ stale = await session.execute(
+ update(SubagentConcurrencyPermit)
+ .where(
+ SubagentConcurrencyPermit.root_execution_id
+ == root_execution_id,
+ SubagentConcurrencyPermit.status == "active",
+ SubagentConcurrencyPermit.acquired_at < stale_before,
+ )
+ .values(status="released", released_at=now)
+ )
+ stale_count = max(int(stale.rowcount or 0), 0)
+ if stale_count:
+ await session.execute(
+ update(SubagentConcurrencyCounter)
+ .where(
+ SubagentConcurrencyCounter.root_execution_id
+ == root_execution_id
+ )
+ .values(
+ active_count=case(
+ (
+ SubagentConcurrencyCounter.active_count
+ >= stale_count,
+ SubagentConcurrencyCounter.active_count
+ - stale_count,
+ ),
+ else_=0,
+ ),
+ updated_at=now,
+ )
+ )
+
existing = await session.execute(
- select(SubagentConcurrencyPermit).where(SubagentConcurrencyPermit.permit_id == permit_id)
+ select(SubagentConcurrencyPermit).where(
+ SubagentConcurrencyPermit.permit_id == permit_id
+ )
)
permit = existing.scalar_one_or_none()
if permit:
- return permit.root_execution_id == root_execution_id and permit.status == "active"
+ return {
+ "acquired": (
+ permit.root_execution_id == root_execution_id
+ and permit.status == "active"
+ )
+ }
- await session.execute(text(
- "INSERT OR IGNORE INTO subagent_concurrency_counters "
- "(root_execution_id, active_count, updated_at) VALUES (:root, 0, CURRENT_TIMESTAMP)"
- ), {"root": root_execution_id})
+ await session.execute(
+ text(
+ "INSERT OR IGNORE INTO subagent_concurrency_counters "
+ "(root_execution_id, active_count, updated_at) "
+ "VALUES (:root, 0, CURRENT_TIMESTAMP)"
+ ),
+ {"root": root_execution_id},
+ )
claimed = await session.execute(
update(SubagentConcurrencyCounter)
.where(
- SubagentConcurrencyCounter.root_execution_id == root_execution_id,
+ SubagentConcurrencyCounter.root_execution_id
+ == root_execution_id,
SubagentConcurrencyCounter.active_count < limit,
)
.values(
active_count=SubagentConcurrencyCounter.active_count + 1,
- updated_at=datetime.now(timezone.utc),
+ updated_at=now,
)
)
if claimed.rowcount != 1:
- await session.rollback()
- return False
- session.add(SubagentConcurrencyPermit(
- permit_id=permit_id, root_execution_id=root_execution_id, status="active"
- ))
- await session.commit()
- return True
+ return {"acquired": False}
+ session.add(
+ SubagentConcurrencyPermit(
+ permit_id=permit_id,
+ root_execution_id=root_execution_id,
+ status="active",
+ acquired_at=now,
+ )
+ )
+ return {"acquired": True}
+
+ result, _ = await self.run_runtime_mutation(
+ resource_type="subagent_concurrency",
+ resource_id=root_execution_id,
+ operation="acquire",
+ mutate=_acquire,
+ )
+ return bool(result.get("acquired"))
except IntegrityError:
# A concurrent retry inserted the same permit. Resolve ownership
# from durable state rather than incrementing the counter again.
async with self.get_session() as session:
result = await session.execute(
- select(SubagentConcurrencyPermit).where(SubagentConcurrencyPermit.permit_id == permit_id)
+ select(SubagentConcurrencyPermit).where(
+ SubagentConcurrencyPermit.permit_id == permit_id
+ )
)
permit = result.scalar_one_or_none()
- return bool(permit and permit.root_execution_id == root_execution_id and permit.status == "active")
+ return bool(
+ permit
+ and permit.root_execution_id == root_execution_id
+ and permit.status == "active"
+ )
except Exception as e:
- logger.error("Failed to acquire subagent permit", root_execution_id=root_execution_id, permit_id=permit_id, error=str(e))
+ logger.error(
+ "Failed to acquire subagent permit",
+ root_execution_id=root_execution_id,
+ permit_id=permit_id,
+ error=str(e),
+ )
return False
async def release_subagent_permit(self, root_execution_id: str, permit_id: str) -> bool:
- """Idempotently release a permit and decrement its root counter once."""
+ """Idempotently release a permit and decrement its root counter once.
+
+ A lease that was never created is an expected no-op (not an ownership
+ error). A lease owned by another root remains a hard failure.
+ """
try:
async with self.get_session() as session:
released = await session.execute(
update(SubagentConcurrencyPermit)
.where(
SubagentConcurrencyPermit.permit_id == permit_id,
- SubagentConcurrencyPermit.root_execution_id == root_execution_id,
+ SubagentConcurrencyPermit.root_execution_id
+ == root_execution_id,
SubagentConcurrencyPermit.status == "active",
)
- .values(status="released", released_at=datetime.now(timezone.utc))
+ .values(
+ status="released",
+ released_at=datetime.now(timezone.utc),
+ )
)
if released.rowcount == 1:
await session.execute(
update(SubagentConcurrencyCounter)
.where(
- SubagentConcurrencyCounter.root_execution_id == root_execution_id,
+ SubagentConcurrencyCounter.root_execution_id
+ == root_execution_id,
SubagentConcurrencyCounter.active_count > 0,
)
.values(
- active_count=SubagentConcurrencyCounter.active_count - 1,
+ active_count=SubagentConcurrencyCounter.active_count
+ - 1,
updated_at=datetime.now(timezone.utc),
)
)
@@ -3484,16 +3690,26 @@ class Database:
return True
existing = await session.execute(
select(SubagentConcurrencyPermit).where(
- SubagentConcurrencyPermit.permit_id == permit_id,
- SubagentConcurrencyPermit.root_execution_id == root_execution_id,
+ SubagentConcurrencyPermit.permit_id == permit_id
)
)
permit = existing.scalar_one_or_none()
- already_released = bool(permit and permit.status == "released")
+ if permit is None:
+ await session.rollback()
+ return True
+ already_released = bool(
+ permit.root_execution_id == root_execution_id
+ and permit.status == "released"
+ )
await session.rollback()
return already_released
except Exception as e:
- logger.error("Failed to release subagent permit", root_execution_id=root_execution_id, permit_id=permit_id, error=str(e))
+ logger.error(
+ "Failed to release subagent permit",
+ root_execution_id=root_execution_id,
+ permit_id=permit_id,
+ error=str(e),
+ )
return False
# ============================================================================
diff --git a/server/nodes/scheduler/cron_scheduler/_workflow.py b/server/nodes/scheduler/cron_scheduler/_workflow.py
index b018b29e..605e250e 100644
--- a/server/nodes/scheduler/cron_scheduler/_workflow.py
+++ b/server/nodes/scheduler/cron_scheduler/_workflow.py
@@ -37,6 +37,12 @@ from temporalio.common import WorkflowIDReusePolicy
from temporalio.workflow import ParentClosePolicy
+CHILD_SEARCH_ATTRIBUTES_PATCH = "cron-child-search-attributes-v1"
+COOPERATIVE_PAUSE_SCHEDULING_PATCH = (
+ "cron-cooperative-pause-scheduling-v1"
+)
+
+
@workflow.defn(name="CronTriggerWorkflow", sandboxed=False)
class CronTriggerWorkflow:
"""One-shot per-firing workflow that spawns a child MachinaWorkflow.
@@ -46,6 +52,21 @@ class CronTriggerWorkflow:
everything else is computed from the static action args.
"""
+ def __init__(self) -> None:
+ self._control_paused = False
+
+ @workflow.signal
+ async def pause(self) -> None:
+ self._control_paused = True
+
+ @workflow.signal
+ async def resume(self) -> None:
+ self._control_paused = False
+
+ async def _wait_until_resumed(self) -> None:
+ if self._control_paused:
+ await workflow.wait_condition(lambda: not self._control_paused)
+
@workflow.run
async def run(self, listener_data: Dict[str, Any]) -> Dict[str, Any]:
"""Spawn one MachinaWorkflow child per cron firing.
@@ -108,6 +129,36 @@ class CronTriggerWorkflow:
firing_iso = trigger_output["timestamp"]
child_id = f"{workflow_slug}-{trigger_label}-{firing_iso}"
+ try:
+ use_child_search_attributes = workflow.patched(
+ CHILD_SEARCH_ATTRIBUTES_PATCH
+ )
+ use_cooperative_pause_schedule_gate = workflow.patched(
+ COOPERATIVE_PAUSE_SCHEDULING_PATCH
+ )
+ except RuntimeError: # direct unit invocation outside Temporal runtime
+ use_child_search_attributes = False
+ use_cooperative_pause_schedule_gate = False
+ from services.temporal.trigger_listener_workflow import (
+ event_workflow_search_attributes,
+ )
+
+ child_options = {
+ "id": child_id,
+ "parent_close_policy": ParentClosePolicy.ABANDON,
+ "id_reuse_policy": (
+ WorkflowIDReusePolicy.ALLOW_DUPLICATE_FAILED_ONLY
+ ),
+ "execution_timeout": timedelta(hours=1),
+ "run_timeout": timedelta(hours=1),
+ }
+ if use_child_search_attributes:
+ child_options["search_attributes"] = (
+ event_workflow_search_attributes(deployment_workflow_id)
+ )
+
+ if use_cooperative_pause_schedule_gate:
+ await self._wait_until_resumed()
await workflow.start_child_workflow(
"MachinaWorkflow",
args=[
@@ -120,11 +171,7 @@ class CronTriggerWorkflow:
"tenant_id": tenant_id,
}
],
- id=child_id,
- parent_close_policy=ParentClosePolicy.ABANDON,
- id_reuse_policy=WorkflowIDReusePolicy.ALLOW_DUPLICATE_FAILED_ONLY,
- execution_timeout=timedelta(hours=1),
- run_timeout=timedelta(hours=1),
+ **child_options,
)
workflow.logger.info(f"CronTriggerWorkflow spawned child run: child_id={child_id} " f"timestamp={firing_iso}")
@@ -161,4 +208,8 @@ def _build_trigger_output(listener_data: Dict[str, Any]) -> Dict[str, Any]:
}
-__all__ = ["CronTriggerWorkflow"]
+__all__ = [
+ "CHILD_SEARCH_ATTRIBUTES_PATCH",
+ "COOPERATIVE_PAUSE_SCHEDULING_PATCH",
+ "CronTriggerWorkflow",
+]
diff --git a/server/services/agent_team.py b/server/services/agent_team.py
index ab34c21f..3889de64 100644
--- a/server/services/agent_team.py
+++ b/server/services/agent_team.py
@@ -359,6 +359,32 @@ class AgentTeamService:
return success
+ async def cancel_delegation(
+ self,
+ team_id: str,
+ task_id: str,
+ reason: str,
+ ) -> Optional[Dict[str, Any]]:
+ """Terminally cancel delegated work without failing or requeueing it."""
+ task = await self.database.cancel_team_task(team_id, task_id, reason)
+ if not task:
+ return None
+
+ if task.get("status") == "cancelled":
+ if task.get("assigned_to"):
+ await self.database.update_member_status(
+ team_id,
+ task["assigned_to"],
+ "idle",
+ )
+ if task.get("cancellation_applied") and self.broadcaster:
+ await self.broadcaster.broadcast_team_event(
+ team_id,
+ "team.task.cancelled",
+ task,
+ )
+ return task
+
async def get_claimable_tasks(self, team_id: str) -> List[Dict[str, Any]]:
"""Get tasks ready to be claimed."""
return await self.database.get_claimable_tasks(team_id)
diff --git a/server/services/deployment/control.py b/server/services/deployment/control.py
index 2cea0500..2d00adf9 100644
--- a/server/services/deployment/control.py
+++ b/server/services/deployment/control.py
@@ -72,6 +72,10 @@ class WorkflowControlService:
controller_workflow_id=f"workflow-control-{workflow_id}-g{generation}",
session_id=session_id, graph_hash=_graph_hash(nodes, edges),
graph_snapshot={"nodes": nodes, "edges": edges}, idempotency_key=idempotency_key,
+ # Revisions are monotonic across generations so a delayed request
+ # from an archived generation cannot pass CAS against a newer one
+ # that happens to be at the same lifecycle step.
+ revision=(latest.revision + 1) if latest else 0,
)
try:
return await self.database.create_workflow_control(control), True
diff --git a/server/services/deployment/handlers.py b/server/services/deployment/handlers.py
index 619071f7..8f5acf96 100644
--- a/server/services/deployment/handlers.py
+++ b/server/services/deployment/handlers.py
@@ -42,6 +42,14 @@ from services.deployment.control import WorkflowControlService, serialize_contro
logger = get_logger(__name__)
+class TemporalControlUnavailable(RuntimeError):
+ """A lifecycle Update could not start because no Temporal client exists."""
+
+
+class TemporalControlAckMismatch(RuntimeError):
+ """Temporal completed an Update without returning the requested state."""
+
+
# Per-workflow deployment tasks for proper cancellation (Temporal/n8n pattern).
# Maps workflow_id -> asyncio.Task for parallel workflow deployments.
_deployment_tasks: Dict[str, asyncio.Task] = {}
@@ -137,7 +145,12 @@ async def handle_deploy_workflow(data: Dict[str, Any], websocket: WebSocket) ->
"locked_at": lock_info.get("locked_at"),
}
- await broadcaster.update_workflow_status(executing=True, current_node=None, progress=0)
+ await broadcaster.update_workflow_status(
+ executing=True,
+ current_node=None,
+ progress=0,
+ workflow_id=workflow_id,
+ )
await broadcaster.update_deployment_status(
is_running=True,
status="starting",
@@ -161,7 +174,12 @@ async def handle_deploy_workflow(data: Dict[str, Any], websocket: WebSocket) ->
position = node_data.get("position", 0) if node_data else 0
total = node_data.get("total", 1) if node_data else 1
progress = int((position / total) * 100) if total > 0 else 0
- await broadcaster.update_workflow_status(executing=True, current_node=node_id, progress=progress)
+ await broadcaster.update_workflow_status(
+ executing=True,
+ current_node=node_id,
+ progress=progress,
+ workflow_id=workflow_id,
+ )
async def run_deployment():
try:
@@ -184,6 +202,7 @@ async def handle_deploy_workflow(data: Dict[str, Any], websocket: WebSocket) ->
)
await broadcaster.unlock_workflow(workflow_id)
_deployment_tasks.pop(workflow_id, None)
+ return result
else:
await broadcaster.update_deployment_status(
is_running=True,
@@ -201,6 +220,7 @@ async def handle_deploy_workflow(data: Dict[str, Any], websocket: WebSocket) ->
workflow_id=workflow_id,
triggers=len(result.get("triggers_setup", [])),
)
+ return result
except Exception as e:
logger.error("Deployment task error", workflow_id=workflow_id, error=str(e))
@@ -213,6 +233,7 @@ async def handle_deploy_workflow(data: Dict[str, Any], websocket: WebSocket) ->
)
await broadcaster.unlock_workflow(workflow_id)
_deployment_tasks.pop(workflow_id, None)
+ return {"success": False, "error": str(e), "workflow_id": workflow_id}
_deployment_tasks[workflow_id] = asyncio.create_task(run_deployment())
@@ -271,7 +292,12 @@ async def handle_cancel_deployment(data: Dict[str, Any], websocket: WebSocket) -
for node_id in result.get("cancelled_listener_node_ids", []):
await broadcaster.clear_node_status(node_id)
- await broadcaster.update_workflow_status(executing=False, current_node=None, progress=0)
+ await broadcaster.update_workflow_status(
+ executing=False,
+ current_node=None,
+ progress=0,
+ workflow_id=workflow_id,
+ )
await broadcaster.update_deployment_status(
is_running=False,
status="cancelled",
@@ -380,12 +406,14 @@ def _control_service():
async def _start_controller(control) -> Optional[str]:
- """Start the durable Temporal controller when Temporal is available."""
+ """Start the durable controller, or use local mode when Temporal is disabled."""
from core.container import container
from temporalio.common import SearchAttributeKey, SearchAttributePair, TypedSearchAttributes
wrapper = container.temporal_client()
if wrapper is None or wrapper.client is None:
+ if container.settings().temporal_enabled:
+ raise RuntimeError("temporal_control_unavailable")
return None
handle = await wrapper.client.start_workflow(
"WorkflowControlWorkflow",
@@ -398,7 +426,7 @@ async def _start_controller(control) -> Optional[str]:
"state": "running",
}],
id=control.controller_workflow_id,
- task_queue="machina-tasks",
+ task_queue=container.settings().temporal_task_queue,
search_attributes=TypedSearchAttributes([
SearchAttributePair(
SearchAttributeKey.for_keyword("EventWorkflowId"), control.workflow_id,
@@ -408,17 +436,151 @@ async def _start_controller(control) -> Optional[str]:
return getattr(handle, "result_run_id", None) or getattr(handle, "first_execution_run_id", None)
-async def _signal_controller(control, signal_name: str) -> None:
+def _controller_handle(control):
from core.container import container
wrapper = container.temporal_client()
- if wrapper is not None and wrapper.client is not None and control.controller_workflow_id:
- await wrapper.client.get_workflow_handle(
- control.controller_workflow_id, run_id=control.controller_run_id
- ).signal(signal_name)
+ if wrapper is None or wrapper.client is None or not control.controller_workflow_id:
+ return None
+ return wrapper.client.get_workflow_handle(
+ control.controller_workflow_id,
+ run_id=control.controller_run_id,
+ )
+
+async def _signal_controller(control, signal_name: str) -> None:
+ handle = _controller_handle(control)
+ if handle is not None:
+ await handle.signal(signal_name)
+
+
+async def _update_controller_state(
+ control,
+ requested_state: str,
+ *,
+ update_id: str,
+) -> Optional[Dict[str, Any]]:
+ """Apply and await a durable controller state change.
+
+ Temporal-disabled installations retain the local deployment path. When
+ Temporal is enabled, losing its client is a failed mutation rather than a
+ false-positive pause/resume acknowledgement.
+ """
+ from core.container import container
+
+ handle = _controller_handle(control)
+ if handle is None:
+ if container.settings().temporal_enabled:
+ raise TemporalControlUnavailable("temporal_control_unavailable")
+ return None
+ result = await handle.execute_update(
+ "set_control_state",
+ requested_state,
+ id=update_id,
+ )
+ expected_state = "paused" if requested_state in {"pause", "paused"} else "running"
+ if not isinstance(result, dict) or result.get("state") != expected_state:
+ raise TemporalControlAckMismatch(
+ f"temporal_control_ack_mismatch:{result.get('state') if isinstance(result, dict) else 'missing'}"
+ )
+ return result
-async def _signal_generation_workflows(workflow_id: str, signal_name: str) -> int:
+
+async def _query_controller_state(control) -> Optional[Dict[str, Any]]:
+ """Return controller state when reachable; status reads remain resilient."""
+ handle = _controller_handle(control)
+ if handle is None:
+ return None
+ try:
+ result = await handle.query("status")
+ return result if isinstance(result, dict) else None
+ except Exception as exc:
+ logger.warning(
+ "Workflow controller status query failed",
+ workflow_id=control.workflow_id,
+ controller_workflow_id=control.controller_workflow_id,
+ error=str(exc),
+ )
+ return None
+
+
+def _generation_visibility_query(control) -> str:
+ """Visibility query for controller descendants and standalone trigger roots."""
+ if control.controller_workflow_id:
+ root_id = str(control.controller_workflow_id).replace("'", "''")
+ workflow_id = str(control.workflow_id).replace("'", "''")
+ return (
+ f"(RootWorkflowId='{root_id}' OR "
+ f"EventWorkflowId='{workflow_id}') "
+ "AND ExecutionStatus='Running'"
+ )
+ workflow_id = str(control.workflow_id).replace("'", "''")
+ return f"EventWorkflowId='{workflow_id}' AND ExecutionStatus='Running'"
+
+
+def _visibility_literal(value: Any) -> str:
+ return str(value).replace("'", "''")
+
+
+async def _list_generation_workflows(client, control) -> list[Any]:
+ """Resolve tagged roots and every running descendant in those trees.
+
+ Temporal Search Attributes are not inherited by child workflows. The
+ first query therefore discovers the controller tree plus tagged standalone
+ trigger/graph roots; a second batched RootWorkflowId query expands those
+ roots to active Agent/DelegatedTask descendants.
+ """
+ targets: Dict[tuple[str, str], Any] = {}
+ root_ids: set[str] = set()
+
+ async for execution in client.list_workflows(
+ query=_generation_visibility_query(control)
+ ):
+ execution_id = str(execution.id)
+ run_id = str(getattr(execution, "run_id", "") or "")
+ targets[(execution_id, run_id)] = execution
+ root_ids.add(
+ str(getattr(execution, "root_id", None) or execution_id)
+ )
+
+ ordered_roots = sorted(root_ids)
+ batch_size = 40
+ for offset in range(0, len(ordered_roots), batch_size):
+ batch = ordered_roots[offset : offset + batch_size]
+ root_values = ", ".join(
+ f"'{_visibility_literal(root_id)}'"
+ for root_id in batch
+ )
+ query = (
+ f"RootWorkflowId IN ({root_values}) "
+ "AND ExecutionStatus='Running'"
+ )
+ async for execution in client.list_workflows(query=query):
+ execution_id = str(execution.id)
+ run_id = str(getattr(execution, "run_id", "") or "")
+ targets[(execution_id, run_id)] = execution
+
+ return list(targets.values())
+
+
+def _temporal_target_already_gone(exc: Exception) -> bool:
+ try:
+ from temporalio.service import RPCError, RPCStatusCode
+
+ if isinstance(exc, RPCError) and exc.status == RPCStatusCode.NOT_FOUND:
+ return True
+ except Exception:
+ pass
+ message = str(exc).lower()
+ return "not found" in message or "already completed" in message
+
+
+async def _signal_generation_workflows(
+ control,
+ signal_name: str,
+ *,
+ strict: bool = False,
+) -> int:
"""Best-effort cooperative fan-out to this deployment's live executions.
Visibility is discovery only; durable control state remains authoritative.
@@ -428,23 +590,28 @@ async def _signal_generation_workflows(workflow_id: str, signal_name: str) -> in
wrapper = container.temporal_client()
if wrapper is None or wrapper.client is None:
+ if strict and container.settings().temporal_enabled:
+ raise TemporalControlUnavailable("temporal_control_unavailable")
return 0
- query = (
- f"EventWorkflowId='{workflow_id}' "
- "AND ExecutionStatus='Running'"
- )
signalled = 0
+ failures: list[Exception] = []
try:
- async for execution in wrapper.client.list_workflows(query=query):
+ executions = await _list_generation_workflows(
+ wrapper.client,
+ control,
+ )
+ for execution in executions:
try:
await wrapper.client.get_workflow_handle(
execution.id, run_id=execution.run_id
).signal(signal_name)
signalled += 1
except Exception as exc:
+ if not _temporal_target_already_gone(exc):
+ failures.append(exc)
logger.warning(
"Workflow control signal failed",
- workflow_id=workflow_id,
+ workflow_id=control.workflow_id,
temporal_workflow_id=execution.id,
signal=signal_name,
error=str(exc),
@@ -452,59 +619,116 @@ async def _signal_generation_workflows(workflow_id: str, signal_name: str) -> in
except Exception as exc:
logger.warning(
"Workflow control visibility fan-out failed",
- workflow_id=workflow_id,
+ workflow_id=control.workflow_id,
signal=signal_name,
error=str(exc),
)
+ if strict:
+ raise RuntimeError(
+ "workflow_signal_visibility_failed"
+ ) from exc
+ if strict and failures:
+ raise RuntimeError(
+ f"workflow_signal_failed:{len(failures)}"
+ ) from failures[0]
return signalled
-async def _terminate_generation_workflows(workflow_id: str) -> int:
+async def _terminate_generation_workflows(control, *, strict: bool = False) -> int:
"""Immediately terminate every visible execution in one application tree."""
from core.container import container
wrapper = container.temporal_client()
if wrapper is None or wrapper.client is None:
+ if strict and container.settings().temporal_enabled:
+ raise TemporalControlUnavailable("temporal_control_unavailable")
return 0
terminated = 0
+ failures: list[Exception] = []
try:
- async for execution in wrapper.client.list_workflows(
- query=f"EventWorkflowId='{workflow_id}' AND ExecutionStatus='Running'"
- ):
+ executions = await _list_generation_workflows(
+ wrapper.client,
+ control,
+ )
+ for execution in executions:
try:
await wrapper.client.get_workflow_handle(
execution.id, run_id=execution.run_id
).terminate(reason="workflow_reset")
terminated += 1
except Exception as exc:
+ if not _temporal_target_already_gone(exc):
+ failures.append(exc)
logger.warning(
"Workflow reset termination failed",
- workflow_id=workflow_id,
+ workflow_id=control.workflow_id,
temporal_workflow_id=execution.id,
error=str(exc),
)
except Exception as exc:
logger.warning(
"Workflow reset visibility scan failed",
- workflow_id=workflow_id,
+ workflow_id=control.workflow_id,
error=str(exc),
)
+ if strict:
+ raise RuntimeError("workflow_visibility_cleanup_failed") from exc
+ if strict and failures:
+ raise RuntimeError(
+ f"workflow_termination_failed:{len(failures)}"
+ ) from failures[0]
return terminated
def _expected_revision(data: Dict[str, Any], control) -> int:
supplied = data.get("expected_revision")
- return control.revision if supplied is None else int(supplied)
+ if supplied is None:
+ raise ValueError("expected_revision_required")
+ return int(supplied)
-async def _set_cron_pause(workflow_id: str, *, paused: bool) -> int:
+async def _set_cron_pause(
+ workflow_id: str,
+ *,
+ paused: bool,
+ strict: bool = False,
+) -> int:
from core.container import container
from services.temporal.schedules import set_cron_schedules_paused
wrapper = container.temporal_client()
if wrapper is None or wrapper.client is None:
+ if strict and container.settings().temporal_enabled:
+ raise TemporalControlUnavailable("temporal_control_unavailable")
return 0
- return await set_cron_schedules_paused(wrapper.client, workflow_id, paused=paused)
+ return await set_cron_schedules_paused(
+ wrapper.client,
+ workflow_id,
+ paused=paused,
+ strict=strict,
+ )
+
+
+async def _delete_cron_schedules(
+ workflow_id: str,
+ *,
+ strict: bool = False,
+) -> int:
+ from core.container import container
+ from services.temporal.schedules import (
+ delete_cron_schedules_for_deployment,
+ )
+
+ wrapper = container.temporal_client()
+ if wrapper is None or wrapper.client is None:
+ if strict and container.settings().temporal_enabled:
+ raise TemporalControlUnavailable("temporal_control_unavailable")
+ return 0
+ return await delete_cron_schedules_for_deployment(
+ wrapper.client,
+ workflow_id,
+ strict=strict,
+ )
async def _with_runtime_counts(payload: Dict[str, Any], workflow_id: str) -> Dict[str, Any]:
@@ -515,13 +739,240 @@ async def _with_runtime_counts(payload: Dict[str, Any], workflow_id: str) -> Dic
**payload,
"active_count": status.get("active_runs", 0),
"in_flight_count": status.get("active_runs", 0),
- "queued_count": 0,
+ "queued_count": (
+ int(payload.get("queued_count", 0) or 0)
+ + int(status.get("queued_events", 0) or 0)
+ ),
+ }
+
+
+def _close_local_admission(workflow_id: str) -> None:
+ """Synchronously gate legacy trigger callbacks before durable cleanup."""
+ from core.container import container
+
+ container.workflow_service().pause_deployment(workflow_id)
+
+
+async def _control_payload(
+ control,
+ *,
+ extra: Optional[Dict[str, Any]] = None,
+ controller_status: Optional[Dict[str, Any]] = None,
+) -> Dict[str, Any]:
+ payload = serialize_control(control)
+ if controller_status is not None:
+ payload.update({
+ "temporal_state": controller_status.get("state"),
+ "temporal_revision": controller_status.get("revision"),
+ "queued_count": controller_status.get("queued_events", 0),
+ "temporal_available": True,
+ })
+ payload.update(extra or {})
+ return await _with_runtime_counts(payload, control.workflow_id)
+
+
+async def _broadcast_control(
+ control,
+ *,
+ extra: Optional[Dict[str, Any]] = None,
+ controller_status: Optional[Dict[str, Any]] = None,
+) -> Dict[str, Any]:
+ from services.status_broadcaster import get_status_broadcaster
+
+ payload = await _control_payload(
+ control,
+ extra=extra,
+ controller_status=controller_status,
+ )
+ await get_status_broadcaster().broadcast({
+ "type": "workflow_control_status",
+ "workflow_id": control.workflow_id,
+ "data": payload,
+ })
+ return payload
+
+
+async def _reconcile_control(service: WorkflowControlService, control):
+ """Finish an interrupted DB transition from acknowledged Temporal state."""
+ from core.container import container
+
+ controller_status = await _query_controller_state(control)
+ transition_target = {
+ "pausing": ("paused", "paused"),
+ "resuming": ("running", "running"),
+ }.get(control.status)
+ if transition_target is None:
+ return control, controller_status
+ requested_state, stable_state = transition_target
+
+ if controller_status is None:
+ if container.settings().temporal_enabled:
+ return control, None
+ if control.status == "pausing":
+ container.workflow_service().pause_deployment(control.workflow_id)
+ if controller_status is not None and controller_status.get("state") != stable_state:
+ controller_status = await _update_controller_state(
+ control,
+ requested_state,
+ update_id=(
+ f"reconcile:{control.id}:{control.revision}:{stable_state}"
+ ),
+ )
+
+ if control.status == "pausing":
+ paused_schedules = await _set_cron_pause(
+ control.workflow_id,
+ paused=True,
+ strict=True,
+ )
+ paused_triggers = await container.workflow_service().update_trigger_pause_status(
+ control.workflow_id,
+ paused=True,
+ )
+ signalled = await _signal_generation_workflows(
+ control,
+ "pause",
+ strict=True,
+ )
+ transition_details = {
+ "signalled_executions": signalled,
+ "paused_schedules": paused_schedules,
+ "paused_triggers": paused_triggers,
+ }
+ else:
+ resumed_schedules = await _set_cron_pause(
+ control.workflow_id,
+ paused=False,
+ strict=True,
+ )
+ signalled = await _signal_generation_workflows(
+ control,
+ "resume",
+ strict=True,
+ )
+ queued = await container.workflow_service().resume_deployment(
+ control.workflow_id,
+ )
+ resumed_triggers = await container.workflow_service().update_trigger_pause_status(
+ control.workflow_id,
+ paused=False,
+ )
+ transition_details = {
+ "resumed_queued_events": queued,
+ "signalled_executions": signalled,
+ "resumed_schedules": resumed_schedules,
+ "resumed_triggers": resumed_triggers,
+ }
+ try:
+ control = await service.transition(
+ control,
+ expected_revision=control.revision,
+ from_statuses={control.status},
+ status=stable_state,
+ )
+ await _broadcast_control(
+ control,
+ controller_status=controller_status,
+ extra=transition_details,
+ )
+ except ValueError as exc:
+ if str(exc) != "control_revision_conflict":
+ raise
+ latest = await service.database.get_latest_workflow_control(control.workflow_id)
+ if latest is not None and latest.generation == control.generation:
+ control = latest
+ return control, controller_status
+
+
+async def _await_deployment_setup(workflow_id: str) -> Dict[str, Any]:
+ """Wait for trigger/listener setup started by the legacy deploy handler."""
+ task = _deployment_tasks.get(workflow_id)
+ if task is None:
+ return {
+ "success": False,
+ "error": "deployment_setup_task_missing",
+ "workflow_id": workflow_id,
+ }
+ result = await asyncio.shield(task)
+ if isinstance(result, dict):
+ return result
+ return {
+ "success": False,
+ "error": "deployment_setup_did_not_return_status",
+ "workflow_id": workflow_id,
}
+async def _restore_control_after_failed_update(
+ service: WorkflowControlService,
+ control,
+ *,
+ transitional_state: str,
+ stable_state: str,
+):
+ """Undo the DB projection when Temporal rejected a lifecycle update."""
+ try:
+ restored = await service.transition(
+ control,
+ expected_revision=control.revision,
+ from_statuses={transitional_state},
+ status=stable_state,
+ )
+ except ValueError:
+ latest = await service.database.get_latest_workflow_control(control.workflow_id)
+ restored = latest if latest is not None else control
+ await _broadcast_control(restored)
+ return restored
+
+
+async def _duplicate_start_response(
+ control,
+ *,
+ controller_status: Optional[Dict[str, Any]] = None,
+) -> Dict[str, Any]:
+ """Report the durable outcome of a retried Start idempotency key."""
+ payload = await _control_payload(
+ control,
+ controller_status=controller_status,
+ )
+ if control.status == "starting":
+ return {
+ "success": False,
+ "error": "workflow_start_pending",
+ "idempotent": True,
+ **payload,
+ }
+ if control.status == "failed":
+ return {
+ "success": False,
+ "error": control.terminal_reason or "workflow_start_failed",
+ "idempotent": True,
+ **payload,
+ }
+ return {"success": True, "idempotent": True, **payload}
+
+
@ws_handler("workflow_id")
async def handle_get_workflow_control_status(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
- return await _with_runtime_counts(await _control_service().get_status(data["workflow_id"]), data["workflow_id"])
+ service = _control_service()
+ control = await service.database.get_latest_workflow_control(data["workflow_id"])
+ if control is None:
+ return await _with_runtime_counts(
+ serialize_control(None),
+ data["workflow_id"],
+ )
+ control, controller_status = await _reconcile_control(service, control)
+ return await _control_payload(
+ control,
+ controller_status=controller_status,
+ extra={
+ "temporal_available": (
+ controller_status is not None
+ if control.controller_run_id
+ else False
+ ),
+ },
+ )
@ws_handler("workflow_id")
@@ -530,12 +981,35 @@ async def handle_start_workflow(data: Dict[str, Any], websocket: WebSocket) -> D
workflow_id = data["workflow_id"]
key = data.get("idempotency_key") or f"start:{workflow_id}:{uuid.uuid4().hex}"
service = _control_service()
+ existing = await service.database.get_workflow_control_by_idempotency_key(
+ workflow_id,
+ key,
+ )
+ if existing is not None:
+ existing, controller_status = await _reconcile_control(service, existing)
+ return await _duplicate_start_response(
+ existing,
+ controller_status=controller_status,
+ )
+
+ latest = await service.database.get_latest_workflow_control(workflow_id)
+ if data.get("expected_revision") is None:
+ raise ValueError("expected_revision_required")
+ expected_revision = int(data["expected_revision"])
+ if expected_revision != (latest.revision if latest else 0):
+ raise ValueError("control_revision_conflict")
+
control, created = await service.begin_generation(
workflow_id=workflow_id, nodes=data.get("nodes", []), edges=data.get("edges", []),
session_id=data.get("session_id", "default"), idempotency_key=key,
)
if not created:
- return await _with_runtime_counts({"success": True, "idempotent": True, **serialize_control(control)}, workflow_id)
+ control, controller_status = await _reconcile_control(service, control)
+ return await _duplicate_start_response(
+ control,
+ controller_status=controller_status,
+ )
+ await _broadcast_control(control)
try:
run_id = await _start_controller(control)
if run_id:
@@ -546,6 +1020,7 @@ async def handle_start_workflow(data: Dict[str, Any], websocket: WebSocket) -> D
await service.database.update_workflow_run_data_scope(
control.data_scope_id or control.execution_id, temporal_run_id=run_id,
)
+ await _broadcast_control(control)
# Runtime persistence is generation-scoped. The caller's session is
# retained on the scope for provenance, but must never namespace node
# outputs for a controlled run.
@@ -557,14 +1032,39 @@ async def handle_start_workflow(data: Dict[str, Any], websocket: WebSocket) -> D
}
deployed = await handle_deploy_workflow(deploy_data, websocket)
if not deployed.get("success"):
- await service.fail(control, str(deployed.get("error", "deployment_failed")))
- return deployed
+ raise RuntimeError(str(deployed.get("error", "deployment_failed")))
+ deployed = await _await_deployment_setup(workflow_id)
+ if not deployed.get("success"):
+ raise RuntimeError(str(deployed.get("error", "deployment_failed")))
control = await service.transition(control, expected_revision=control.revision, from_statuses={"starting"}, status="running")
- return await _with_runtime_counts({"success": True, **serialize_control(control)}, workflow_id)
except Exception as exc:
- await service.fail(control, str(exc))
+ latest = await service.database.get_latest_workflow_control(workflow_id)
+ if latest is not None and latest.generation == control.generation:
+ control = latest
+ if control.status == "starting":
+ control = await service.fail(control, str(exc))
+ try:
+ await _signal_controller(control, "reset")
+ except Exception as reset_exc:
+ logger.warning(
+ "Failed to reset controller after deployment setup error",
+ workflow_id=workflow_id,
+ error=str(reset_exc),
+ )
+ await _terminate_generation_workflows(control)
+ await _broadcast_control(control)
raise
+ # Reaching ``running`` commits successful deployment setup. Projection
+ # failures after that point must not tear down a live durable generation;
+ # the client can recover the committed state through its status resync.
+ controller_status = await _query_controller_state(control)
+ payload = await _broadcast_control(
+ control,
+ controller_status=controller_status,
+ )
+ return {"success": True, **payload}
+
@ws_handler("workflow_id")
async def handle_pause_workflow(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
@@ -575,20 +1075,64 @@ async def handle_pause_workflow(data: Dict[str, Any], websocket: WebSocket) -> D
control = await service.database.get_latest_workflow_control(workflow_id)
if control is None:
return {"success": False, "error": "workflow_never_started"}
+ control, controller_status = await _reconcile_control(service, control)
+ if control.status == "paused":
+ return {
+ "success": True,
+ "idempotent": True,
+ **await _control_payload(control, controller_status=controller_status),
+ }
+ if control.status == "pausing":
+ return {
+ "success": False,
+ "error": "workflow_control_transition_pending",
+ **await _control_payload(control, controller_status=controller_status),
+ }
control = await service.transition(
control, expected_revision=_expected_revision(data, control), from_statuses={"running"}, status="pausing"
)
+ await _broadcast_control(control)
container.workflow_service().pause_deployment(workflow_id)
- await _signal_controller(control, "pause")
- paused_schedules = await _set_cron_pause(workflow_id, paused=True)
+ try:
+ controller_status = await _update_controller_state(
+ control,
+ "paused",
+ update_id=(
+ f"{control.id}:{control.revision}:"
+ f"{data.get('idempotency_key') or uuid.uuid4().hex}:paused"
+ ),
+ )
+ except (TemporalControlUnavailable, TemporalControlAckMismatch):
+ await container.workflow_service().resume_deployment(workflow_id)
+ await _restore_control_after_failed_update(
+ service,
+ control,
+ transitional_state="pausing",
+ stable_state="running",
+ )
+ raise
+ paused_schedules = await _set_cron_pause(
+ workflow_id,
+ paused=True,
+ strict=True,
+ )
paused_triggers = await container.workflow_service().update_trigger_pause_status(workflow_id, paused=True)
- signalled = await _signal_generation_workflows(workflow_id, "pause")
+ signalled = await _signal_generation_workflows(
+ control,
+ "pause",
+ strict=True,
+ )
control = await service.transition(control, expected_revision=control.revision, from_statuses={"pausing"}, status="paused")
- return await _with_runtime_counts({
+ payload = await _broadcast_control(control, controller_status=controller_status, extra={
+ "signalled_executions": signalled,
+ "paused_schedules": paused_schedules,
+ "paused_triggers": paused_triggers,
+ })
+ return {
"success": True, "signalled_executions": signalled,
"paused_schedules": paused_schedules, "paused_triggers": paused_triggers,
- **serialize_control(control),
- }, workflow_id)
+ **payload,
+ }
@ws_handler("workflow_id")
@@ -600,20 +1144,64 @@ async def handle_resume_workflow(data: Dict[str, Any], websocket: WebSocket) ->
control = await service.database.get_latest_workflow_control(workflow_id)
if control is None:
return {"success": False, "error": "workflow_never_started"}
+ control, controller_status = await _reconcile_control(service, control)
+ if control.status == "running":
+ return {
+ "success": True,
+ "idempotent": True,
+ **await _control_payload(control, controller_status=controller_status),
+ }
+ if control.status == "resuming":
+ return {
+ "success": False,
+ "error": "workflow_control_transition_pending",
+ **await _control_payload(control, controller_status=controller_status),
+ }
control = await service.transition(
control, expected_revision=_expected_revision(data, control), from_statuses={"paused"}, status="resuming"
)
- await _signal_controller(control, "resume")
- resumed_schedules = await _set_cron_pause(workflow_id, paused=False)
- signalled = await _signal_generation_workflows(workflow_id, "resume")
+ await _broadcast_control(control)
+ try:
+ controller_status = await _update_controller_state(
+ control,
+ "running",
+ update_id=(
+ f"{control.id}:{control.revision}:"
+ f"{data.get('idempotency_key') or uuid.uuid4().hex}:running"
+ ),
+ )
+ except (TemporalControlUnavailable, TemporalControlAckMismatch):
+ await _restore_control_after_failed_update(
+ service,
+ control,
+ transitional_state="resuming",
+ stable_state="paused",
+ )
+ raise
+ resumed_schedules = await _set_cron_pause(
+ workflow_id,
+ paused=False,
+ strict=True,
+ )
+ signalled = await _signal_generation_workflows(
+ control,
+ "resume",
+ strict=True,
+ )
queued = await container.workflow_service().resume_deployment(workflow_id)
resumed_triggers = await container.workflow_service().update_trigger_pause_status(workflow_id, paused=False)
control = await service.transition(control, expected_revision=control.revision, from_statuses={"resuming"}, status="running")
- return await _with_runtime_counts({
+ payload = await _broadcast_control(control, controller_status=controller_status, extra={
+ "resumed_queued_events": queued,
+ "signalled_executions": signalled,
+ "resumed_schedules": resumed_schedules,
+ "resumed_triggers": resumed_triggers,
+ })
+ return {
"success": True, "resumed_queued_events": queued, "signalled_executions": signalled,
"resumed_schedules": resumed_schedules, "resumed_triggers": resumed_triggers,
- **serialize_control(control),
- }, workflow_id)
+ **payload,
+ }
@ws_handler("workflow_id")
@@ -621,35 +1209,97 @@ async def handle_reset_workflow(data: Dict[str, Any], websocket: WebSocket) -> D
workflow_id = data["workflow_id"]
service = _control_service()
current = await service.database.get_latest_workflow_control(workflow_id)
- if current is not None:
+ if current is None:
+ return await _with_runtime_counts(
+ await service.get_status(workflow_id),
+ workflow_id,
+ )
+
+ # ``reset`` is a completed cleanup barrier. Re-running generation-wide
+ # sweeps here would race a concurrent Start and could terminate resources
+ # from the next generation because standalone triggers use the stable
+ # application workflow id.
+ if current.status == "reset":
+ return {
+ "success": True,
+ "idempotent": True,
+ **await _control_payload(current),
+ }
+
+ if current.status != "resetting":
current = await service.transition(
current, expected_revision=_expected_revision(data, current),
from_statuses={"starting", "running", "pausing", "paused", "resuming", "failed"}, status="resetting",
values={"terminal_reason": "workflow_reset", "completed_at": datetime.now(timezone.utc)},
)
- try:
- await _signal_controller(current, "reset")
- finally:
- terminated = await _terminate_generation_workflows(workflow_id)
- await handle_cancel_deployment({"workflow_id": workflow_id}, websocket)
- else:
- terminated = 0
- if current is None:
- return await _with_runtime_counts(await service.get_status(workflow_id), workflow_id)
- current = await service.transition(
- current, expected_revision=current.revision, from_statuses={"resetting"}, status="reset",
- values={"terminal_reason": "workflow_reset", "completed_at": datetime.now(timezone.utc)},
+
+ # Quiesce every producer before the final execution sweep. The local gate
+ # is synchronous, so no callback can be admitted while broadcasts or
+ # Temporal cleanup yield control.
+ _close_local_admission(workflow_id)
+ await _broadcast_control(current)
+
+ try:
+ await _signal_controller(current, "reset")
+ except Exception as exc:
+ logger.warning(
+ "Workflow reset signal failed; continuing with termination",
+ workflow_id=workflow_id,
+ error=str(exc),
+ )
+
+ # Remove cron producers before terminating executions; otherwise a firing
+ # between the execution scan and schedule deletion could survive Reset.
+ deleted_schedules = await _delete_cron_schedules(
+ workflow_id,
+ strict=True,
)
- await service.database.update_workflow_run_data_scope(
+ cancelled = await handle_cancel_deployment(
+ {"workflow_id": workflow_id},
+ websocket,
+ )
+ # Durable Temporal cleanup above is authoritative. The process-local
+ # deployment may legitimately be absent after an API-server restart.
+ # Any other local teardown failure can leave a listener/admission path
+ # alive, so keep the durable control in ``resetting`` for a safe retry.
+ local_cleanup_completed = bool(cancelled.get("success"))
+ if not local_cleanup_completed:
+ # ``handle_cancel_deployment`` retains its historical envelope and
+ # exposes manager errors as ``message``. Accept ``error`` as well for
+ # direct/internal callers and future wire compatibility.
+ local_error = str(
+ cancelled.get("error")
+ or cancelled.get("message")
+ or "unknown"
+ )
+ expected_absent_error = f"Workflow {workflow_id} is not deployed"
+ if local_error != expected_absent_error:
+ raise RuntimeError(
+ f"workflow_local_cleanup_failed:{local_error}"
+ )
+
+ # Controller, cron, and legacy local admission paths are now closed. This
+ # final strict sweep therefore observes a fixed set of generation runs.
+ terminated = await _terminate_generation_workflows(current, strict=True)
+
+ archived = await service.database.update_workflow_run_data_scope(
current.data_scope_id or current.execution_id,
status="archived", archived_at=datetime.now(timezone.utc),
)
+ if not archived:
+ raise RuntimeError("workflow_data_scope_archive_failed")
+
from services.status_broadcaster import get_status_broadcaster
from services.deployment.runtime_state import archive_and_reset_node_state
broadcaster = get_status_broadcaster()
node_state = await archive_and_reset_node_state(
current, service.database, broadcaster,
)
+
+ current = await service.transition(
+ current, expected_revision=current.revision, from_statuses={"resetting"}, status="reset",
+ values={"terminal_reason": "workflow_reset", "completed_at": datetime.now(timezone.utc)},
+ )
await broadcaster.broadcast({
"type": "workflow_runtime_reset",
"workflow_id": workflow_id,
@@ -658,16 +1308,23 @@ async def handle_reset_workflow(data: Dict[str, Any], websocket: WebSocket) -> D
"archived_nodes": node_state["archived_nodes"],
"reset_nodes": node_state["reset_nodes"],
})
- return await _with_runtime_counts(
- {
- "success": True,
- "terminated_executions": terminated,
- "archived_nodes": node_state["archived_nodes"],
- "reset_nodes": node_state["reset_nodes"],
- **serialize_control(current),
- },
- workflow_id,
- )
+ payload = await _broadcast_control(current, extra={
+ "terminated_executions": terminated,
+ "deleted_schedules": deleted_schedules,
+ "local_cleanup_completed": local_cleanup_completed,
+ "archived_nodes": node_state["archived_nodes"],
+ "reset_nodes": node_state["reset_nodes"],
+ })
+ return {
+ "success": True,
+ "idempotent": False,
+ "terminated_executions": terminated,
+ "deleted_schedules": deleted_schedules,
+ "local_cleanup_completed": local_cleanup_completed,
+ "archived_nodes": node_state["archived_nodes"],
+ "reset_nodes": node_state["reset_nodes"],
+ **payload,
+ }
WS_HANDLERS: Dict[str, Any] = {
diff --git a/server/services/deployment/manager.py b/server/services/deployment/manager.py
index 0df370fd..a2eada8b 100644
--- a/server/services/deployment/manager.py
+++ b/server/services/deployment/manager.py
@@ -117,6 +117,11 @@ class DeploymentManager:
await self._spawn_run(trigger_node_id, trigger_data, workflow_id=workflow_id)
return len(queued)
+ def _clear_paused_state(self, workflow_id: str) -> None:
+ """Drop admission and queued-event state for a cancelled generation."""
+ self._paused_workflows.discard(workflow_id)
+ self._paused_events.pop(workflow_id, None)
+
async def update_trigger_pause_status(self, workflow_id: str, *, paused: bool) -> int:
"""Reflect deployment admission state on every armed trigger node."""
state = self._deployments.get(workflow_id)
@@ -263,11 +268,18 @@ class DeploymentManager:
# Find workflow to cancel
if workflow_id:
if not self.is_workflow_deployed(workflow_id):
+ # Reset/cancel callers may arrive after deployment teardown.
+ # With no live admission path, stale paused events are safe to
+ # discard immediately.
+ self._clear_paused_state(workflow_id)
return {"success": False, "error": f"Workflow {workflow_id} is not deployed"}
else:
# Backward compatibility: cancel first running deployment
deployed = self.get_deployed_workflows()
if not deployed:
+ # Any pause entries without a live deployment are stale.
+ self._paused_workflows.clear()
+ self._paused_events.clear()
return {"success": False, "error": "No deployment running"}
workflow_id = deployed[0]
@@ -278,6 +290,11 @@ class DeploymentManager:
deployment_id = state.deployment_id
logger.info("Cancelling deployment", deployment_id=deployment_id, workflow_id=workflow_id)
+ # Close admission before taking the active-run snapshot. Keep this
+ # barrier set until every teardown step succeeds; otherwise a callback
+ # racing cancellation could launch an untracked run.
+ self._paused_workflows.add(workflow_id)
+
# Get trigger manager for this workflow
trigger_manager = self._trigger_managers.get(workflow_id)
if trigger_manager:
@@ -355,6 +372,7 @@ class DeploymentManager:
self._active_runs.pop(workflow_id, None)
self._run_counters.pop(workflow_id, None)
self._status_callbacks.pop(workflow_id, None)
+ self._clear_paused_state(workflow_id)
return {
"success": True,
@@ -389,6 +407,7 @@ class DeploymentManager:
"active_runs": len(execution_runs),
"active_listeners": len(workflow_runs) - len(execution_runs),
"run_counter": self._run_counters.get(workflow_id, 0),
+ "queued_events": len(self._paused_events.get(workflow_id, [])),
"deployed_at": state.deployed_at,
}
diff --git a/server/services/temporal/agent_activities.py b/server/services/temporal/agent_activities.py
index 3bda5366..498eafbb 100644
--- a/server/services/temporal/agent_activities.py
+++ b/server/services/temporal/agent_activities.py
@@ -1456,6 +1456,7 @@ async def clear_agent_skills(payload: Dict[str, Any]) -> Dict[str, Any]:
async def begin_agent_delegation(payload: Dict[str, Any]) -> Dict[str, Any]:
"""Idempotently persist and claim a delegation before child startup."""
from services.agent_team import get_agent_team_service
+ from temporalio.exceptions import ApplicationError
team_id = str(payload.get("team_id") or "")
task_id = str(payload.get("team_task_id") or "")
@@ -1476,14 +1477,39 @@ async def begin_agent_delegation(payload: Dict[str, Any]) -> Dict[str, Any]:
if not task:
raise RuntimeError("Failed to persist delegated team task")
- if str(task.get("status") or "") in {"pending", "queued", ""}:
+ terminal_statuses = {
+ "submitted",
+ "accepted",
+ "failed",
+ "cancelled",
+ "skipped",
+ "completed",
+ }
+
+ def raise_terminal(status: str) -> None:
+ raise ApplicationError(
+ f"Delegated team task {task_id} is already terminal ({status})",
+ type="DelegationTaskTerminal",
+ non_retryable=True,
+ )
+
+ status = str(task.get("status") or "")
+ if status in {"pending", "queued"}:
claimed = await service.claim_task(team_id, task_id, child_id)
- if not claimed:
- tasks = await service.database.get_team_tasks(team_id)
- task = next((item for item in tasks if item.get("id") == task_id), None)
- if not task or task.get("assigned_to") != child_id:
- raise RuntimeError("Failed to claim delegated team task")
- elif task.get("assigned_to") and task.get("assigned_to") != child_id:
+ tasks = await service.database.get_team_tasks(team_id)
+ task = next((item for item in tasks if item.get("id") == task_id), None)
+ status = str((task or {}).get("status") or "")
+ if status in terminal_statuses:
+ raise_terminal(status)
+ if (
+ not claimed
+ or status != "running"
+ or task.get("assigned_to") != child_id
+ ):
+ raise RuntimeError("Failed to claim delegated team task")
+ elif status in terminal_statuses:
+ raise_terminal(status)
+ elif status != "running" or task.get("assigned_to") != child_id:
raise RuntimeError("Delegated team task is claimed by another agent")
message = await service.send_message(
@@ -1499,6 +1525,21 @@ async def begin_agent_delegation(payload: Dict[str, Any]) -> Dict[str, Any]:
)
if not message:
raise RuntimeError("Failed to persist delegation assignment event")
+
+ # Cancellation may commit while the idempotent assignment message is
+ # being persisted. Re-read immediately before returning permission to
+ # start the child so a terminal task is never reported as claimed.
+ tasks = await service.database.get_team_tasks(team_id)
+ task = next((item for item in tasks if item.get("id") == task_id), None)
+ status = str((task or {}).get("status") or "")
+ if status in terminal_statuses:
+ raise_terminal(status)
+ if (
+ not task
+ or status != "running"
+ or task.get("assigned_to") != child_id
+ ):
+ raise RuntimeError("Delegated team task lost its claim before child startup")
return {"team_id": team_id, "team_task_id": task_id, "claimed": True}
@@ -1535,6 +1576,74 @@ async def queue_agent_delegation(payload: Dict[str, Any]) -> Dict[str, Any]:
return {"team_id": team_id, "team_task_id": task_id, "status": "queued"}
[email protected](name="agent.cancel_delegation.v1")
+async def cancel_agent_delegation(payload: Dict[str, Any]) -> Dict[str, Any]:
+ """Persist cancellation as a terminal state without failure requeueing."""
+ from services.agent_team import get_agent_team_service
+
+ team_id = str(payload.get("team_id") or "")
+ task_id = str(payload.get("team_task_id") or "")
+ child_id = str(payload.get("child_agent_node_id") or "")
+ parent_id = str(payload.get("parent_agent_node_id") or "")
+ reason = str(
+ payload.get("reason")
+ or payload.get("error")
+ or "Delegated agent cancelled"
+ )
+ if not all((team_id, task_id, child_id, parent_id)):
+ raise ValueError("Incomplete delegation cancellation payload")
+
+ service = get_agent_team_service()
+ task = await service.cancel_delegation(team_id, task_id, reason)
+ if task is None:
+ raise RuntimeError("Failed to cancel delegated team task")
+
+ status = str(task.get("status") or "")
+ cancellation_applied = bool(task.get("cancellation_applied"))
+ if status == "cancelled":
+ persisted_reason = str(task.get("cancellation_reason") or reason)
+ message = await service.send_message(
+ team_id,
+ child_id,
+ f"Task {task_id} cancelled: {persisted_reason}",
+ to_agent=parent_id,
+ message_type="error",
+ event_id=str(
+ payload.get("terminal_event_id") or f"{task_id}:cancelled"
+ ),
+ extra_data={
+ "status": "cancelled",
+ "task_id": task_id,
+ "reason": persisted_reason,
+ "root_execution_id": payload.get("root_execution_id"),
+ "trace_id": payload.get("trace_id"),
+ },
+ )
+ if not message:
+ raise RuntimeError("Failed to persist delegation cancellation event")
+ else:
+ persisted_reason = None
+
+ return {
+ "team_id": team_id,
+ "team_task_id": task_id,
+ "status": status,
+ "cancelled": status == "cancelled",
+ "cancellation_applied": cancellation_applied,
+ "reason": persisted_reason,
+ }
+
+
+def _subagent_lease_id(
+ permit_id: str,
+ temporal_activity_id: str,
+ attempt: int,
+) -> str:
+ """Return the bounded physical lease identity for one activity attempt."""
+ identity = f"{permit_id}\0{temporal_activity_id}\0{attempt}".encode()
+ return f"subagent-lease-v2-{hashlib.sha256(identity).hexdigest()}"
+
+
@activity.defn(name="agent.acquire_subagent_permit.v1")
async def acquire_subagent_permit(payload: Dict[str, Any]) -> Dict[str, Any]:
"""Poll the durable root coordinator until this delegation is admitted."""
@@ -1543,14 +1652,91 @@ async def acquire_subagent_permit(payload: Dict[str, Any]) -> Dict[str, Any]:
root_id = str(payload.get("root_execution_id") or "")
permit_id = str(payload.get("permit_id") or "")
limit = max(1, int(payload.get("limit") or 3))
+ lease_version = int(payload.get("lease_version") or 1)
if not root_id or not permit_id:
raise ValueError("root_execution_id and permit_id are required")
+ if lease_version not in {1, 2}:
+ raise ValueError("lease_version must be 1 or 2")
+
service = get_agent_team_service()
- while True:
- if await service.acquire_subagent_permit(root_id, permit_id, limit):
- return {"root_execution_id": root_id, "permit_id": permit_id, "acquired": True}
- activity.heartbeat({"root_execution_id": root_id, "permit_id": permit_id, "status": "queued"})
- await asyncio.sleep(1)
+ lease_id = permit_id
+ attempt = 1
+ if lease_version == 2:
+ info = activity.info()
+ temporal_activity_id = str(info.activity_id)
+ attempt = max(1, int(info.attempt))
+ lease_id = _subagent_lease_id(
+ permit_id,
+ temporal_activity_id,
+ attempt,
+ )
+ for prior_attempt in range(1, attempt):
+ prior_lease_id = _subagent_lease_id(
+ permit_id,
+ temporal_activity_id,
+ prior_attempt,
+ )
+ if not await service.release_subagent_permit(
+ root_id,
+ prior_lease_id,
+ ):
+ raise RuntimeError(
+ "Failed to reconcile prior subagent permit lease "
+ f"{prior_lease_id}"
+ )
+
+ try:
+ while True:
+ if await service.acquire_subagent_permit(root_id, lease_id, limit):
+ result = {
+ "root_execution_id": root_id,
+ "permit_id": permit_id,
+ "acquired": True,
+ }
+ if lease_version == 2:
+ result.update({
+ "lease_version": 2,
+ "lease_id": lease_id,
+ "attempt": attempt,
+ })
+ return result
+
+ heartbeat = {
+ "root_execution_id": root_id,
+ "permit_id": permit_id,
+ "status": "queued",
+ }
+ if lease_version == 2:
+ heartbeat.update({
+ "lease_version": 2,
+ "lease_id": lease_id,
+ "attempt": attempt,
+ })
+ activity.heartbeat(heartbeat)
+ await asyncio.sleep(1)
+ except asyncio.CancelledError:
+ # The database commit may have completed even when cancellation wins
+ # the await race and the activity result never reaches workflow
+ # history. Always compensate the attempted permit locally. Releasing
+ # a permit that was never acquired is an expected no-op here.
+ try:
+ compensated = await service.release_subagent_permit(
+ root_id,
+ lease_id,
+ )
+ if not compensated:
+ raise RuntimeError("Permit compensation was rejected")
+ except Exception as release_exc: # noqa: BLE001
+ logger.error(
+ "Failed to compensate cancelled subagent permit acquisition",
+ extra={
+ "root_execution_id": root_id,
+ "permit_id": permit_id,
+ "lease_id": lease_id,
+ "error": str(release_exc),
+ },
+ )
+ raise
@activity.defn(name="agent.release_subagent_permit.v1")
@@ -1560,12 +1746,24 @@ async def release_subagent_permit(payload: Dict[str, Any]) -> Dict[str, Any]:
root_id = str(payload.get("root_execution_id") or "")
permit_id = str(payload.get("permit_id") or "")
- if not root_id or not permit_id:
+ lease_id = str(payload.get("lease_id") or permit_id)
+ lease_version = int(payload.get("lease_version") or (2 if payload.get("lease_id") else 1))
+ if not root_id or not permit_id or not lease_id:
raise ValueError("root_execution_id and permit_id are required")
- released = await get_agent_team_service().release_subagent_permit(root_id, permit_id)
+ released = await get_agent_team_service().release_subagent_permit(
+ root_id,
+ lease_id,
+ )
if not released:
raise RuntimeError("Failed to release subagent permit")
- return {"root_execution_id": root_id, "permit_id": permit_id, "released": True}
+ result = {
+ "root_execution_id": root_id,
+ "permit_id": permit_id,
+ "released": True,
+ }
+ if lease_version == 2:
+ result.update({"lease_version": 2, "lease_id": lease_id})
+ return result
@activity.defn(name="agent.finish_delegation.v1")
diff --git a/server/services/temporal/agent_workflow.py b/server/services/temporal/agent_workflow.py
index 7a73bb20..c8ed3744 100644
--- a/server/services/temporal/agent_workflow.py
+++ b/server/services/temporal/agent_workflow.py
@@ -52,7 +52,7 @@ from typing import Any, Dict, List, Optional
from temporalio import workflow
from temporalio.common import RetryPolicy # kept for type hints
-from temporalio.workflow import ParentClosePolicy
+from temporalio.workflow import ActivityCancellationType, ParentClosePolicy
from services.node_registry import get_node_class
@@ -105,12 +105,33 @@ def _default_max_iterations() -> int:
# non_retryable_error_types include ``NodeUserError`` — user-correctable
# failures inside the LLM step fail fast instead of burning 3 retries.
AGENT_ACTIVITY_RETRY: RetryPolicy = DEFAULT_ACTIVITY_RETRY
+DELEGATION_CLEANUP_RETRY: RetryPolicy = RetryPolicy(
+ initial_interval=timedelta(seconds=1),
+ maximum_interval=timedelta(seconds=30),
+ maximum_attempts=10,
+)
# Temporal patch marker for per-call command identity and duplicate-name
# validation. Existing histories must retain their recorded activity/child
# ids and last-wins tool index; new histories take the isolated path.
TOOL_CALL_IDENTITY_V2_PATCH = "agent-tool-call-identity-v2"
TASK_MANAGER_DELEGATION_PATCH = "agent-task-manager-delegation-v1"
+COOPERATIVE_PAUSE_SCHEDULING_PATCH = (
+ "agent-cooperative-pause-scheduling-v1"
+)
+DELEGATED_TASK_COOPERATIVE_PAUSE_SCHEDULING_PATCH = (
+ "delegated-task-cooperative-pause-scheduling-v1"
+)
+DELEGATION_PERMIT_CLEANUP_PATCH = "agent-delegation-permit-cleanup-v1"
+DELEGATION_ACQUIRE_CANCELLATION_PATCH = (
+ "agent-delegation-acquire-cancellation-v1"
+)
+DELEGATED_TASK_CANCELLATION_TERMINAL_PATCH = (
+ "delegated-task-cancellation-terminal-v1"
+)
+DELEGATED_TASK_SEARCH_ATTRIBUTES_PATCH = (
+ "agent-delegated-task-search-attributes-v1"
+)
DUPLICATE_TOOL_NAME_ERROR_TYPE = "DuplicateToolNameError"
@@ -298,10 +319,28 @@ class DelegatedTaskWorkflow:
@workflow.run
async def run(self, request: Dict[str, Any]) -> Dict[str, Any]:
+ use_cooperative_pause_schedule_gate = workflow.patched(
+ DELEGATED_TASK_COOPERATIVE_PAUSE_SCHEDULING_PATCH
+ )
+ use_acquire_cancellation_completion = workflow.patched(
+ DELEGATION_ACQUIRE_CANCELLATION_PATCH
+ )
+ use_cancellation_terminal = workflow.patched(
+ DELEGATED_TASK_CANCELLATION_TERMINAL_PATCH
+ )
+ acquire_cancellation_options = (
+ {
+ "cancellation_type": (
+ ActivityCancellationType.WAIT_CANCELLATION_COMPLETED
+ )
+ }
+ if use_acquire_cancellation_completion
+ else {}
+ )
lifecycle = request["lifecycle"]
task_id = lifecycle["team_task_id"]
root_id = lifecycle["root_execution_id"]
- acquired = False
+ acquired_permit_id: Optional[str] = None
began = False
try:
await self._wait_until_resumed()
@@ -313,23 +352,43 @@ class DelegatedTaskWorkflow:
"child_workflow_id": request["child_workflow_id"]}],
start_to_close_timeout=PERSIST_TURN_TIMEOUT,
retry_policy=AGENT_ACTIVITY_RETRY,
+ **acquire_cancellation_options,
)
- await workflow.execute_activity(
+ if use_cooperative_pause_schedule_gate:
+ await self._wait_until_resumed()
+ acquire_payload = {
+ "root_execution_id": root_id,
+ "permit_id": task_id,
+ "limit": request.get("limit", 3),
+ }
+ if use_acquire_cancellation_completion:
+ acquire_payload["lease_version"] = 2
+ acquire_result = await workflow.execute_activity(
"agent.acquire_subagent_permit.v1",
- args=[{"root_execution_id": root_id, "permit_id": task_id,
- "limit": request.get("limit", 3)}],
+ args=[acquire_payload],
start_to_close_timeout=timedelta(hours=1),
heartbeat_timeout=timedelta(seconds=10),
retry_policy=AGENT_ACTIVITY_RETRY,
+ **acquire_cancellation_options,
+ )
+ acquired_permit_id = str(
+ (acquire_result or {}).get("lease_id")
+ or task_id
)
- acquired = True
+ if use_cooperative_pause_schedule_gate:
+ await self._wait_until_resumed()
await workflow.execute_activity(
"agent.begin_delegation.v1", args=[lifecycle],
start_to_close_timeout=PERSIST_TURN_TIMEOUT,
retry_policy=AGENT_ACTIVITY_RETRY,
+ **acquire_cancellation_options,
)
began = True
await self._wait_until_resumed()
+ if use_acquire_cancellation_completion:
+ request["child_context"]["team_permit_id"] = (
+ acquired_permit_id
+ )
child_handle = await workflow.start_child_workflow(
"AgentWorkflow", args=[request["child_context"]],
id=request["child_workflow_id"],
@@ -355,6 +414,74 @@ class DelegatedTaskWorkflow:
start_to_close_timeout=PERSIST_TURN_TIMEOUT,
retry_policy=AGENT_ACTIVITY_RETRY,
)
+ except asyncio.CancelledError:
+ if use_cancellation_terminal:
+ if began:
+ try:
+ await workflow.execute_activity(
+ "agent.cancel_delegation.v1",
+ args=[{
+ **lifecycle,
+ "reason": (
+ "Delegated task workflow cancelled"
+ ),
+ "terminal_event_id": f"{task_id}:terminal",
+ }],
+ activity_id="cancel-delegation",
+ start_to_close_timeout=PERSIST_TURN_TIMEOUT,
+ retry_policy=DELEGATION_CLEANUP_RETRY,
+ **acquire_cancellation_options,
+ )
+ except Exception as cleanup_exc: # noqa: BLE001
+ workflow.logger.error(
+ "DelegatedTaskWorkflow failed to persist "
+ f"cancellation for {task_id}: {cleanup_exc}"
+ )
+ else:
+ # Queue/acquire/begin may have completed even when their
+ # result lost the cancellation race. The dedicated
+ # transition is idempotent for absent/already-terminal
+ # tasks and never invokes the normal failure/requeue path.
+ try:
+ await workflow.execute_activity(
+ "agent.cancel_delegation.v1",
+ args=[{
+ **lifecycle,
+ "reason": (
+ "Delegated task workflow cancelled"
+ ),
+ "terminal_event_id": f"{task_id}:terminal",
+ }],
+ activity_id="cancel-delegation-before-begin",
+ start_to_close_timeout=PERSIST_TURN_TIMEOUT,
+ retry_policy=DELEGATION_CLEANUP_RETRY,
+ **acquire_cancellation_options,
+ )
+ except Exception as cleanup_exc: # noqa: BLE001
+ workflow.logger.error(
+ "DelegatedTaskWorkflow failed to persist early "
+ f"cancellation for {task_id}: {cleanup_exc}"
+ )
+ if acquired_permit_id:
+ try:
+ await workflow.execute_activity(
+ "agent.release_subagent_permit.v1",
+ args=[{
+ "root_execution_id": root_id,
+ "permit_id": acquired_permit_id,
+ }],
+ activity_id="release-permit-cancelled",
+ start_to_close_timeout=PERSIST_TURN_TIMEOUT,
+ retry_policy=DELEGATION_CLEANUP_RETRY,
+ **acquire_cancellation_options,
+ )
+ acquired_permit_id = None
+ except Exception as cleanup_exc: # noqa: BLE001
+ workflow.logger.error(
+ "DelegatedTaskWorkflow failed to release "
+ f"cancelled permit {task_id}: {cleanup_exc}"
+ )
+ raise
except Exception as exc:
if began:
await workflow.execute_activity(
@@ -367,13 +494,33 @@ class DelegatedTaskWorkflow:
)
raise
finally:
- if acquired:
- await workflow.execute_activity(
- "agent.release_subagent_permit.v1",
- args=[{"root_execution_id": root_id, "permit_id": task_id}],
- start_to_close_timeout=PERSIST_TURN_TIMEOUT,
- retry_policy=AGENT_ACTIVITY_RETRY,
+ if acquired_permit_id:
+ release_options = (
+ {
+ "activity_id": "release-permit-final",
+ "retry_policy": DELEGATION_CLEANUP_RETRY,
+ }
+ if use_cancellation_terminal
+ else {"retry_policy": AGENT_ACTIVITY_RETRY}
)
+ try:
+ await workflow.execute_activity(
+ "agent.release_subagent_permit.v1",
+ args=[{
+ "root_execution_id": root_id,
+ "permit_id": acquired_permit_id,
+ }],
+ start_to_close_timeout=PERSIST_TURN_TIMEOUT,
+ **acquire_cancellation_options,
+ **release_options,
+ )
+ except Exception as release_exc:
+ if not use_cancellation_terminal:
+ raise
+ workflow.logger.error(
+ "DelegatedTaskWorkflow final permit release failed "
+ f"for {task_id}: {release_exc}"
+ )
@workflow.defn(sandboxed=False, name="AgentWorkflow")
@@ -477,6 +624,24 @@ class AgentWorkflow:
# deterministic duplicate-name validation.
use_tool_call_identity_v2 = workflow.patched(TOOL_CALL_IDENTITY_V2_PATCH)
use_task_manager_delegation = workflow.patched(TASK_MANAGER_DELEGATION_PATCH)
+ use_cooperative_pause_schedule_gate = workflow.patched(
+ COOPERATIVE_PAUSE_SCHEDULING_PATCH
+ )
+ use_delegation_permit_cleanup = workflow.patched(
+ DELEGATION_PERMIT_CLEANUP_PATCH
+ )
+ use_acquire_cancellation_completion = workflow.patched(
+ DELEGATION_ACQUIRE_CANCELLATION_PATCH
+ )
+ acquire_cancellation_options = (
+ {
+ "cancellation_type": (
+ ActivityCancellationType.WAIT_CANCELLATION_COMPLETED
+ )
+ }
+ if use_acquire_cancellation_completion
+ else {}
+ )
# ---- Step 0: Resolve payload via the prep activity --------------
# DB lookups + edge walking + tool schema build happen here, NOT
@@ -648,6 +813,8 @@ class AgentWorkflow:
max_iterations,
phase="llm_step",
)
+ if use_cooperative_pause_schedule_gate:
+ await self._wait_until_resumed()
# Strip per-turn fields the activity doesn't need. Native turns
# receive only provider-neutral JSON tool definitions. Legacy
@@ -862,6 +1029,11 @@ class AgentWorkflow:
break
# ---- Schedule tool activities -------------------------------
+ if use_cooperative_pause_schedule_gate:
+ # The LLM activity and phase broadcasts yield long enough for
+ # a pause signal to arrive after the iteration-level check.
+ # Admit the resulting tool/delegation command batch afresh.
+ await self._wait_until_resumed()
calls = step_result.get("calls") or []
workflow.logger.info(f"AgentWorkflow scheduling {len(calls)} tool call(s)")
@@ -891,13 +1063,120 @@ class AgentWorkflow:
]
delegation_handles: Dict[int, Any] = {}
delegation_permits: Dict[int, str] = {}
+ delegation_lifecycles: Dict[int, Dict[str, Any]] = {}
+ delegation_release_attempts: Dict[int, int] = {}
yielded_own_permit = False
+ async def _release_delegation_permit(call_index: int) -> None:
+ """Idempotently release one confirmed permit exactly once locally."""
+ permit_id = delegation_permits.get(call_index)
+ if not permit_id:
+ return
+ release_attempt = delegation_release_attempts.get(
+ call_index,
+ 0,
+ )
+ base_activity_id = (
+ f"release-permit-{iteration + 1}-{call_index + 1}"
+ )
+ release_activity_id = (
+ base_activity_id
+ if release_attempt == 0
+ else f"{base_activity_id}-recovery-{release_attempt}"
+ )
+ delegation_release_attempts[call_index] = (
+ release_attempt + 1
+ )
+ await workflow.execute_activity(
+ "agent.release_subagent_permit.v1",
+ args=[{
+ "root_execution_id": root_execution_id,
+ "permit_id": permit_id,
+ }],
+ activity_id=release_activity_id,
+ start_to_close_timeout=PERSIST_TURN_TIMEOUT,
+ retry_policy=(
+ DELEGATION_CLEANUP_RETRY
+ if use_delegation_permit_cleanup
+ else AGENT_ACTIVITY_RETRY
+ ),
+ )
+ delegation_permits.pop(call_index, None)
+
+ async def _release_delegation_permit_for_cleanup(
+ call_index: int,
+ ) -> None:
+ """Best-effort release that cannot replace an active failure."""
+ permit_id = delegation_permits.get(call_index)
+ try:
+ await _release_delegation_permit(call_index)
+ except Exception as release_exc: # noqa: BLE001
+ workflow.logger.error(
+ "AgentWorkflow failed to release delegation permit "
+ f"{permit_id or call_index}: {release_exc}"
+ )
+
+ async def _release_remaining_delegation_permits() -> None:
+ """Best-effort drain all confirmed permits in deterministic order."""
+ for pending_index in sorted(list(delegation_permits)):
+ await _release_delegation_permit_for_cleanup(
+ pending_index
+ )
+
+ async def _cancel_remaining_delegations() -> None:
+ """Terminalize every task whose runner this workflow owns."""
+ for pending_index in sorted(
+ list(delegation_lifecycles)
+ ):
+ lifecycle = delegation_lifecycles[pending_index]
+ try:
+ await workflow.execute_activity(
+ "agent.cancel_delegation.v1",
+ args=[{
+ **lifecycle,
+ "reason": (
+ "Parent agent workflow cancelled"
+ ),
+ "terminal_event_id": (
+ f"{lifecycle['team_task_id']}:terminal"
+ ),
+ }],
+ activity_id=(
+ "cancel-delegation-"
+ f"{iteration + 1}-{pending_index + 1}"
+ ),
+ start_to_close_timeout=PERSIST_TURN_TIMEOUT,
+ retry_policy=DELEGATION_CLEANUP_RETRY,
+ )
+ delegation_lifecycles.pop(
+ pending_index,
+ None,
+ )
+ except Exception as cancellation_exc: # noqa: BLE001
+ workflow.logger.error(
+ "AgentWorkflow failed to persist delegation "
+ f"cancellation for call {pending_index}: "
+ f"{cancellation_exc}"
+ )
+
+ async def _cleanup_cancelled_delegations() -> None:
+ # Terminal state and capacity are independent cleanup
+ # obligations. Always attempt both and preserve the original
+ # cancellation/error at the caller.
+ await _cancel_remaining_delegations()
+ await _release_remaining_delegation_permits()
+
# A child that waits for grandchildren must not retain a slot,
# otherwise N admitted children can all block forever trying to
# acquire the N+1th permit. Yield while orchestrating descendants
# and reacquire before resuming this agent's next LLM turn.
- own_permit_id = str(context.get("team_task_id") or "")
+ own_logical_permit_id = str(
+ context.get("team_task_id") or ""
+ )
+ own_permit_id = str(
+ context.get("team_permit_id")
+ or own_logical_permit_id
+ )
if delegation_call_indices and own_permit_id:
await workflow.execute_activity(
"agent.release_subagent_permit.v1",
@@ -979,6 +1258,11 @@ class AgentWorkflow:
f"{child_context['team_task_id']}:assigned"
),
}
+ delegation_lifecycles[call_index] = (
+ lifecycle_payload
+ )
+ if use_cooperative_pause_schedule_gate:
+ await self._wait_until_resumed()
await workflow.execute_activity(
"agent.queue_delegation.v1",
args=[{
@@ -992,26 +1276,42 @@ class AgentWorkflow:
),
start_to_close_timeout=PERSIST_TURN_TIMEOUT,
retry_policy=AGENT_ACTIVITY_RETRY,
+ **acquire_cancellation_options,
)
- await workflow.execute_activity(
+ if use_cooperative_pause_schedule_gate:
+ await self._wait_until_resumed()
+ acquire_payload = {
+ "root_execution_id": root_execution_id,
+ "permit_id": permit_id,
+ "limit": max_concurrent_subagents,
+ }
+ if use_acquire_cancellation_completion:
+ acquire_payload["lease_version"] = 2
+ acquire_result = await workflow.execute_activity(
"agent.acquire_subagent_permit.v1",
- args=[{
- "root_execution_id": root_execution_id,
- "permit_id": permit_id,
- "limit": max_concurrent_subagents,
- }],
+ args=[acquire_payload],
activity_id=(
f"acquire-permit-{iteration + 1}-{call_index + 1}"
),
start_to_close_timeout=timedelta(hours=1),
heartbeat_timeout=timedelta(seconds=10),
retry_policy=AGENT_ACTIVITY_RETRY,
+ **acquire_cancellation_options,
+ )
+ delegation_permits[call_index] = str(
+ (acquire_result or {}).get("lease_id")
+ or permit_id
)
- delegation_permits[call_index] = permit_id
+ if use_acquire_cancellation_completion:
+ child_context["team_permit_id"] = (
+ delegation_permits[call_index]
+ )
# Claim only after admission. Persistence failure still
# prevents child startup and the task remains pending
# while the coordinator queues it.
try:
+ if use_cooperative_pause_schedule_gate:
+ await self._wait_until_resumed()
await workflow.execute_activity(
"agent.begin_delegation.v1",
args=[lifecycle_payload],
@@ -1020,20 +1320,14 @@ class AgentWorkflow:
),
start_to_close_timeout=PERSIST_TURN_TIMEOUT,
retry_policy=AGENT_ACTIVITY_RETRY,
+ **acquire_cancellation_options,
)
+ except asyncio.CancelledError:
+ if not use_delegation_permit_cleanup:
+ await _release_delegation_permit(call_index)
+ raise
except BaseException:
- await workflow.execute_activity(
- "agent.release_subagent_permit.v1",
- args=[{
- "root_execution_id": root_execution_id,
- "permit_id": permit_id,
- }],
- activity_id=(
- f"release-permit-{iteration + 1}-{call_index + 1}"
- ),
- start_to_close_timeout=PERSIST_TURN_TIMEOUT,
- retry_policy=AGENT_ACTIVITY_RETRY,
- )
+ await _release_delegation_permit(call_index)
raise
child_id = (
_delegation_child_id_v2(
@@ -1046,6 +1340,8 @@ class AgentWorkflow:
else f"{workflow.info().workflow_id}-delegate-{candidate_tool['tool_node_id']}-{iteration}"
)
try:
+ if use_cooperative_pause_schedule_gate:
+ await self._wait_until_resumed()
delegation_handles[call_index] = await workflow.start_child_workflow(
"AgentWorkflow",
args=[child_context],
@@ -1063,21 +1359,19 @@ class AgentWorkflow:
activity_id=f"register-delegation-{iteration + 1}-{call_index + 1}",
start_to_close_timeout=PERSIST_TURN_TIMEOUT,
retry_policy=AGENT_ACTIVITY_RETRY,
+ **acquire_cancellation_options,
)
+ except asyncio.CancelledError:
+ if (
+ not use_delegation_permit_cleanup
+ and team_id
+ and call_index in delegation_permits
+ ):
+ await _release_delegation_permit(call_index)
+ raise
except BaseException:
if team_id and call_index in delegation_permits:
- await workflow.execute_activity(
- "agent.release_subagent_permit.v1",
- args=[{
- "root_execution_id": root_execution_id,
- "permit_id": delegation_permits[call_index],
- }],
- activity_id=(
- f"release-permit-{iteration + 1}-{call_index + 1}"
- ),
- start_to_close_timeout=PERSIST_TURN_TIMEOUT,
- retry_policy=AGENT_ACTIVITY_RETRY,
- )
+ await _release_delegation_permit(call_index)
raise
async def _run_task_manager_delegation(
@@ -1155,6 +1449,8 @@ class AgentWorkflow:
# queue_delegation is intentionally retained: it is
# idempotent for the pre-created task and records the same
# lifecycle event as direct delegation without duplicating it.
+ if use_cooperative_pause_schedule_gate:
+ await self._wait_until_resumed()
await workflow.execute_activity(
"agent.queue_delegation.v1",
args=[{**lifecycle, "queued_event_id": f"{task_id}:queued"}],
@@ -1179,13 +1475,34 @@ class AgentWorkflow:
workflow.info().workflow_id, assignee_id, iteration, call_index
) + f"-{task_id}"
runner_id = f"{child_id}-runner"
+ if use_cooperative_pause_schedule_gate:
+ await self._wait_until_resumed()
+ delegated_task_options: Dict[str, Any] = {
+ "id": runner_id,
+ "parent_close_policy": ParentClosePolicy.ABANDON,
+ "execution_timeout": timedelta(hours=2),
+ "run_timeout": timedelta(hours=2),
+ }
+ use_delegated_task_search_attributes = workflow.patched(
+ DELEGATED_TASK_SEARCH_ATTRIBUTES_PATCH
+ )
+ if use_delegated_task_search_attributes:
+ from services.temporal.trigger_listener_workflow import (
+ event_workflow_search_attributes,
+ )
+
+ search_attributes = event_workflow_search_attributes(
+ payload.get("workflow_id")
+ )
+ if search_attributes is not None:
+ delegated_task_options["search_attributes"] = (
+ search_attributes
+ )
await workflow.start_child_workflow(
"DelegatedTaskWorkflow",
args=[{"lifecycle": lifecycle, "child_context": child_context,
"child_workflow_id": child_id, "limit": max_concurrent_subagents}],
- id=runner_id,
- parent_close_policy=ParentClosePolicy.ABANDON,
- execution_timeout=timedelta(hours=2), run_timeout=timedelta(hours=2),
+ **delegated_task_options,
)
return {"status": "queued", "result": None, "runner_workflow_id": runner_id}
@@ -1233,6 +1550,13 @@ class AgentWorkflow:
**preflight_metadata,
}
task_manager_preflight_indices.append(preflight_index)
+ if (
+ not task_manager_preflight_handles
+ and use_cooperative_pause_schedule_gate
+ ):
+ # ``start_activity`` does not yield; one admission
+ # check protects this whole concurrent batch.
+ await self._wait_until_resumed()
task_manager_preflight_handles.append(
workflow.start_activity(
f"node.taskManager.v{preflight_tool['version']}",
@@ -1287,9 +1611,16 @@ class AgentWorkflow:
)
next_delegation_to_start = 0
- for delegation_index in delegation_call_indices[:max_concurrent_subagents]:
- await _start_delegation(delegation_index)
- next_delegation_to_start += 1
+ try:
+ for delegation_index in delegation_call_indices[
+ :max_concurrent_subagents
+ ]:
+ await _start_delegation(delegation_index)
+ next_delegation_to_start += 1
+ except BaseException:
+ if use_delegation_permit_cleanup:
+ await _cleanup_cancelled_delegations()
+ raise
for call_index, call in enumerate(calls):
if llm_engine == NATIVE_LLM_ENGINE and call.get("parse_error"):
@@ -1459,41 +1790,41 @@ class AgentWorkflow:
else f"node.{tool_info['node_type']}.v{tool_info['version']}"
)
- await self._emit_phase(
- agent_node_id,
- agent_workflow_id,
- iteration,
- max_iterations,
- phase="executing_tool",
- extra={
- "tool_name": call.get("name", ""),
- "tool_node_id": tool_info["tool_node_id"],
- "tool_call_id": str(
- call.get("id") or f"{iteration + 1}:{call_index + 1}"
- ),
- },
- )
+ try:
+ await self._emit_phase(
+ agent_node_id,
+ agent_workflow_id,
+ iteration,
+ max_iterations,
+ phase="executing_tool",
+ extra={
+ "tool_name": call.get("name", ""),
+ "tool_node_id": tool_info["tool_node_id"],
+ "tool_call_id": str(
+ call.get("id")
+ or f"{iteration + 1}:{call_index + 1}"
+ ),
+ },
+ )
+ except asyncio.CancelledError:
+ if use_delegation_permit_cleanup:
+ await _cleanup_cancelled_delegations()
+ raise
try:
if is_delegation and tool_info["node_type"] in AGENT_WORKFLOW_TYPES:
handle = delegation_handles[call_index]
try:
tool_result = await handle
- finally:
- permit_id = delegation_permits.get(call_index)
- if permit_id:
- await workflow.execute_activity(
- "agent.release_subagent_permit.v1",
- args=[{
- "root_execution_id": root_execution_id,
- "permit_id": permit_id,
- }],
- activity_id=(
- f"release-permit-{iteration + 1}-{call_index + 1}"
- ),
- start_to_close_timeout=PERSIST_TURN_TIMEOUT,
- retry_policy=AGENT_ACTIVITY_RETRY,
- )
+ except asyncio.CancelledError:
+ if not use_delegation_permit_cleanup:
+ await _release_delegation_permit(call_index)
+ raise
+ except BaseException:
+ await _release_delegation_permit(call_index)
+ raise
+ else:
+ await _release_delegation_permit(call_index)
child_succeeded = (
bool(tool_result.get("success", True))
if isinstance(tool_result, dict) else True
@@ -1538,6 +1869,10 @@ class AgentWorkflow:
start_to_close_timeout=PERSIST_TURN_TIMEOUT,
retry_policy=AGENT_ACTIVITY_RETRY,
)
+ delegation_lifecycles.pop(
+ call_index,
+ None,
+ )
tool_result = {
"success": child_succeeded,
"status": "submitted" if child_succeeded else "failed",
@@ -1565,6 +1900,8 @@ class AgentWorkflow:
if isinstance(tool_result, BaseException):
raise tool_result
else:
+ if use_cooperative_pause_schedule_gate:
+ await self._wait_until_resumed()
tool_result = await workflow.execute_activity(
tool_activity_name,
args=[tool_payload],
@@ -1639,6 +1976,8 @@ class AgentWorkflow:
"agent_node_type": payload.get("node_type") or context.get("node_type"),
**call_metadata,
}
+ if use_cooperative_pause_schedule_gate:
+ await self._wait_until_resumed()
refresh_result = await workflow.execute_activity(
"agent.refresh_tools.v1",
args=[refresh_payload],
@@ -1692,6 +2031,10 @@ class AgentWorkflow:
len(added_tools),
len(tools),
)
+ except asyncio.CancelledError:
+ if use_delegation_permit_cleanup:
+ await _cleanup_cancelled_delegations()
+ raise
except Exception as e: # noqa: BLE001 — Temporal handles retries
# After all retries exhausted, surface the error to
# the LLM (per user decision: LLM sees error and
@@ -1703,52 +2046,72 @@ class AgentWorkflow:
f"task-{root_execution_id}-{agent_node_id}-"
f"{iteration + 1}-{call_index + 1}"
)
- await workflow.execute_activity(
- "agent.finish_delegation.v1",
- args=[{
- "team_id": team_id,
- "team_task_id": task_id,
- "parent_agent_node_id": agent_node_id,
- "child_agent_node_id": tool_info["tool_node_id"],
- "child_agent_name": str(
- (tool_info.get("tool_info") or {}).get("label")
- or tool_info.get("node_type")
- or "agent"
+ try:
+ await workflow.execute_activity(
+ "agent.finish_delegation.v1",
+ args=[{
+ "team_id": team_id,
+ "team_task_id": task_id,
+ "parent_agent_node_id": agent_node_id,
+ "child_agent_node_id": tool_info["tool_node_id"],
+ "child_agent_name": str(
+ (tool_info.get("tool_info") or {}).get(
+ "label"
+ )
+ or tool_info.get("node_type")
+ or "agent"
+ ),
+ "workflow_id": payload.get("workflow_id"),
+ "parent_agent_workflow_id": (
+ workflow.info().workflow_id
+ ),
+ "root_execution_id": root_execution_id,
+ "trace_id": str(call.get("id", "") or ""),
+ "success": False,
+ "error": f"{type(e).__name__}: {e}",
+ "terminal_event_id": f"{task_id}:terminal",
+ }],
+ activity_id=(
+ "finish-delegation-"
+ f"{iteration + 1}-{call_index + 1}"
),
- "workflow_id": payload.get("workflow_id"),
- "parent_agent_workflow_id": workflow.info().workflow_id,
- "root_execution_id": root_execution_id,
- "trace_id": str(call.get("id", "") or ""),
- "success": False,
- "error": f"{type(e).__name__}: {e}",
- "terminal_event_id": f"{task_id}:terminal",
- }],
- activity_id=(
- f"finish-delegation-{iteration + 1}-{call_index + 1}"
- ),
- start_to_close_timeout=PERSIST_TURN_TIMEOUT,
- retry_policy=AGENT_ACTIVITY_RETRY,
- )
+ start_to_close_timeout=PERSIST_TURN_TIMEOUT,
+ retry_policy=AGENT_ACTIVITY_RETRY,
+ )
+ delegation_lifecycles.pop(
+ call_index,
+ None,
+ )
+ except asyncio.CancelledError:
+ if use_delegation_permit_cleanup:
+ await _cleanup_cancelled_delegations()
+ raise
tool_content = f'{{"error": "{type(e).__name__}: {e}"}}'
- await self._emit_phase(
- agent_node_id,
- agent_workflow_id,
- iteration,
- max_iterations,
- phase="tool_completed",
- extra={
- "tool_name": call.get("name", ""),
- "tool_node_id": tool_info["tool_node_id"],
- "tool_call_id": str(
- call.get("id") or f"{iteration + 1}:{call_index + 1}"
- ),
- # Do not put raw failures into public status
- # events. This safe flag is enough for the
- # broadcaster to retain ``tool <name>`` with a
- # failed capability state.
- "tool_failed": True,
- },
- )
+ try:
+ await self._emit_phase(
+ agent_node_id,
+ agent_workflow_id,
+ iteration,
+ max_iterations,
+ phase="tool_completed",
+ extra={
+ "tool_name": call.get("name", ""),
+ "tool_node_id": tool_info["tool_node_id"],
+ "tool_call_id": str(
+ call.get("id")
+ or f"{iteration + 1}:{call_index + 1}"
+ ),
+ # Do not put raw failures into public status
+ # events. This safe flag is enough for the
+ # broadcaster to retain ``tool <name>`` with a
+ # failed capability state.
+ "tool_failed": True,
+ },
+ )
+ except asyncio.CancelledError:
+ if use_delegation_permit_cleanup:
+ await _cleanup_cancelled_delegations()
+ raise
_append_tool_result_message(
messages,
@@ -1759,6 +2122,8 @@ class AgentWorkflow:
)
if yielded_own_permit:
+ if use_cooperative_pause_schedule_gate:
+ await self._wait_until_resumed()
await workflow.execute_activity(
"agent.acquire_subagent_permit.v1",
args=[{
@@ -1770,6 +2135,7 @@ class AgentWorkflow:
start_to_close_timeout=timedelta(hours=1),
heartbeat_timeout=timedelta(seconds=10),
retry_policy=AGENT_ACTIVITY_RETRY,
+ **acquire_cancellation_options,
)
# ---- Persist this turn (append-per-turn) -------------------
@@ -1828,6 +2194,8 @@ class AgentWorkflow:
"api_key": payload["api_key"],
"model": payload["model"],
}
+ if use_cooperative_pause_schedule_gate:
+ await self._wait_until_resumed()
compact_result = await workflow.execute_activity(
"agent.compact_memory.v1",
args=[compact_payload],
diff --git a/server/services/temporal/polling_trigger_workflow.py b/server/services/temporal/polling_trigger_workflow.py
index 64961c93..2fa18be2 100644
--- a/server/services/temporal/polling_trigger_workflow.py
+++ b/server/services/temporal/polling_trigger_workflow.py
@@ -57,6 +57,9 @@ _DEFAULT_POLL_INTERVAL_S = 60
# headroom for slow Gmail / Twitter responses without hanging forever.
# Workflow ``RetryPolicy`` (default) handles transient failures.
_ACTIVITY_TIMEOUT_MULT = 4
+COOPERATIVE_PAUSE_SCHEDULING_PATCH = (
+ "polling-trigger-cooperative-pause-scheduling-v1"
+)
@workflow.defn(name="PollingTriggerWorkflow", sandboxed=False)
@@ -84,6 +87,10 @@ class PollingTriggerWorkflow:
async def resume(self) -> None:
self._control_paused = False
+ async def _wait_until_resumed(self) -> None:
+ if self._control_paused:
+ await workflow.wait_condition(lambda: not self._control_paused)
+
@workflow.run
async def run(self, listener_data: Dict[str, Any]) -> Dict[str, Any]:
"""Poll loop body.
@@ -110,6 +117,14 @@ class PollingTriggerWorkflow:
node_type = listener_data["node_type"]
version = listener_data.get("version", 1)
activity_name = f"poll.{node_type}.v{version}"
+ try:
+ use_cooperative_pause_schedule_gate = workflow.patched(
+ COOPERATIVE_PAUSE_SCHEDULING_PATCH
+ )
+ except Exception: # direct unit invocation outside Temporal runtime
+ if workflow.in_workflow():
+ raise
+ use_cooperative_pause_schedule_gate = False
params = listener_data.get("filter_params", {}) or {}
poll_interval = int(params.get("poll_interval") or _DEFAULT_POLL_INTERVAL_S)
@@ -141,6 +156,8 @@ class PollingTriggerWorkflow:
}
else:
await workflow.sleep(timedelta(seconds=poll_interval))
+ if use_cooperative_pause_schedule_gate:
+ await self._wait_until_resumed()
cycle_payload = {
"node_id": listener_data["trigger_node_id"],
"params": params,
@@ -177,7 +194,28 @@ class PollingTriggerWorkflow:
continue
self._seen_event_ids.add(event_id)
try:
- await self._spawn_child_run(event, listener_data)
+ if use_cooperative_pause_schedule_gate:
+ await self._wait_until_resumed()
+ from services.temporal.trigger_listener_workflow import (
+ event_workflow_search_attributes,
+ )
+
+ await self._spawn_child_run(
+ event,
+ listener_data,
+ admission_check=(
+ self._wait_until_resumed
+ if use_cooperative_pause_schedule_gate
+ else None
+ ),
+ search_attributes=(
+ event_workflow_search_attributes(
+ listener_data.get("workflow_id")
+ )
+ if use_cooperative_pause_schedule_gate
+ else None
+ ),
+ )
except Exception as spawn_exc: # noqa: BLE001
# Per-event spawn failure logged; subsequent events
# still try. Same isolation contract as the push
@@ -187,6 +225,11 @@ class PollingTriggerWorkflow:
if self._processed_count >= _MAX_EVENTS_BEFORE_CONTINUE_AS_NEW:
workflow.logger.info(f"PollingTriggerWorkflow continue_as_new: processed={self._processed_count}")
+ if use_cooperative_pause_schedule_gate:
+ # A pause may arrive during the final waiting-status
+ # broadcast. Continue only after Resume so the next run
+ # cannot reset a still-active pause flag.
+ await self._wait_until_resumed()
# Carry seen_ids forward so the new run doesn't re-emit
# what's already been seen by the provider.
listener_data["seen_ids"] = list(seen_ids)
@@ -196,6 +239,8 @@ class PollingTriggerWorkflow:
self,
event: Dict[str, Any],
listener_data: Dict[str, Any],
+ admission_check=None,
+ search_attributes=None,
) -> None:
"""Start a child :class:`MachinaWorkflow` with the trigger
pre-executed against this event payload.
@@ -275,6 +320,21 @@ class PollingTriggerWorkflow:
event_type=listener_data.get("event_type", ""),
)
+ if admission_check is not None:
+ await admission_check()
+
+ child_options = {
+ "id": child_id,
+ "parent_close_policy": ParentClosePolicy.ABANDON,
+ "id_reuse_policy": (
+ WorkflowIDReusePolicy.ALLOW_DUPLICATE_FAILED_ONLY
+ ),
+ "execution_timeout": timedelta(hours=1),
+ "run_timeout": timedelta(hours=1),
+ }
+ if search_attributes is not None:
+ child_options["search_attributes"] = search_attributes
+
await workflow.start_child_workflow(
"MachinaWorkflow",
args=[
@@ -287,11 +347,7 @@ class PollingTriggerWorkflow:
"tenant_id": tenant_id,
}
],
- id=child_id,
- parent_close_policy=ParentClosePolicy.ABANDON,
- id_reuse_policy=WorkflowIDReusePolicy.ALLOW_DUPLICATE_FAILED_ONLY,
- execution_timeout=timedelta(hours=1),
- run_timeout=timedelta(hours=1),
+ **child_options,
)
workflow.logger.info(f"PollingTriggerWorkflow spawned child run: child_id={child_id} " f"event.id={event.get('id')}")
diff --git a/server/services/temporal/schedules.py b/server/services/temporal/schedules.py
index cdcedf4a..6ef40cf9 100644
--- a/server/services/temporal/schedules.py
+++ b/server/services/temporal/schedules.py
@@ -20,6 +20,7 @@ Refs:
from __future__ import annotations
+from dataclasses import replace
from datetime import timedelta
from typing import Any, Dict
@@ -32,6 +33,7 @@ from temporalio.client import (
ScheduleOverlapPolicy,
SchedulePolicy,
ScheduleSpec,
+ ScheduleUpdate,
)
from temporalio.common import (
SearchAttributeKey,
@@ -111,6 +113,22 @@ async def create_cron_schedule(
"""
schedule_id = cron_schedule_id(workflow_slug, trigger_label)
action_workflow_id = cron_action_workflow_id(workflow_slug, trigger_label)
+ execution_search_attributes = TypedSearchAttributes(
+ [
+ SearchAttributePair(
+ SearchAttributeKey.for_keyword("EventWorkflowId"),
+ workflow_id,
+ ),
+ SearchAttributePair(
+ SearchAttributeKey.for_keyword("TriggerNodeId"),
+ node_id,
+ ),
+ SearchAttributePair(
+ SearchAttributeKey.for_keyword("EventTriggerKind"),
+ "cron",
+ ),
+ ]
+ )
schedule = Schedule(
action=ScheduleActionStartWorkflow(
@@ -118,6 +136,7 @@ async def create_cron_schedule(
args=[listener_data],
id=action_workflow_id,
task_queue=task_queue,
+ typed_search_attributes=execution_search_attributes,
),
spec=ScheduleSpec(
cron_expressions=[cron_expression],
@@ -141,22 +160,7 @@ async def create_cron_schedule(
# ``search_attributes`` is a ``client.create_schedule`` kwarg, not
# a field on the ``Schedule`` dataclass itself (per the Temporal
# SDK API).
- schedule_search_attributes = TypedSearchAttributes(
- [
- SearchAttributePair(
- SearchAttributeKey.for_keyword("EventWorkflowId"),
- workflow_id,
- ),
- SearchAttributePair(
- SearchAttributeKey.for_keyword("TriggerNodeId"),
- node_id,
- ),
- SearchAttributePair(
- SearchAttributeKey.for_keyword("EventTriggerKind"),
- "cron",
- ),
- ]
- )
+ schedule_search_attributes = execution_search_attributes
try:
await client.create_schedule(
@@ -171,10 +175,51 @@ async def create_cron_schedule(
timezone=timezone,
)
except ScheduleAlreadyRunningError:
- # Idempotent re-deploy. The pre-existing Schedule keeps its
- # state; the caller's create-or-reuse contract is satisfied.
+ # Idempotent re-deploy must still refresh the frozen action args and
+ # execution Search Attributes. Otherwise a Schedule created before
+ # the lifecycle-control release would keep spawning untagged roots
+ # that Reset cannot discover. Preserve the server-owned paused state
+ # while replacing the deployment snapshot and metadata.
+ handle = client.get_schedule_handle(schedule_id)
+
+ def update_existing(update_input):
+ description = update_input.description
+ existing_attributes = {
+ pair.key.name: pair.value
+ for pair in description.typed_search_attributes
+ }
+ existing_owner = existing_attributes.get("EventWorkflowId")
+ if existing_owner is None:
+ # Compatibility with Schedules created before execution
+ # Search Attributes were copied onto the action. Only claim a
+ # legacy id when its frozen payload proves the same owner.
+ existing_args = getattr(
+ description.schedule.action,
+ "args",
+ (),
+ )
+ if (
+ existing_args
+ and isinstance(existing_args[0], dict)
+ ):
+ existing_owner = existing_args[0].get("workflow_id")
+ if str(existing_owner or "") != str(workflow_id):
+ raise RuntimeError(
+ "cron_schedule_ownership_conflict:"
+ f"{schedule_id}"
+ )
+ updated_schedule = replace(
+ schedule,
+ state=description.schedule.state,
+ )
+ return ScheduleUpdate(
+ schedule=updated_schedule,
+ search_attributes=schedule_search_attributes,
+ )
+
+ await handle.update(update_existing)
logger.info(
- "Temporal cron Schedule already exists (re-deploy idempotency)",
+ "Updated existing Temporal cron Schedule",
schedule_id=schedule_id,
)
@@ -184,19 +229,23 @@ async def create_cron_schedule(
async def delete_cron_schedules_for_deployment(
client: Client,
deployment_workflow_id: str,
+ *,
+ strict: bool = False,
) -> int:
"""Delete every cron Schedule for ``deployment_workflow_id``.
Visibility-equivalent sweep: ``client.list_schedules(query=...)``
is the registry, no local handle dict. Per-Schedule failures don't
- block the sweep (mirror of
- :meth:`DeploymentManager._cancel_canary_listeners` semantics).
+ block the sweep. With ``strict=True``, the sweep still visits every
+ target and then raises if any deletion failed, so Reset cannot publish
+ completion while a durable Schedule survives.
Returns count of Schedules deleted.
"""
query = f"EventWorkflowId='{deployment_workflow_id}' " f"AND EventTriggerKind='cron'"
deleted = 0
+ failures: list[Exception] = []
try:
# ``Client.list_schedules`` is ``async def`` (returns a coroutine
# that resolves to ``ScheduleAsyncIterator``), unlike
@@ -213,6 +262,7 @@ async def delete_cron_schedules_for_deployment(
await handle.delete()
deleted += 1
except Exception as exc: # noqa: BLE001
+ failures.append(exc)
logger.warning(
f"Failed to delete cron Schedule {sched_id!r}: {exc}",
deployment_workflow_id=deployment_workflow_id,
@@ -222,6 +272,13 @@ async def delete_cron_schedules_for_deployment(
f"Visibility query for cron Schedules failed: {exc} " f"(query={query!r})",
deployment_workflow_id=deployment_workflow_id,
)
+ if strict:
+ raise RuntimeError("cron_schedule_visibility_cleanup_failed") from exc
+
+ if strict and failures:
+ raise RuntimeError(
+ f"cron_schedule_cleanup_failed:{len(failures)}"
+ ) from failures[0]
if deleted:
logger.info(
@@ -233,11 +290,22 @@ async def delete_cron_schedules_for_deployment(
async def set_cron_schedules_paused(
- client: Client, deployment_workflow_id: str, *, paused: bool,
+ client: Client,
+ deployment_workflow_id: str,
+ *,
+ paused: bool,
+ strict: bool = False,
) -> int:
- """Pause or unpause every cron Schedule owned by one deployment."""
+ """Pause or unpause every cron Schedule owned by one deployment.
+
+ Strict lifecycle transitions still sweep every matching Schedule, then
+ fail if visibility or any individual mutation failed. This prevents the
+ database from publishing a stable Pause/Resume state while a producer is
+ known to be in the opposite state.
+ """
query = f"EventWorkflowId='{deployment_workflow_id}' AND EventTriggerKind='cron'"
changed = 0
+ failures: list[Exception] = []
try:
iterator = await client.list_schedules(query=query)
async for desc in iterator:
@@ -249,6 +317,7 @@ async def set_cron_schedules_paused(
await handle.unpause(note="OpenCompany workflow resumed")
changed += 1
except Exception as exc: # noqa: BLE001
+ failures.append(exc)
logger.warning(
f"Failed to {'pause' if paused else 'resume'} cron Schedule {desc.id!r}: {exc}",
deployment_workflow_id=deployment_workflow_id,
@@ -258,6 +327,14 @@ async def set_cron_schedules_paused(
f"Cron Schedule pause sweep failed: {exc}",
deployment_workflow_id=deployment_workflow_id,
)
+ if strict:
+ raise RuntimeError(
+ "cron_schedule_pause_visibility_failed"
+ ) from exc
+ if strict and failures:
+ raise RuntimeError(
+ f"cron_schedule_pause_failed:{len(failures)}"
+ ) from failures[0]
return changed
diff --git a/server/services/temporal/trigger_listener_workflow.py b/server/services/temporal/trigger_listener_workflow.py
index 277c41b9..00d2ee8b 100644
--- a/server/services/temporal/trigger_listener_workflow.py
+++ b/server/services/temporal/trigger_listener_workflow.py
@@ -34,7 +34,12 @@ from datetime import timedelta
from typing import Any, Dict, List, Optional, Set
from temporalio import workflow
-from temporalio.common import WorkflowIDReusePolicy
+from temporalio.common import (
+ SearchAttributeKey,
+ SearchAttributePair,
+ TypedSearchAttributes,
+ WorkflowIDReusePolicy,
+)
from temporalio.workflow import ParentClosePolicy
@@ -45,6 +50,26 @@ from temporalio.workflow import ParentClosePolicy
# so this caps at ~16K triggers between continueAsNew checkpoints.
# Per-event histogram is empirical; tune later.
_MAX_EVENTS_BEFORE_CONTINUE_AS_NEW = 16_000
+CHILD_SEARCH_ATTRIBUTES_PATCH = "trigger-child-search-attributes-v1"
+COOPERATIVE_PAUSE_SCHEDULING_PATCH = (
+ "trigger-cooperative-pause-scheduling-v1"
+)
+
+
+def event_workflow_search_attributes(
+ workflow_id: Optional[str],
+) -> Optional[TypedSearchAttributes]:
+ """Tag detached execution roots for generation cleanup visibility."""
+ if not workflow_id:
+ return None
+ return TypedSearchAttributes(
+ [
+ SearchAttributePair(
+ SearchAttributeKey.for_keyword("EventWorkflowId"),
+ str(workflow_id),
+ )
+ ]
+ )
@workflow.defn(name="TriggerListenerWorkflow", sandboxed=False)
@@ -71,6 +96,10 @@ class TriggerListenerWorkflow:
async def resume(self) -> None:
self._control_paused = False
+ async def _wait_until_resumed(self) -> None:
+ if self._control_paused:
+ await workflow.wait_condition(lambda: not self._control_paused)
+
@workflow.signal
async def on_event(self, event_payload: Dict[str, Any]) -> None:
"""Receive an event from ``services.events.dispatch.emit``.
@@ -116,6 +145,12 @@ class TriggerListenerWorkflow:
f"node={listener_data.get('trigger_node_id')} "
f"event_type={listener_data.get('event_type')}"
)
+ use_child_search_attributes = workflow.patched(
+ CHILD_SEARCH_ATTRIBUTES_PATCH
+ )
+ use_cooperative_pause_schedule_gate = workflow.patched(
+ COOPERATIVE_PAUSE_SCHEDULING_PATCH
+ )
while True:
await workflow.wait_condition(
@@ -126,7 +161,22 @@ class TriggerListenerWorkflow:
continue
try:
- await self._spawn_child_run(event, listener_data)
+ await self._spawn_child_run(
+ event,
+ listener_data,
+ admission_check=(
+ self._wait_until_resumed
+ if use_cooperative_pause_schedule_gate
+ else None
+ ),
+ search_attributes=(
+ event_workflow_search_attributes(
+ listener_data.get("workflow_id")
+ )
+ if use_child_search_attributes
+ else None
+ ),
+ )
except Exception as exc: # noqa: BLE001
# Per-event spawn failures don't kill the listener.
# Log and move on; the rejected event still counts as
@@ -138,6 +188,10 @@ class TriggerListenerWorkflow:
self._processed_count += 1
if self._processed_count >= _MAX_EVENTS_BEFORE_CONTINUE_AS_NEW:
workflow.logger.info(f"TriggerListener continue_as_new: processed={self._processed_count}")
+ if use_cooperative_pause_schedule_gate:
+ # Do not let continue-as-new reset a pause that landed
+ # during the final spawn/status broadcast.
+ await self._wait_until_resumed()
workflow.continue_as_new(args=[listener_data])
async def _spawn_child_run(
@@ -145,6 +199,7 @@ class TriggerListenerWorkflow:
event: Dict[str, Any],
listener_data: Dict[str, Any],
admission_check=None,
+ search_attributes: Optional[TypedSearchAttributes] = None,
) -> None:
"""Build the filtered downstream graph + start a child MachinaWorkflow.
@@ -230,6 +285,18 @@ class TriggerListenerWorkflow:
if admission_check is not None:
await admission_check()
+ child_options: Dict[str, Any] = {
+ "id": child_id,
+ "parent_close_policy": ParentClosePolicy.ABANDON,
+ "id_reuse_policy": (
+ WorkflowIDReusePolicy.ALLOW_DUPLICATE_FAILED_ONLY
+ ),
+ "execution_timeout": timedelta(hours=1),
+ "run_timeout": timedelta(hours=1),
+ }
+ if search_attributes is not None:
+ child_options["search_attributes"] = search_attributes
+
await workflow.start_child_workflow(
"MachinaWorkflow",
args=[
@@ -245,11 +312,7 @@ class TriggerListenerWorkflow:
"data_scope_id": listener_data.get("data_scope_id"),
}
],
- id=child_id,
- parent_close_policy=ParentClosePolicy.ABANDON,
- id_reuse_policy=WorkflowIDReusePolicy.ALLOW_DUPLICATE_FAILED_ONLY,
- execution_timeout=timedelta(hours=1),
- run_timeout=timedelta(hours=1),
+ **child_options,
)
workflow.logger.info(
diff --git a/server/services/temporal/workflow.py b/server/services/temporal/workflow.py
index b759c525..e695a301 100644
--- a/server/services/temporal/workflow.py
+++ b/server/services/temporal/workflow.py
@@ -92,6 +92,9 @@ AGENT_WORKFLOW_TYPES = frozenset(
# namespace's replay/reset window, then deprecate the marker in a later
# release before removing that branch.
AGENT_CHILD_ID_V2_PATCH = "machina-agent-child-id-v2"
+COOPERATIVE_PAUSE_SCHEDULING_PATCH = (
+ "machina-cooperative-pause-scheduling-v1"
+)
def _agent_child_workflow_id_v2(root_workflow_id: str, agent_node_id: str) -> str:
@@ -141,6 +144,10 @@ class MachinaWorkflow:
async def resume(self) -> None:
self._control_paused = False
+ async def _wait_until_resumed(self) -> None:
+ if self._control_paused:
+ await workflow.wait_condition(lambda: not self._control_paused)
+
@workflow.signal
async def on_event(self, event_payload: Dict[str, Any]) -> None:
"""Wave 12 A7: receive an event from ``services.events.dispatch.emit``.
@@ -249,6 +256,9 @@ class MachinaWorkflow:
# new histories use the root Temporal id + exact canvas node id so
# two same-label agents can start in the same graph execution.
use_agent_child_id_v2 = workflow.patched(AGENT_CHILD_ID_V2_PATCH)
+ use_cooperative_pause_schedule_gate = workflow.patched(
+ COOPERATIVE_PAUSE_SCHEDULING_PATCH
+ )
workflow.logger.info(f"Starting workflow orchestration: {len(nodes)} nodes, {len(edges)} edges")
@@ -305,6 +315,8 @@ class MachinaWorkflow:
# ``_trigger_output`` is ``{not_triggered: True}``, no
# downstream template should resolve against them.
if not trigger_output.get("not_triggered"):
+ if use_cooperative_pause_schedule_gate:
+ await self._wait_until_resumed()
await workflow.execute_activity(
"store_node_output_activity",
{
@@ -387,6 +399,11 @@ class MachinaWorkflow:
# flag is on. Tool calls inside the agent loop become
# per-type activities (F4.A path) automatically.
dispatch = self._resolve_dispatch(node_type)
+ if use_cooperative_pause_schedule_gate:
+ # A preceding child start yields to the workflow event
+ # loop, so a pause signal may have landed since ``ready``
+ # was computed. Re-admit every command in the batch.
+ await self._wait_until_resumed()
if dispatch["kind"] == "child_workflow":
# ``workflow.start_child_workflow`` is ``async def`` —
# awaiting it returns the ``ChildWorkflowHandle`` once
diff --git a/server/services/temporal/workflow_control_workflow.py b/server/services/temporal/workflow_control_workflow.py
index 77d6d424..34f218dc 100644
--- a/server/services/temporal/workflow_control_workflow.py
+++ b/server/services/temporal/workflow_control_workflow.py
@@ -5,10 +5,16 @@ from datetime import timedelta
from typing import Any, Dict
from temporalio import workflow
+from temporalio.exceptions import ApplicationError
from services.temporal._retry_policies import DEFAULT_ACTIVITY_RETRY
+COOPERATIVE_PAUSE_SCHEDULING_PATCH = (
+ "workflow-control-cooperative-pause-scheduling-v1"
+)
+
+
@workflow.defn(name="WorkflowControlWorkflow", sandboxed=False)
class WorkflowControlWorkflow:
"""Own control state and trigger scheduling without listener workflows.
@@ -26,17 +32,47 @@ class WorkflowControlWorkflow:
self._events: list[tuple[str, Dict[str, Any]]] = []
self._seen_event_ids: set[str] = set()
self._poll_tasks: dict[str, asyncio.Task] = {}
+ self._use_cooperative_pause_schedule_gate = False
@workflow.signal
async def pause(self) -> None:
- if self._state == "running":
- self._state = "paused"
- self._revision += 1
+ self._apply_control_state("paused")
@workflow.signal
async def resume(self) -> None:
- if self._state == "paused":
- self._state = "running"
+ self._apply_control_state("running")
+
+ @workflow.update
+ async def set_control_state(self, requested_state: str) -> Dict[str, Any]:
+ """Idempotently apply and acknowledge a pause/resume transition.
+
+ Updates give callers a durable acknowledgement that the controller
+ processed the request. The legacy signals above remain registered for
+ older callers and histories.
+ """
+ normalized = str(requested_state).strip().lower()
+ aliases = {
+ "pause": "paused",
+ "paused": "paused",
+ "resume": "running",
+ "running": "running",
+ }
+ target_state = aliases.get(normalized)
+ if target_state is None:
+ raise ApplicationError(
+ "Control state must be one of: pause, paused, resume, running",
+ type="InvalidWorkflowControlState",
+ non_retryable=True,
+ )
+ self._apply_control_state(target_state)
+ return self.status()
+
+ def _apply_control_state(self, target_state: str) -> None:
+ """Apply one valid live-state transition without double revision."""
+ if self._state not in {"running", "paused"}:
+ return
+ if self._state != target_state:
+ self._state = target_state
self._revision += 1
@workflow.signal
@@ -82,6 +118,14 @@ class WorkflowControlWorkflow:
@workflow.run
async def run(self, control_data: Dict[str, Any]) -> Dict[str, Any]:
+ try:
+ self._use_cooperative_pause_schedule_gate = workflow.patched(
+ COOPERATIVE_PAUSE_SCHEDULING_PATCH
+ )
+ except Exception: # direct unit invocation outside Temporal runtime
+ if workflow.in_workflow():
+ raise
+ self._use_cooperative_pause_schedule_gate = False
self._state = control_data.get("state", "running")
while not self._closed:
await workflow.wait_condition(
@@ -96,11 +140,27 @@ class WorkflowControlWorkflow:
return {"state": self._state, "generation": control_data.get("generation")}
async def _spawn_push_run(self, event: Dict[str, Any], spec: Dict[str, Any]) -> None:
- from services.temporal.trigger_listener_workflow import TriggerListenerWorkflow
+ from services.temporal.trigger_listener_workflow import (
+ TriggerListenerWorkflow,
+ event_workflow_search_attributes,
+ )
listener = TriggerListenerWorkflow()
+ listener_args = spec["listener_args"]
await listener._spawn_child_run(
- event, spec["listener_args"], admission_check=self._wait_until_running,
+ event,
+ listener_args,
+ # This admission check predates the patch marker and must remain
+ # unconditional for replay compatibility with existing controller
+ # histories. The marker gates only the new child metadata.
+ admission_check=self._wait_until_running,
+ search_attributes=(
+ event_workflow_search_attributes(
+ listener_args.get("workflow_id")
+ )
+ if self._use_cooperative_pause_schedule_gate
+ else None
+ ),
)
async def _wait_until_running(self) -> None:
@@ -115,6 +175,9 @@ class WorkflowControlWorkflow:
_ACTIVITY_TIMEOUT_MULT,
_DEFAULT_POLL_INTERVAL_S,
)
+ from services.temporal.trigger_listener_workflow import (
+ event_workflow_search_attributes,
+ )
listener_data = spec["listener_args"]
node_type = listener_data["node_type"]
@@ -163,7 +226,25 @@ class WorkflowControlWorkflow:
await workflow.wait_condition(lambda: self._closed or self._state == "running")
if self._closed:
return
- await runner._spawn_child_run(event, listener_data)
+ await runner._spawn_child_run(
+ event,
+ listener_data,
+ admission_check=(
+ self._wait_until_running
+ if self._use_cooperative_pause_schedule_gate
+ else None
+ ),
+ search_attributes=(
+ event_workflow_search_attributes(
+ listener_data.get("workflow_id")
+ )
+ if self._use_cooperative_pause_schedule_gate
+ else None
+ ),
+ )
-__all__ = ["WorkflowControlWorkflow"]
+__all__ = [
+ "COOPERATIVE_PAUSE_SCHEDULING_PATCH",
+ "WorkflowControlWorkflow",
+]
diff --git a/server/tests/services/test_workflow_control.py b/server/tests/services/test_workflow_control.py
index 5881cb0b..45c39a9e 100644
--- a/server/tests/services/test_workflow_control.py
+++ b/server/tests/services/test_workflow_control.py
@@ -51,6 +51,7 @@ async def test_begin_generation_is_idempotent():
assert created is True
assert control.generation == 1
+ assert control.revision == 0
assert control.status == "starting"
assert control.data_scope_id == control.execution_id
assert serialize_control(control)["data_scope_id"] == control.execution_id
@@ -202,3 +203,4 @@ async def test_reset_generation_is_ready_for_explicit_start():
)
assert created is True
assert control.generation == 4
+ assert control.revision == 10
diff --git a/server/tests/services/test_workflow_control_handlers.py b/server/tests/services/test_workflow_control_handlers.py
new file mode 100644
index 00000000..a5830312
--- /dev/null
+++ b/server/tests/services/test_workflow_control_handlers.py
@@ -0,0 +1,1142 @@
+"""Control-handler contract tests for acknowledged Temporal lifecycle changes."""
+
+from types import SimpleNamespace
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+
+from services.deployment import handlers
+
+
+def _control(
+ status: str,
+ revision: int,
+ *,
+ terminal_reason: str | None = None,
+):
+ return SimpleNamespace(
+ id="workflow-control:wf:1",
+ workflow_id="wf",
+ generation=1,
+ execution_id="execution-1",
+ root_execution_id="execution-1",
+ data_scope_id="execution-1",
+ controller_workflow_id="workflow-control-wf-g1",
+ controller_run_id="controller-run-1",
+ status=status,
+ revision=revision,
+ created_at=None,
+ updated_at=None,
+ terminal_reason=terminal_reason,
+ )
+
+
+async def test_start_waits_for_deployment_setup_before_publishing_running(monkeypatch):
+ starting = _control("starting", 0)
+ starting_with_run = _control("starting", 1)
+ running = _control("running", 2)
+ order: list[str] = []
+
+ async def transition(_control_value, **kwargs):
+ order.append(f"db:{kwargs['status']}")
+ return (
+ starting_with_run
+ if kwargs["status"] == "starting"
+ else running
+ )
+
+ service = SimpleNamespace(
+ database=SimpleNamespace(
+ get_workflow_control_by_idempotency_key=AsyncMock(return_value=None),
+ get_latest_workflow_control=AsyncMock(return_value=None),
+ update_workflow_run_data_scope=AsyncMock(return_value=True),
+ ),
+ begin_generation=AsyncMock(return_value=(starting, True)),
+ transition=AsyncMock(side_effect=transition),
+ )
+
+ async def broadcast(control, **_kwargs):
+ order.append(f"broadcast:{control.status}:{control.revision}")
+ return {"state": control.status, "revision": control.revision}
+
+ async def deployment_setup(_workflow_id):
+ order.append("deployment:ready")
+ return {"success": True}
+
+ monkeypatch.setattr(handlers, "_control_service", lambda: service)
+ monkeypatch.setattr(handlers, "_broadcast_control", broadcast)
+ monkeypatch.setattr(
+ handlers,
+ "_start_controller",
+ AsyncMock(return_value="controller-run-1"),
+ )
+ monkeypatch.setattr(
+ handlers,
+ "handle_deploy_workflow",
+ AsyncMock(return_value={"success": True}),
+ )
+ monkeypatch.setattr(handlers, "_await_deployment_setup", deployment_setup)
+ monkeypatch.setattr(
+ handlers,
+ "_query_controller_state",
+ AsyncMock(return_value={"state": "running", "revision": 0}),
+ )
+
+ result = await handlers.handle_start_workflow(
+ {
+ "workflow_id": "wf",
+ "nodes": [{"id": "start-1", "type": "start", "data": {}}],
+ "edges": [],
+ "expected_revision": 0,
+ "idempotency_key": "start-request-1",
+ },
+ None,
+ )
+
+ assert result["success"] is True
+ assert result["state"] == "running"
+ assert order == [
+ "broadcast:starting:0",
+ "db:starting",
+ "broadcast:starting:1",
+ "deployment:ready",
+ "db:running",
+ "broadcast:running:2",
+ ]
+
+
+async def test_start_post_commit_broadcast_failure_does_not_tear_down_running(
+ monkeypatch,
+):
+ starting = _control("starting", 0)
+ starting_with_run = _control("starting", 1)
+ running = _control("running", 2)
+
+ async def transition(_control_value, **kwargs):
+ if kwargs["status"] == "starting":
+ return starting_with_run
+ return running
+
+ async def broadcast(control, **_kwargs):
+ if control.status == "running":
+ raise RuntimeError("status broadcast unavailable")
+ return {"state": control.status, "revision": control.revision}
+
+ service = SimpleNamespace(
+ database=SimpleNamespace(
+ get_workflow_control_by_idempotency_key=AsyncMock(
+ return_value=None,
+ ),
+ get_latest_workflow_control=AsyncMock(return_value=None),
+ update_workflow_run_data_scope=AsyncMock(return_value=True),
+ ),
+ begin_generation=AsyncMock(return_value=(starting, True)),
+ transition=AsyncMock(side_effect=transition),
+ fail=AsyncMock(),
+ )
+ signal_controller = AsyncMock()
+ terminate_generation = AsyncMock()
+
+ monkeypatch.setattr(handlers, "_control_service", lambda: service)
+ monkeypatch.setattr(handlers, "_broadcast_control", broadcast)
+ monkeypatch.setattr(
+ handlers,
+ "_start_controller",
+ AsyncMock(return_value="controller-run-1"),
+ )
+ monkeypatch.setattr(
+ handlers,
+ "handle_deploy_workflow",
+ AsyncMock(return_value={"success": True}),
+ )
+ monkeypatch.setattr(
+ handlers,
+ "_await_deployment_setup",
+ AsyncMock(return_value={"success": True}),
+ )
+ monkeypatch.setattr(
+ handlers,
+ "_query_controller_state",
+ AsyncMock(return_value={"state": "running", "revision": 0}),
+ )
+ monkeypatch.setattr(handlers, "_signal_controller", signal_controller)
+ monkeypatch.setattr(
+ handlers,
+ "_terminate_generation_workflows",
+ terminate_generation,
+ )
+
+ result = await handlers.handle_start_workflow(
+ {
+ "workflow_id": "wf",
+ "nodes": [{"id": "start-1", "type": "start", "data": {}}],
+ "edges": [],
+ "expected_revision": 0,
+ "idempotency_key": "start-request-1",
+ },
+ None,
+ )
+
+ assert result == {
+ "success": False,
+ "error": "status broadcast unavailable",
+ }
+ assert service.transition.await_count == 2
+ assert service.transition.await_args.kwargs["status"] == "running"
+ service.fail.assert_not_awaited()
+ signal_controller.assert_not_awaited()
+ terminate_generation.assert_not_awaited()
+
+
+ ("control", "expected_error"),
+ [
+ (_control("starting", 1), "workflow_start_pending"),
+ (
+ _control(
+ "failed",
+ 2,
+ terminal_reason="deployment_setup_failed",
+ ),
+ "deployment_setup_failed",
+ ),
+ ],
+)
+async def test_duplicate_start_reports_non_successful_durable_outcome(
+ monkeypatch,
+ control,
+ expected_error,
+):
+ service = SimpleNamespace(
+ database=SimpleNamespace(
+ get_workflow_control_by_idempotency_key=AsyncMock(
+ return_value=control,
+ ),
+ ),
+ )
+ monkeypatch.setattr(handlers, "_control_service", lambda: service)
+ monkeypatch.setattr(
+ handlers,
+ "_reconcile_control",
+ AsyncMock(return_value=(control, None)),
+ )
+ monkeypatch.setattr(
+ handlers,
+ "_control_payload",
+ AsyncMock(
+ return_value={
+ "state": control.status,
+ "revision": control.revision,
+ }
+ ),
+ )
+
+ result = await handlers.handle_start_workflow(
+ {
+ "workflow_id": "wf",
+ "idempotency_key": "start-request-1",
+ },
+ None,
+ )
+
+ assert result == {
+ "success": False,
+ "error": expected_error,
+ "idempotent": True,
+ "state": control.status,
+ "revision": control.revision,
+ }
+
+
+async def test_concurrent_duplicate_start_reports_pending_winner(monkeypatch):
+ starting = _control("starting", 1)
+ service = SimpleNamespace(
+ database=SimpleNamespace(
+ get_workflow_control_by_idempotency_key=AsyncMock(
+ return_value=None,
+ ),
+ get_latest_workflow_control=AsyncMock(return_value=None),
+ ),
+ begin_generation=AsyncMock(return_value=(starting, False)),
+ )
+ start_controller = AsyncMock()
+ monkeypatch.setattr(handlers, "_control_service", lambda: service)
+ monkeypatch.setattr(
+ handlers,
+ "_reconcile_control",
+ AsyncMock(return_value=(starting, None)),
+ )
+ monkeypatch.setattr(
+ handlers,
+ "_control_payload",
+ AsyncMock(return_value={"state": "starting", "revision": 1}),
+ )
+ monkeypatch.setattr(handlers, "_start_controller", start_controller)
+
+ result = await handlers.handle_start_workflow(
+ {
+ "workflow_id": "wf",
+ "expected_revision": 0,
+ "idempotency_key": "start-request-1",
+ },
+ None,
+ )
+
+ assert result["success"] is False
+ assert result["error"] == "workflow_start_pending"
+ assert result["idempotent"] is True
+ start_controller.assert_not_awaited()
+
+
+async def test_controller_state_change_waits_for_acknowledged_update(monkeypatch):
+ handle = SimpleNamespace(
+ execute_update=AsyncMock(return_value={
+ "state": "paused",
+ "revision": 3,
+ "queued_events": 2,
+ }),
+ )
+ monkeypatch.setattr(handlers, "_controller_handle", lambda _control: handle)
+
+ result = await handlers._update_controller_state(
+ _control("pausing", 2),
+ "paused",
+ update_id="request-1:paused",
+ )
+
+ assert result["state"] == "paused"
+ handle.execute_update.assert_awaited_once_with(
+ "set_control_state",
+ "paused",
+ id="request-1:paused",
+ )
+
+
+async def test_reconcile_completes_interrupted_pause_from_temporal(monkeypatch):
+ from core import container as container_module
+
+ pausing = _control("pausing", 3)
+ paused = _control("paused", 4)
+ service = SimpleNamespace(
+ transition=AsyncMock(return_value=paused),
+ database=SimpleNamespace(get_latest_workflow_control=AsyncMock()),
+ )
+ update = AsyncMock(return_value={
+ "state": "paused",
+ "revision": 1,
+ "queued_events": 0,
+ })
+ broadcast = AsyncMock(return_value={})
+ workflow_service = SimpleNamespace(
+ pause_deployment=MagicMock(),
+ update_trigger_pause_status=AsyncMock(return_value=1),
+ )
+ monkeypatch.setattr(
+ handlers,
+ "_query_controller_state",
+ AsyncMock(return_value={"state": "running", "revision": 0}),
+ )
+ monkeypatch.setattr(handlers, "_update_controller_state", update)
+ monkeypatch.setattr(handlers, "_broadcast_control", broadcast)
+ monkeypatch.setattr(handlers, "_set_cron_pause", AsyncMock(return_value=1))
+ monkeypatch.setattr(
+ handlers,
+ "_signal_generation_workflows",
+ AsyncMock(return_value=2),
+ )
+ monkeypatch.setattr(
+ container_module.container,
+ "workflow_service",
+ lambda: workflow_service,
+ )
+
+ reconciled, temporal_status = await handlers._reconcile_control(
+ service,
+ pausing,
+ )
+
+ assert reconciled is paused
+ assert temporal_status["state"] == "paused"
+ update.assert_awaited_once_with(
+ pausing,
+ "paused",
+ update_id="reconcile:workflow-control:wf:1:3:paused",
+ )
+ service.transition.assert_awaited_once_with(
+ pausing,
+ expected_revision=3,
+ from_statuses={"pausing"},
+ status="paused",
+ )
+ broadcast.assert_awaited_once_with(
+ paused,
+ controller_status=temporal_status,
+ extra={
+ "signalled_executions": 2,
+ "paused_schedules": 1,
+ "paused_triggers": 1,
+ },
+ )
+
+
+async def test_pause_response_is_final_only_after_temporal_ack(monkeypatch):
+ from core import container as container_module
+
+ running = _control("running", 2)
+ pausing = _control("pausing", 3)
+ paused = _control("paused", 4)
+ order: list[str] = []
+ update_calls: list[tuple[object, str, str]] = []
+
+ async def transition(control, **kwargs):
+ order.append(f"db:{kwargs['status']}")
+ return pausing if kwargs["status"] == "pausing" else paused
+
+ service = SimpleNamespace(
+ database=SimpleNamespace(
+ get_latest_workflow_control=AsyncMock(return_value=running),
+ ),
+ transition=AsyncMock(side_effect=transition),
+ )
+ workflow_service = SimpleNamespace(
+ pause_deployment=MagicMock(side_effect=lambda _workflow_id: order.append("local:pause")),
+ resume_deployment=AsyncMock(),
+ update_trigger_pause_status=AsyncMock(return_value=1),
+ get_deployment_status=MagicMock(return_value={
+ "active_runs": 0,
+ "queued_events": 0,
+ }),
+ )
+
+ async def update_controller(control, requested_state, *, update_id):
+ order.append("temporal:ack")
+ update_calls.append((control, requested_state, update_id))
+ return {"state": "paused", "revision": 1, "queued_events": 0}
+
+ async def broadcast(control, **_kwargs):
+ order.append(f"broadcast:{control.status}")
+ return {
+ "state": control.status,
+ "revision": control.revision,
+ "can_pause": control.status == "running",
+ "can_resume": control.status == "paused",
+ }
+
+ monkeypatch.setattr(handlers, "_control_service", lambda: service)
+ monkeypatch.setattr(
+ handlers,
+ "_reconcile_control",
+ AsyncMock(return_value=(running, {"state": "running", "revision": 0})),
+ )
+ monkeypatch.setattr(handlers, "_update_controller_state", update_controller)
+ monkeypatch.setattr(handlers, "_broadcast_control", broadcast)
+ monkeypatch.setattr(handlers, "_set_cron_pause", AsyncMock(return_value=1))
+ monkeypatch.setattr(
+ handlers,
+ "_signal_generation_workflows",
+ AsyncMock(return_value=2),
+ )
+ monkeypatch.setattr(
+ container_module.container,
+ "workflow_service",
+ lambda: workflow_service,
+ )
+
+ result = await handlers.handle_pause_workflow(
+ {
+ "workflow_id": "wf",
+ "expected_revision": 2,
+ "idempotency_key": "request-1",
+ },
+ None,
+ )
+
+ assert result["success"] is True
+ assert result["state"] == "paused"
+ assert order == [
+ "db:pausing",
+ "broadcast:pausing",
+ "local:pause",
+ "temporal:ack",
+ "db:paused",
+ "broadcast:paused",
+ ]
+ assert update_calls == [
+ (
+ pausing,
+ "paused",
+ "workflow-control:wf:1:3:request-1:paused",
+ )
+ ]
+
+
+ (
+ "operation",
+ "initial_status",
+ "transitional_status",
+ "requested_state",
+ "signal_name",
+ "cron_paused",
+ ),
+ [
+ ("pause", "running", "pausing", "paused", "pause", True),
+ ("resume", "paused", "resuming", "running", "resume", False),
+ ],
+)
+async def test_stable_state_is_not_published_when_strict_fanout_fails(
+ monkeypatch,
+ operation,
+ initial_status,
+ transitional_status,
+ requested_state,
+ signal_name,
+ cron_paused,
+):
+ from core import container as container_module
+
+ initial = _control(initial_status, 2)
+ transitional = _control(transitional_status, 3)
+ service = SimpleNamespace(
+ database=SimpleNamespace(
+ get_latest_workflow_control=AsyncMock(return_value=initial),
+ ),
+ transition=AsyncMock(return_value=transitional),
+ )
+ workflow_service = SimpleNamespace(
+ pause_deployment=MagicMock(),
+ resume_deployment=AsyncMock(return_value=0),
+ update_trigger_pause_status=AsyncMock(return_value=1),
+ )
+ set_cron_pause = AsyncMock(return_value=1)
+ fanout = AsyncMock(
+ side_effect=RuntimeError("workflow_signal_failed:1"),
+ )
+
+ monkeypatch.setattr(handlers, "_control_service", lambda: service)
+ monkeypatch.setattr(
+ handlers,
+ "_reconcile_control",
+ AsyncMock(
+ return_value=(
+ initial,
+ {"state": initial_status, "revision": 0},
+ )
+ ),
+ )
+ monkeypatch.setattr(
+ handlers,
+ "_update_controller_state",
+ AsyncMock(
+ return_value={
+ "state": requested_state,
+ "revision": 1,
+ "queued_events": 0,
+ }
+ ),
+ )
+ monkeypatch.setattr(
+ handlers,
+ "_broadcast_control",
+ AsyncMock(return_value={}),
+ )
+ monkeypatch.setattr(handlers, "_set_cron_pause", set_cron_pause)
+ monkeypatch.setattr(
+ handlers,
+ "_signal_generation_workflows",
+ fanout,
+ )
+ monkeypatch.setattr(
+ container_module.container,
+ "workflow_service",
+ lambda: workflow_service,
+ )
+
+ handler = (
+ handlers.handle_pause_workflow
+ if operation == "pause"
+ else handlers.handle_resume_workflow
+ )
+ result = await handler(
+ {
+ "workflow_id": "wf",
+ "expected_revision": 2,
+ "idempotency_key": "request-1",
+ },
+ None,
+ )
+
+ assert result == {
+ "success": False,
+ "error": "workflow_signal_failed:1",
+ }
+ service.transition.assert_awaited_once_with(
+ initial,
+ expected_revision=2,
+ from_statuses={initial_status},
+ status=transitional_status,
+ )
+ set_cron_pause.assert_awaited_once_with(
+ "wf",
+ paused=cron_paused,
+ strict=True,
+ )
+ fanout.assert_awaited_once_with(
+ transitional,
+ signal_name,
+ strict=True,
+ )
+
+
+async def test_pause_temporal_failure_restores_running_projection(monkeypatch):
+ from core import container as container_module
+
+ running = _control("running", 2)
+ pausing = _control("pausing", 3)
+ service = SimpleNamespace(
+ database=SimpleNamespace(
+ get_latest_workflow_control=AsyncMock(return_value=running),
+ ),
+ transition=AsyncMock(return_value=pausing),
+ )
+ workflow_service = SimpleNamespace(
+ pause_deployment=MagicMock(),
+ resume_deployment=AsyncMock(return_value=0),
+ )
+ restore = AsyncMock(return_value=running)
+
+ monkeypatch.setattr(handlers, "_control_service", lambda: service)
+ monkeypatch.setattr(
+ handlers,
+ "_reconcile_control",
+ AsyncMock(return_value=(running, {"state": "running", "revision": 0})),
+ )
+ monkeypatch.setattr(handlers, "_broadcast_control", AsyncMock(return_value={}))
+ monkeypatch.setattr(
+ handlers,
+ "_update_controller_state",
+ AsyncMock(
+ side_effect=handlers.TemporalControlUnavailable(
+ "temporal_control_unavailable"
+ )
+ ),
+ )
+ monkeypatch.setattr(
+ handlers,
+ "_restore_control_after_failed_update",
+ restore,
+ )
+ monkeypatch.setattr(
+ container_module.container,
+ "workflow_service",
+ lambda: workflow_service,
+ )
+
+ result = await handlers.handle_pause_workflow(
+ {"workflow_id": "wf", "expected_revision": 2},
+ None,
+ )
+
+ assert result == {
+ "success": False,
+ "error": "temporal_control_unavailable",
+ }
+ workflow_service.resume_deployment.assert_awaited_once_with("wf")
+ restore.assert_awaited_once_with(
+ service,
+ pausing,
+ transitional_state="pausing",
+ stable_state="running",
+ )
+
+
+async def test_unknown_update_outcome_stays_transitional_for_reconciliation(
+ monkeypatch,
+):
+ from core import container as container_module
+
+ running = _control("running", 2)
+ pausing = _control("pausing", 3)
+ service = SimpleNamespace(
+ database=SimpleNamespace(
+ get_latest_workflow_control=AsyncMock(return_value=running),
+ ),
+ transition=AsyncMock(return_value=pausing),
+ )
+ workflow_service = SimpleNamespace(
+ pause_deployment=MagicMock(),
+ resume_deployment=AsyncMock(return_value=0),
+ )
+ restore = AsyncMock()
+
+ monkeypatch.setattr(handlers, "_control_service", lambda: service)
+ monkeypatch.setattr(
+ handlers,
+ "_reconcile_control",
+ AsyncMock(return_value=(running, {"state": "running", "revision": 0})),
+ )
+ monkeypatch.setattr(handlers, "_broadcast_control", AsyncMock(return_value={}))
+ monkeypatch.setattr(
+ handlers,
+ "_update_controller_state",
+ AsyncMock(side_effect=TimeoutError("update outcome unknown")),
+ )
+ monkeypatch.setattr(
+ handlers,
+ "_restore_control_after_failed_update",
+ restore,
+ )
+ monkeypatch.setattr(
+ container_module.container,
+ "workflow_service",
+ lambda: workflow_service,
+ )
+
+ result = await handlers.handle_pause_workflow(
+ {"workflow_id": "wf", "expected_revision": 2},
+ None,
+ )
+
+ assert result == {"success": False, "error": "update outcome unknown"}
+ workflow_service.resume_deployment.assert_not_awaited()
+ restore.assert_not_awaited()
+
+
+async def test_runtime_counts_sum_controller_and_local_queues(monkeypatch):
+ from core import container as container_module
+
+ workflow_service = SimpleNamespace(
+ get_deployment_status=MagicMock(
+ return_value={
+ "active_runs": 4,
+ "queued_events": 3,
+ }
+ ),
+ )
+ monkeypatch.setattr(
+ container_module.container,
+ "workflow_service",
+ lambda: workflow_service,
+ )
+
+ result = await handlers._with_runtime_counts(
+ {"queued_count": 2},
+ "wf",
+ )
+
+ assert result["active_count"] == 4
+ assert result["in_flight_count"] == 4
+ assert result["queued_count"] == 5
+
+
+async def test_reset_cleanup_failure_leaves_control_resetting(monkeypatch):
+ running = _control("running", 2)
+ resetting = _control("resetting", 3)
+ cleanup_order: list[str] = []
+ database = SimpleNamespace(
+ get_latest_workflow_control=AsyncMock(return_value=running),
+ update_workflow_run_data_scope=AsyncMock(return_value=True),
+ )
+ service = SimpleNamespace(
+ database=database,
+ transition=AsyncMock(return_value=resetting),
+ )
+
+ async def delete_schedules_impl(*_args, **_kwargs):
+ cleanup_order.append("delete_schedules")
+ return 0
+
+ async def cancel_deployment_impl(*_args, **_kwargs):
+ cleanup_order.append("cancel_local")
+ return {"success": True}
+
+ async def terminate_impl(*_args, **_kwargs):
+ cleanup_order.append("terminate")
+ raise RuntimeError("workflow_termination_failed:1")
+
+ def close_local_admission(_workflow_id):
+ cleanup_order.append("close_local_admission")
+
+ async def signal_controller(*_args, **_kwargs):
+ cleanup_order.append("close_controller")
+
+ delete_schedules = AsyncMock(side_effect=delete_schedules_impl)
+ cancel_deployment = AsyncMock(side_effect=cancel_deployment_impl)
+ terminate = AsyncMock(side_effect=terminate_impl)
+
+ monkeypatch.setattr(handlers, "_control_service", lambda: service)
+ monkeypatch.setattr(
+ handlers,
+ "_broadcast_control",
+ AsyncMock(return_value={}),
+ )
+ monkeypatch.setattr(
+ handlers,
+ "_close_local_admission",
+ close_local_admission,
+ )
+ monkeypatch.setattr(
+ handlers,
+ "_signal_controller",
+ AsyncMock(side_effect=signal_controller),
+ )
+ monkeypatch.setattr(
+ handlers,
+ "_terminate_generation_workflows",
+ terminate,
+ )
+ monkeypatch.setattr(
+ handlers,
+ "_delete_cron_schedules",
+ delete_schedules,
+ )
+ monkeypatch.setattr(
+ handlers,
+ "handle_cancel_deployment",
+ cancel_deployment,
+ )
+
+ result = await handlers.handle_reset_workflow(
+ {"workflow_id": "wf", "expected_revision": 2},
+ None,
+ )
+
+ assert result == {
+ "success": False,
+ "error": "workflow_termination_failed:1",
+ }
+ assert service.transition.await_count == 1
+ first_transition = service.transition.await_args
+ assert first_transition.args == (running,)
+ assert first_transition.kwargs["expected_revision"] == 2
+ assert first_transition.kwargs["status"] == "resetting"
+ assert cleanup_order == [
+ "close_local_admission",
+ "close_controller",
+ "delete_schedules",
+ "cancel_local",
+ "terminate",
+ ]
+ delete_schedules.assert_awaited_once_with("wf", strict=True)
+ cancel_deployment.assert_awaited_once_with(
+ {"workflow_id": "wf"},
+ None,
+ )
+ terminate.assert_awaited_once_with(resetting, strict=True)
+ database.update_workflow_run_data_scope.assert_not_awaited()
+
+
+async def test_reset_local_cleanup_failure_leaves_control_resetting(
+ monkeypatch,
+):
+ resetting = _control("resetting", 3)
+ database = SimpleNamespace(
+ get_latest_workflow_control=AsyncMock(return_value=resetting),
+ update_workflow_run_data_scope=AsyncMock(return_value=True),
+ )
+ service = SimpleNamespace(
+ database=database,
+ transition=AsyncMock(),
+ )
+
+ monkeypatch.setattr(handlers, "_control_service", lambda: service)
+ monkeypatch.setattr(
+ handlers,
+ "_broadcast_control",
+ AsyncMock(return_value={}),
+ )
+ monkeypatch.setattr(handlers, "_signal_controller", AsyncMock())
+ monkeypatch.setattr(
+ handlers,
+ "_terminate_generation_workflows",
+ AsyncMock(return_value=2),
+ )
+ monkeypatch.setattr(
+ handlers,
+ "_delete_cron_schedules",
+ AsyncMock(return_value=1),
+ )
+ monkeypatch.setattr(
+ handlers,
+ "handle_cancel_deployment",
+ AsyncMock(
+ return_value={
+ "success": False,
+ "error": "listener teardown failed",
+ }
+ ),
+ )
+
+ result = await handlers.handle_reset_workflow(
+ {"workflow_id": "wf"},
+ None,
+ )
+
+ assert result == {
+ "success": False,
+ "error": (
+ "workflow_local_cleanup_failed:listener teardown failed"
+ ),
+ }
+ service.transition.assert_not_awaited()
+ database.update_workflow_run_data_scope.assert_not_awaited()
+
+
+async def test_already_reset_is_idempotent_without_repeating_cleanup(
+ monkeypatch,
+):
+ reset = _control("reset", 4)
+ database = SimpleNamespace(
+ get_latest_workflow_control=AsyncMock(return_value=reset),
+ update_workflow_run_data_scope=AsyncMock(),
+ )
+ service = SimpleNamespace(
+ database=database,
+ transition=AsyncMock(),
+ )
+ close_local_admission = MagicMock()
+ signal_controller = AsyncMock()
+ delete_schedules = AsyncMock()
+ cancel_deployment = AsyncMock()
+ terminate = AsyncMock()
+ control_payload = AsyncMock(
+ return_value={
+ "state": "ready",
+ "revision": 4,
+ "generation": 1,
+ }
+ )
+
+ monkeypatch.setattr(handlers, "_control_service", lambda: service)
+ monkeypatch.setattr(
+ handlers,
+ "_close_local_admission",
+ close_local_admission,
+ )
+ monkeypatch.setattr(handlers, "_signal_controller", signal_controller)
+ monkeypatch.setattr(
+ handlers,
+ "_delete_cron_schedules",
+ delete_schedules,
+ )
+ monkeypatch.setattr(
+ handlers,
+ "handle_cancel_deployment",
+ cancel_deployment,
+ )
+ monkeypatch.setattr(
+ handlers,
+ "_terminate_generation_workflows",
+ terminate,
+ )
+ monkeypatch.setattr(handlers, "_control_payload", control_payload)
+
+ result = await handlers.handle_reset_workflow(
+ {
+ "workflow_id": "wf",
+ "expected_revision": 4,
+ "idempotency_key": "reset-request-1",
+ },
+ None,
+ )
+
+ assert result == {
+ "success": True,
+ "idempotent": True,
+ "state": "ready",
+ "revision": 4,
+ "generation": 1,
+ }
+ control_payload.assert_awaited_once_with(reset)
+ service.transition.assert_not_awaited()
+ close_local_admission.assert_not_called()
+ signal_controller.assert_not_awaited()
+ delete_schedules.assert_not_awaited()
+ cancel_deployment.assert_not_awaited()
+ terminate.assert_not_awaited()
+ database.update_workflow_run_data_scope.assert_not_awaited()
+
+
+async def test_reset_retry_resumes_cleanup_before_marking_ready(monkeypatch):
+ from services.deployment import runtime_state
+ from services import status_broadcaster
+
+ resetting = _control("resetting", 3)
+ reset = _control("reset", 4)
+ database = SimpleNamespace(
+ get_latest_workflow_control=AsyncMock(return_value=resetting),
+ update_workflow_run_data_scope=AsyncMock(return_value=True),
+ )
+ service = SimpleNamespace(
+ database=database,
+ transition=AsyncMock(return_value=reset),
+ )
+ terminate = AsyncMock(return_value=2)
+ delete_schedules = AsyncMock(return_value=3)
+ cancel_deployment = AsyncMock(
+ return_value={
+ "success": False,
+ "message": "Workflow wf is not deployed",
+ }
+ )
+ archive_nodes = AsyncMock(
+ return_value={
+ "archived_nodes": 2,
+ "reset_nodes": ["agent-1"],
+ }
+ )
+ broadcaster = SimpleNamespace(broadcast=AsyncMock())
+
+ async def broadcast_control(control, **_kwargs):
+ return {
+ "state": "ready" if control.status == "reset" else control.status,
+ "revision": control.revision,
+ }
+
+ monkeypatch.setattr(handlers, "_control_service", lambda: service)
+ monkeypatch.setattr(
+ handlers,
+ "_broadcast_control",
+ AsyncMock(side_effect=broadcast_control),
+ )
+ monkeypatch.setattr(handlers, "_signal_controller", AsyncMock())
+ monkeypatch.setattr(
+ handlers,
+ "_terminate_generation_workflows",
+ terminate,
+ )
+ monkeypatch.setattr(
+ handlers,
+ "_delete_cron_schedules",
+ delete_schedules,
+ )
+ monkeypatch.setattr(
+ handlers,
+ "handle_cancel_deployment",
+ cancel_deployment,
+ )
+ monkeypatch.setattr(
+ runtime_state,
+ "archive_and_reset_node_state",
+ archive_nodes,
+ )
+ monkeypatch.setattr(
+ status_broadcaster,
+ "get_status_broadcaster",
+ lambda: broadcaster,
+ )
+
+ result = await handlers.handle_reset_workflow(
+ {"workflow_id": "wf"},
+ None,
+ )
+
+ assert result["success"] is True
+ assert result["state"] == "ready"
+ assert result["revision"] == 4
+ assert result["terminated_executions"] == 2
+ assert result["deleted_schedules"] == 3
+ assert result["local_cleanup_completed"] is False
+ terminate.assert_awaited_once_with(resetting, strict=True)
+ delete_schedules.assert_awaited_once_with("wf", strict=True)
+ cancel_deployment.assert_awaited_once_with(
+ {"workflow_id": "wf"},
+ None,
+ )
+ database.update_workflow_run_data_scope.assert_awaited_once()
+ archive_nodes.assert_awaited_once_with(
+ resetting,
+ database,
+ broadcaster,
+ )
+ service.transition.assert_awaited_once()
+ final_transition = service.transition.await_args
+ assert final_transition.args == (resetting,)
+ assert final_transition.kwargs["expected_revision"] == 3
+ assert final_transition.kwargs["from_statuses"] == {"resetting"}
+ assert final_transition.kwargs["status"] == "reset"
+
+
+def test_generation_fanout_uses_temporal_root_tree():
+ query = handlers._generation_visibility_query(_control("running", 2))
+
+ assert query == (
+ "(RootWorkflowId='workflow-control-wf-g1' OR "
+ "EventWorkflowId='wf') "
+ "AND ExecutionStatus='Running'"
+ )
+
+
+async def test_generation_fanout_expands_tagged_standalone_root_trees():
+ queries: list[str] = []
+ controller = SimpleNamespace(
+ id="workflow-control-wf-g1",
+ run_id="controller-run",
+ root_id="workflow-control-wf-g1",
+ )
+ cron_graph = SimpleNamespace(
+ id="cron-graph",
+ run_id="graph-run",
+ root_id="cron-firing-root",
+ )
+ cron_agent = SimpleNamespace(
+ id="cron-agent",
+ run_id="agent-run",
+ root_id="cron-firing-root",
+ )
+
+ def list_workflows(*, query):
+ queries.append(query)
+
+ async def iterate():
+ values = (
+ [controller, cron_graph]
+ if query.startswith("(RootWorkflowId=")
+ else [controller, cron_graph, cron_agent]
+ )
+ for value in values:
+ yield value
+
+ return iterate()
+
+ client = SimpleNamespace(list_workflows=list_workflows)
+
+ executions = await handlers._list_generation_workflows(
+ client,
+ _control("running", 2),
+ )
+
+ assert {
+ execution.id
+ for execution in executions
+ } == {
+ "workflow-control-wf-g1",
+ "cron-graph",
+ "cron-agent",
+ }
+ assert queries[0] == (
+ "(RootWorkflowId='workflow-control-wf-g1' OR "
+ "EventWorkflowId='wf') "
+ "AND ExecutionStatus='Running'"
+ )
+ assert queries[1] == (
+ "RootWorkflowId IN "
+ "('cron-firing-root', 'workflow-control-wf-g1') "
+ "AND ExecutionStatus='Running'"
+ )
diff --git a/server/tests/temporal/test_cooperative_pause.py b/server/tests/temporal/test_cooperative_pause.py
index 6dd71735..b7e3c5eb 100644
--- a/server/tests/temporal/test_cooperative_pause.py
+++ b/server/tests/temporal/test_cooperative_pause.py
@@ -1,7 +1,15 @@
"""Cooperative pause flags shared by Temporal orchestration workflows."""
+import asyncio
+import inspect
+from types import SimpleNamespace
+from unittest.mock import AsyncMock, MagicMock
+
import pytest
+from temporalio.exceptions import ApplicationError
+from temporalio.workflow import ActivityCancellationType
+from nodes.scheduler.cron_scheduler._workflow import CronTriggerWorkflow
from services.temporal.agent_workflow import AgentWorkflow, DelegatedTaskWorkflow
from services.temporal.polling_trigger_workflow import PollingTriggerWorkflow
from services.temporal.trigger_listener_workflow import TriggerListenerWorkflow
@@ -18,6 +26,7 @@ from services.temporal.workflow_control_workflow import WorkflowControlWorkflow
DelegatedTaskWorkflow,
TriggerListenerWorkflow,
PollingTriggerWorkflow,
+ CronTriggerWorkflow,
],
)
async def test_pause_and_resume_mutate_durable_workflow_state(workflow_type):
@@ -30,6 +39,1322 @@ async def test_pause_and_resume_mutate_durable_workflow_state(workflow_type):
assert instance._control_paused is False
+def test_controller_registers_acknowledged_control_update():
+ definition = WorkflowControlWorkflow.__temporal_workflow_definition
+
+ assert "set_control_state" in definition.updates
+ assert {"pause", "resume"} <= set(definition.signals)
+
+
+async def test_controller_control_update_is_acknowledged_and_idempotent():
+ controller = WorkflowControlWorkflow()
+
+ paused = await controller.set_control_state("pause")
+ assert paused["state"] == "paused"
+ assert paused["revision"] == 1
+
+ repeated = await controller.set_control_state("paused")
+ assert repeated["state"] == "paused"
+ assert repeated["revision"] == 1
+
+ resumed = await controller.set_control_state("resume")
+ assert resumed["state"] == "running"
+ assert resumed["revision"] == 2
+
+ repeated = await controller.set_control_state("running")
+ assert repeated["state"] == "running"
+ assert repeated["revision"] == 2
+
+
+async def test_controller_control_update_rejects_unknown_state_without_mutation():
+ controller = WorkflowControlWorkflow()
+
+ with pytest.raises(ApplicationError, match="Control state"):
+ await controller.set_control_state("stopped")
+
+ assert controller.status()["state"] == "running"
+ assert controller.status()["revision"] == 0
+
+
+def test_pause_scheduling_changes_are_replay_patch_guarded():
+ from nodes.scheduler.cron_scheduler import _workflow as cron_module
+ from services.temporal import agent_workflow as agent_module
+ from services.temporal import polling_trigger_workflow as polling_module
+ from services.temporal import trigger_listener_workflow as trigger_module
+ from services.temporal import workflow as machina_module
+ from services.temporal import workflow_control_workflow as control_module
+
+ expected = {
+ machina_module.COOPERATIVE_PAUSE_SCHEDULING_PATCH:
+ inspect.getsource(MachinaWorkflow.run),
+ polling_module.COOPERATIVE_PAUSE_SCHEDULING_PATCH:
+ inspect.getsource(PollingTriggerWorkflow.run),
+ agent_module.COOPERATIVE_PAUSE_SCHEDULING_PATCH:
+ inspect.getsource(AgentWorkflow.run),
+ agent_module.DELEGATED_TASK_COOPERATIVE_PAUSE_SCHEDULING_PATCH:
+ inspect.getsource(DelegatedTaskWorkflow.run),
+ agent_module.DELEGATION_PERMIT_CLEANUP_PATCH:
+ inspect.getsource(AgentWorkflow.run),
+ agent_module.DELEGATION_ACQUIRE_CANCELLATION_PATCH:
+ inspect.getsource(AgentWorkflow.run),
+ agent_module.DELEGATED_TASK_CANCELLATION_TERMINAL_PATCH:
+ inspect.getsource(DelegatedTaskWorkflow.run),
+ control_module.COOPERATIVE_PAUSE_SCHEDULING_PATCH:
+ inspect.getsource(WorkflowControlWorkflow.run),
+ trigger_module.COOPERATIVE_PAUSE_SCHEDULING_PATCH:
+ inspect.getsource(TriggerListenerWorkflow.run),
+ cron_module.COOPERATIVE_PAUSE_SCHEDULING_PATCH:
+ inspect.getsource(CronTriggerWorkflow.run),
+ }
+
+ assert set(expected) == {
+ "machina-cooperative-pause-scheduling-v1",
+ "polling-trigger-cooperative-pause-scheduling-v1",
+ "agent-cooperative-pause-scheduling-v1",
+ "delegated-task-cooperative-pause-scheduling-v1",
+ "agent-delegation-permit-cleanup-v1",
+ "agent-delegation-acquire-cancellation-v1",
+ "delegated-task-cancellation-terminal-v1",
+ "workflow-control-cooperative-pause-scheduling-v1",
+ "trigger-cooperative-pause-scheduling-v1",
+ "cron-cooperative-pause-scheduling-v1",
+ }
+ for patch_id, source in expected.items():
+ assert "workflow.patched(" in source
+ assert "COOPERATIVE_PAUSE_SCHEDULING_PATCH" in source
+ assert patch_id.endswith("-v1")
+
+
+async def test_controller_preserves_legacy_admission_and_patch_gates_metadata(
+ monkeypatch,
+):
+ from services.temporal.trigger_listener_workflow import (
+ TriggerListenerWorkflow,
+ )
+
+ spawn = AsyncMock()
+ monkeypatch.setattr(TriggerListenerWorkflow, "_spawn_child_run", spawn)
+ controller = WorkflowControlWorkflow()
+ spec = {
+ "listener_args": {
+ "trigger_node_id": "push-1",
+ "workflow_id": "workflow-1",
+ }
+ }
+
+ controller._use_cooperative_pause_schedule_gate = False
+ await controller._spawn_push_run({"id": "legacy"}, spec)
+ assert (
+ spawn.await_args.kwargs["admission_check"]
+ == controller._wait_until_running
+ )
+ assert spawn.await_args.kwargs["search_attributes"] is None
+
+ controller._use_cooperative_pause_schedule_gate = True
+ await controller._spawn_push_run({"id": "new"}, spec)
+ assert (
+ spawn.await_args.kwargs["admission_check"]
+ == controller._wait_until_running
+ )
+ assert spawn.await_args.kwargs["search_attributes"] is not None
+
+
+async def _wait_for_predicate(predicate, blocked: asyncio.Event) -> None:
+ if not predicate():
+ blocked.set()
+ while not predicate():
+ await asyncio.sleep(0)
+
+
+async def test_machina_rechecks_pause_between_ready_child_starts(monkeypatch):
+ from services.temporal import workflow as workflow_module
+
+ temporal_workflow = workflow_module.workflow
+ monkeypatch.setattr(temporal_workflow, "logger", MagicMock())
+ monkeypatch.setattr(temporal_workflow, "patched", lambda _patch_id: True)
+ monkeypatch.setattr(
+ temporal_workflow,
+ "info",
+ lambda: SimpleNamespace(workflow_id="root-run"),
+ )
+
+ paused_gate = asyncio.Event()
+ monkeypatch.setattr(
+ temporal_workflow,
+ "wait_condition",
+ lambda predicate: _wait_for_predicate(predicate, paused_gate),
+ )
+
+ instance = MachinaWorkflow()
+ scheduled: list[str] = []
+
+ async def start_child(_name, *, args, **_kwargs):
+ node_id = args[0]["node_id"]
+ scheduled.append(node_id)
+ handle = asyncio.get_running_loop().create_future()
+ handle.set_result(
+ {"success": True, "node_id": node_id, "result": {"node": node_id}}
+ )
+ if len(scheduled) == 1:
+ instance._control_paused = True
+ return handle
+
+ monkeypatch.setattr(temporal_workflow, "start_child_workflow", start_child)
+ monkeypatch.setattr(
+ MachinaWorkflow,
+ "_resolve_dispatch",
+ lambda self, _node_type: {
+ "kind": "child_workflow",
+ "name": "AgentWorkflow",
+ },
+ )
+
+ task = asyncio.create_task(
+ instance.run(
+ {
+ "nodes": [
+ {"id": "agent-a", "type": "aiAgent", "data": {}},
+ {"id": "agent-b", "type": "aiAgent", "data": {}},
+ ],
+ "edges": [],
+ "workflow_id": "graph",
+ "workflow_slug": "graph",
+ "execution_id": "root-run",
+ }
+ )
+ )
+
+ await asyncio.wait_for(paused_gate.wait(), timeout=1)
+ assert scheduled == ["agent-a"]
+
+ await instance.resume()
+ result = await asyncio.wait_for(task, timeout=1)
+
+ assert scheduled == ["agent-a", "agent-b"]
+ assert result["success"] is True
+
+
+def _poll_listener_data() -> dict:
+ return {
+ "node_type": "testPoll",
+ "version": 1,
+ "trigger_node_id": "poll-1",
+ "filter_params": {"poll_interval": 1},
+ "seen_ids": ["existing"],
+ "nodes": [],
+ "edges": [],
+ "workflow_id": None,
+ "session_id": "test",
+ }
+
+
+async def test_polling_rechecks_pause_after_sleep_before_activity(monkeypatch):
+ from services.temporal import polling_trigger_workflow as polling_module
+
+ temporal_workflow = polling_module.workflow
+ monkeypatch.setattr(temporal_workflow, "logger", MagicMock())
+ monkeypatch.setattr(temporal_workflow, "patched", lambda _patch_id: True)
+
+ instance = PollingTriggerWorkflow()
+ paused_gate = asyncio.Event()
+ activity_started = asyncio.Event()
+
+ async def sleep_then_pause(_duration):
+ instance._control_paused = True
+
+ async def execute_activity(*_args, **_kwargs):
+ activity_started.set()
+ raise asyncio.CancelledError
+
+ monkeypatch.setattr(temporal_workflow, "sleep", sleep_then_pause)
+ monkeypatch.setattr(temporal_workflow, "execute_activity", execute_activity)
+ monkeypatch.setattr(
+ temporal_workflow,
+ "wait_condition",
+ lambda predicate: _wait_for_predicate(predicate, paused_gate),
+ )
+
+ task = asyncio.create_task(instance.run(_poll_listener_data()))
+ await asyncio.wait_for(paused_gate.wait(), timeout=1)
+ assert not activity_started.is_set()
+
+ await instance.resume()
+ with pytest.raises(asyncio.CancelledError):
+ await asyncio.wait_for(task, timeout=1)
+ assert activity_started.is_set()
+
+
+async def test_polling_rechecks_pause_after_activity_before_spawn(monkeypatch):
+ from services.temporal import polling_trigger_workflow as polling_module
+
+ temporal_workflow = polling_module.workflow
+ monkeypatch.setattr(temporal_workflow, "logger", MagicMock())
+ monkeypatch.setattr(temporal_workflow, "patched", lambda _patch_id: True)
+
+ instance = PollingTriggerWorkflow()
+ paused_gate = asyncio.Event()
+ child_started = asyncio.Event()
+
+ async def no_sleep(_duration):
+ return None
+
+ async def poll_activity(*_args, **_kwargs):
+ instance._control_paused = True
+ return {
+ "seen_ids": ["existing", "event-1"],
+ "events": [{"id": "event-1"}],
+ }
+
+ async def spawn_child(
+ _event,
+ _listener_data,
+ admission_check=None,
+ search_attributes=None,
+ ):
+ assert admission_check is not None
+ child_started.set()
+ raise asyncio.CancelledError
+
+ monkeypatch.setattr(temporal_workflow, "sleep", no_sleep)
+ monkeypatch.setattr(temporal_workflow, "execute_activity", poll_activity)
+ monkeypatch.setattr(
+ temporal_workflow,
+ "wait_condition",
+ lambda predicate: _wait_for_predicate(predicate, paused_gate),
+ )
+ monkeypatch.setattr(instance, "_spawn_child_run", spawn_child)
+
+ task = asyncio.create_task(instance.run(_poll_listener_data()))
+ await asyncio.wait_for(paused_gate.wait(), timeout=1)
+ assert not child_started.is_set()
+
+ await instance.resume()
+ with pytest.raises(asyncio.CancelledError):
+ await asyncio.wait_for(task, timeout=1)
+ assert child_started.is_set()
+
+
+async def test_polling_child_admission_rechecks_after_idle_broadcast(monkeypatch):
+ from services.temporal import polling_trigger_workflow as polling_module
+ from services.temporal import trigger_listener_workflow as listener_module
+
+ temporal_workflow = polling_module.workflow
+ monkeypatch.setattr(temporal_workflow, "logger", MagicMock())
+ monkeypatch.setattr(temporal_workflow, "patched", lambda _patch_id: False)
+ monkeypatch.setattr(
+ listener_module,
+ "_build_run_graph",
+ lambda **_kwargs: ([], []),
+ )
+
+ instance = PollingTriggerWorkflow()
+ paused_gate = asyncio.Event()
+ child_started = asyncio.Event()
+
+ async def broadcast_idle(**_kwargs):
+ instance._control_paused = True
+
+ async def broadcast_waiting(**_kwargs):
+ return None
+
+ async def start_child(*_args, **_kwargs):
+ child_started.set()
+ return MagicMock()
+
+ monkeypatch.setattr(listener_module, "_broadcast_trigger_idle", broadcast_idle)
+ monkeypatch.setattr(
+ listener_module,
+ "_broadcast_trigger_waiting",
+ broadcast_waiting,
+ )
+ monkeypatch.setattr(temporal_workflow, "start_child_workflow", start_child)
+ monkeypatch.setattr(
+ temporal_workflow,
+ "wait_condition",
+ lambda predicate: _wait_for_predicate(predicate, paused_gate),
+ )
+
+ task = asyncio.create_task(
+ instance._spawn_child_run(
+ {"id": "event-1"},
+ _poll_listener_data(),
+ admission_check=instance._wait_until_resumed,
+ )
+ )
+ await asyncio.wait_for(paused_gate.wait(), timeout=1)
+ assert not child_started.is_set()
+
+ await instance.resume()
+ await asyncio.wait_for(task, timeout=1)
+ assert child_started.is_set()
+
+
+def _agent_payload() -> dict:
+ return {
+ "node_id": "agent-1",
+ "node_type": "aiAgent",
+ "workflow_id": "graph-1",
+ "session_id": "session-1",
+ "provider": "openai",
+ "model": "test-model",
+ "max_tokens": 100,
+ "temperature": 0,
+ "system_message": "Be useful",
+ "user_prompt": "do the work",
+ "tools": [
+ {
+ "name": "do_work",
+ "definition": {
+ "name": "do_work",
+ "description": "Do work",
+ "parameters": {"type": "object", "properties": {}},
+ },
+ "node_type": "testTool",
+ "version": 1,
+ "task_queue": "test",
+ "tool_node_id": "tool-1",
+ "parameters": {},
+ "tool_info": {},
+ }
+ ],
+ "memory_node_id": "",
+ "memory_content": "",
+ "memory_window_size": 10,
+ "max_iterations": 2,
+ "thinking_config": None,
+ "compaction_threshold": None,
+ "llm_engine": "native",
+ "message_wire_version": 2,
+ }
+
+
+async def test_agent_rechecks_pause_before_tool_command_batch(monkeypatch):
+ from services.temporal import agent_workflow as agent_module
+
+ temporal_workflow = agent_module.workflow
+ monkeypatch.setattr(temporal_workflow, "logger", MagicMock())
+ monkeypatch.setattr(temporal_workflow, "patched", lambda _patch_id: True)
+ monkeypatch.setattr(
+ temporal_workflow,
+ "info",
+ lambda: SimpleNamespace(
+ workflow_id="agent-run",
+ run_id="run-id-12345678",
+ ),
+ )
+ monkeypatch.setattr(
+ agent_module,
+ "get_node_class",
+ lambda _node_type: SimpleNamespace(needs_canvas=False),
+ )
+
+ instance = AgentWorkflow()
+ paused_gate = asyncio.Event()
+ tool_started = asyncio.Event()
+ llm_step = 0
+
+ async def execute_activity(name, *, args, **_kwargs):
+ nonlocal llm_step
+ if name == "agent.prepare_payload.v1":
+ return _agent_payload()
+ if name == "agent.broadcast_progress.v1":
+ return {"emitted": True}
+ if name == "agent.execute_llm_step.v1":
+ llm_step += 1
+ if llm_step == 1:
+ instance._control_paused = True
+ return {
+ "kind": "tool_calls",
+ "calls": [
+ {
+ "id": "call-1",
+ "name": "do_work",
+ "args": {},
+ }
+ ],
+ "usage": {},
+ }
+ return {
+ "kind": "final",
+ "content": "done",
+ "usage": {},
+ }
+ if name == "node.testTool.v1":
+ tool_started.set()
+ return {"success": True}
+ if name == "agent.store_output.v1":
+ return {"stored": True}
+ if name == "agent.skill.clear.v1":
+ return {"cleared": True}
+ raise AssertionError(f"Unexpected activity {name}")
+
+ monkeypatch.setattr(temporal_workflow, "execute_activity", execute_activity)
+ monkeypatch.setattr(
+ temporal_workflow,
+ "wait_condition",
+ lambda predicate: _wait_for_predicate(predicate, paused_gate),
+ )
+
+ task = asyncio.create_task(
+ instance.run({"node_id": "agent-1", "execution_id": "root-run"})
+ )
+ await asyncio.wait_for(paused_gate.wait(), timeout=1)
+ assert not tool_started.is_set()
+
+ await instance.resume()
+ result = await asyncio.wait_for(task, timeout=1)
+
+ assert tool_started.is_set()
+ assert result["success"] is True
+
+
+async def test_agent_rechecks_pause_after_tool_phase_before_activity(monkeypatch):
+ from services.temporal import agent_workflow as agent_module
+
+ temporal_workflow = agent_module.workflow
+ monkeypatch.setattr(temporal_workflow, "logger", MagicMock())
+ monkeypatch.setattr(temporal_workflow, "patched", lambda _patch_id: True)
+ monkeypatch.setattr(
+ temporal_workflow,
+ "info",
+ lambda: SimpleNamespace(
+ workflow_id="agent-run",
+ run_id="run-id-12345678",
+ ),
+ )
+ monkeypatch.setattr(
+ agent_module,
+ "get_node_class",
+ lambda _node_type: SimpleNamespace(needs_canvas=False),
+ )
+
+ instance = AgentWorkflow()
+ paused_gate = asyncio.Event()
+ tool_started = asyncio.Event()
+ llm_step = 0
+
+ async def execute_activity(name, *, args, **_kwargs):
+ nonlocal llm_step
+ if name == "agent.prepare_payload.v1":
+ return _agent_payload()
+ if name == "agent.broadcast_progress.v1":
+ if args[0]["phase"] == "executing_tool":
+ instance._control_paused = True
+ return {"emitted": True}
+ if name == "agent.execute_llm_step.v1":
+ llm_step += 1
+ if llm_step == 1:
+ return {
+ "kind": "tool_calls",
+ "calls": [{"id": "call-1", "name": "do_work", "args": {}}],
+ "usage": {},
+ }
+ return {"kind": "final", "content": "done", "usage": {}}
+ if name == "node.testTool.v1":
+ tool_started.set()
+ return {"success": True}
+ if name == "agent.store_output.v1":
+ return {"stored": True}
+ if name == "agent.skill.clear.v1":
+ return {"cleared": True}
+ raise AssertionError(f"Unexpected activity {name}")
+
+ monkeypatch.setattr(temporal_workflow, "execute_activity", execute_activity)
+ monkeypatch.setattr(
+ temporal_workflow,
+ "wait_condition",
+ lambda predicate: _wait_for_predicate(predicate, paused_gate),
+ )
+
+ task = asyncio.create_task(
+ instance.run({"node_id": "agent-1", "execution_id": "root-run"})
+ )
+ await asyncio.wait_for(paused_gate.wait(), timeout=1)
+ assert not tool_started.is_set()
+
+ await instance.resume()
+ result = await asyncio.wait_for(task, timeout=1)
+
+ assert tool_started.is_set()
+ assert result["success"] is True
+
+
+def _delegation_agent_payload() -> dict:
+ payload = _agent_payload()
+ payload.update(
+ {
+ "team_id": "team-1",
+ "tools": [
+ {
+ "name": "delegate_to_child",
+ "definition": {
+ "name": "delegate_to_child",
+ "description": "Delegate work",
+ "parameters": {"type": "object", "properties": {}},
+ },
+ "node_type": "aiAgent",
+ "version": 1,
+ "task_queue": "test",
+ "tool_node_id": "child-agent",
+ "parameters": {},
+ "tool_info": {"label": "Child"},
+ }
+ ],
+ }
+ )
+ return payload
+
+
+class _CompletedChildHandle:
+ id = "child-workflow"
+ first_execution_run_id = "child-run"
+
+ def __await__(self):
+ async def result():
+ return {
+ "success": True,
+ "result": {"response": "delegated result"},
+ }
+
+ return result().__await__()
+
+
+class _PendingChildHandle:
+ first_execution_run_id = "child-run"
+
+ def __init__(self, child_id: str) -> None:
+ self.id = child_id
+ self._result = asyncio.get_running_loop().create_future()
+
+ def __await__(self):
+ return self._result.__await__()
+
+
+def _two_delegation_agent_payload() -> dict:
+ payload = _delegation_agent_payload()
+ template = payload["tools"][0]
+ payload["tools"] = [
+ {
+ **template,
+ "name": "delegate_to_child_a",
+ "definition": {
+ **template["definition"],
+ "name": "delegate_to_child_a",
+ },
+ "tool_node_id": "child-agent-a",
+ "tool_info": {"label": "Child A"},
+ },
+ {
+ **template,
+ "name": "delegate_to_child_b",
+ "definition": {
+ **template["definition"],
+ "name": "delegate_to_child_b",
+ },
+ "tool_node_id": "child-agent-b",
+ "tool_info": {"label": "Child B"},
+ },
+ ]
+ return payload
+
+
+async def test_acquire_activity_compensates_cancelled_attempt(monkeypatch):
+ from services import agent_team
+ from services.temporal.agent_activities import acquire_subagent_permit
+
+ calls: list[tuple[str, str, str]] = []
+
+ async def acquire(root_id, permit_id, _limit):
+ calls.append(("acquire", root_id, permit_id))
+ # Model cancellation winning after the durable acquire side effect but
+ # before an activity result can be recorded in workflow history.
+ raise asyncio.CancelledError
+
+ async def release(root_id, permit_id):
+ calls.append(("release", root_id, permit_id))
+ return True
+
+ service = SimpleNamespace(
+ acquire_subagent_permit=acquire,
+ release_subagent_permit=release,
+ )
+ monkeypatch.setattr(
+ agent_team,
+ "get_agent_team_service",
+ lambda: service,
+ )
+
+ with pytest.raises(asyncio.CancelledError):
+ await acquire_subagent_permit({
+ "root_execution_id": "root-1",
+ "permit_id": "permit-1",
+ "limit": 3,
+ })
+
+ assert calls == [
+ ("acquire", "root-1", "permit-1"),
+ ("release", "root-1", "permit-1"),
+ ]
+
+
[email protected]("patch_enabled", [False, True])
+async def test_delegated_acquire_cancellation_type_is_replay_patch_guarded(
+ monkeypatch,
+ patch_enabled,
+):
+ from services.temporal import agent_workflow as agent_module
+
+ temporal_workflow = agent_module.workflow
+ monkeypatch.setattr(temporal_workflow, "logger", MagicMock())
+ monkeypatch.setattr(
+ temporal_workflow,
+ "patched",
+ lambda patch_id: (
+ patch_enabled
+ if patch_id
+ == agent_module.DELEGATION_ACQUIRE_CANCELLATION_PATCH
+ else False
+ ),
+ )
+ monkeypatch.setattr(
+ temporal_workflow,
+ "info",
+ lambda: SimpleNamespace(
+ workflow_id="delegated-runner",
+ run_id="runner-run-id",
+ ),
+ )
+
+ acquire_kwargs: dict = {}
+
+ async def execute_activity(name, *, args, **kwargs):
+ if name == "agent.register_task_execution.v1":
+ return {}
+ if name == "agent.acquire_subagent_permit.v1":
+ acquire_kwargs.update(kwargs)
+ raise asyncio.CancelledError
+ raise AssertionError(f"Unexpected activity {name}")
+
+ monkeypatch.setattr(temporal_workflow, "execute_activity", execute_activity)
+
+ with pytest.raises(asyncio.CancelledError):
+ await DelegatedTaskWorkflow().run({
+ "lifecycle": {
+ "team_id": "team-1",
+ "team_task_id": "task-1",
+ "root_execution_id": "root-1",
+ },
+ "child_workflow_id": "child-1",
+ "child_context": {},
+ })
+
+ if patch_enabled:
+ assert acquire_kwargs["cancellation_type"] == (
+ ActivityCancellationType.WAIT_CANCELLATION_COMPLETED
+ )
+ else:
+ assert "cancellation_type" not in acquire_kwargs
+
+
[email protected]("patch_enabled", [False, True])
+async def test_agent_acquire_cancellation_type_is_replay_patch_guarded(
+ monkeypatch,
+ patch_enabled,
+):
+ from services.temporal import agent_workflow as agent_module
+
+ temporal_workflow = agent_module.workflow
+ monkeypatch.setattr(temporal_workflow, "logger", MagicMock())
+ monkeypatch.setattr(
+ temporal_workflow,
+ "patched",
+ lambda patch_id: (
+ patch_enabled
+ if patch_id
+ == agent_module.DELEGATION_ACQUIRE_CANCELLATION_PATCH
+ else True
+ ),
+ )
+ monkeypatch.setattr(
+ temporal_workflow,
+ "info",
+ lambda: SimpleNamespace(
+ workflow_id="agent-run",
+ run_id="run-id-12345678",
+ ),
+ )
+
+ acquire_kwargs: dict = {}
+
+ async def execute_activity(name, *, args, **kwargs):
+ if name == "agent.prepare_payload.v1":
+ return _delegation_agent_payload()
+ if name == "agent.broadcast_progress.v1":
+ return {"emitted": True}
+ if name == "agent.execute_llm_step.v1":
+ return {
+ "kind": "tool_calls",
+ "calls": [{
+ "id": "delegate-1",
+ "name": "delegate_to_child",
+ "args": {"task": "research this"},
+ }],
+ "usage": {},
+ }
+ if name == "agent.queue_delegation.v1":
+ return {}
+ if name == "agent.acquire_subagent_permit.v1":
+ acquire_kwargs.update(kwargs)
+ raise asyncio.CancelledError
+ raise AssertionError(f"Unexpected activity {name}")
+
+ monkeypatch.setattr(temporal_workflow, "execute_activity", execute_activity)
+
+ with pytest.raises(asyncio.CancelledError):
+ await AgentWorkflow().run({
+ "node_id": "agent-1",
+ "execution_id": "root-run",
+ "nodes": [],
+ "edges": [],
+ })
+
+ if patch_enabled:
+ assert acquire_kwargs["cancellation_type"] == (
+ ActivityCancellationType.WAIT_CANCELLATION_COMPLETED
+ )
+ else:
+ assert "cancellation_type" not in acquire_kwargs
+
+
+ "failing_cleanup",
+ [None, "agent.finish_delegation.v1", "agent.release_subagent_permit.v1"],
+)
+async def test_delegated_cancellation_persists_terminal_releases_and_reraises(
+ monkeypatch,
+ failing_cleanup,
+):
+ from services.temporal import agent_workflow as agent_module
+
+ temporal_workflow = agent_module.workflow
+ monkeypatch.setattr(temporal_workflow, "logger", MagicMock())
+ monkeypatch.setattr(temporal_workflow, "patched", lambda _patch_id: True)
+ monkeypatch.setattr(
+ temporal_workflow,
+ "info",
+ lambda: SimpleNamespace(
+ workflow_id="delegated-runner",
+ run_id="runner-run-id",
+ ),
+ )
+
+ child_registered = asyncio.Event()
+ calls: list[tuple[str, dict]] = []
+ register_count = 0
+
+ async def execute_activity(name, *, args, **_kwargs):
+ nonlocal register_count
+ calls.append((name, args[0]))
+ if name == "agent.register_task_execution.v1":
+ register_count += 1
+ if register_count == 2:
+ child_registered.set()
+ return {}
+ if name in {
+ "agent.acquire_subagent_permit.v1",
+ "agent.begin_delegation.v1",
+ }:
+ return {}
+ if name == failing_cleanup:
+ raise RuntimeError(f"{name} failed")
+ if name in {
+ "agent.finish_delegation.v1",
+ "agent.release_subagent_permit.v1",
+ }:
+ return {}
+ raise AssertionError(f"Unexpected activity {name}")
+
+ async def start_child(*_args, **kwargs):
+ return _PendingChildHandle(kwargs["id"])
+
+ monkeypatch.setattr(temporal_workflow, "execute_activity", execute_activity)
+ monkeypatch.setattr(temporal_workflow, "start_child_workflow", start_child)
+
+ task = asyncio.create_task(
+ DelegatedTaskWorkflow().run({
+ "lifecycle": {
+ "team_id": "team-1",
+ "team_task_id": "task-1",
+ "root_execution_id": "root-1",
+ "parent_agent_node_id": "agent-1",
+ "child_agent_node_id": "agent-2",
+ },
+ "child_workflow_id": "child-1",
+ "child_context": {},
+ })
+ )
+ await asyncio.wait_for(child_registered.wait(), timeout=1)
+ task.cancel()
+
+ with pytest.raises(asyncio.CancelledError):
+ await asyncio.wait_for(task, timeout=1)
+
+ terminal = next(
+ payload
+ for name, payload in calls
+ if name == "agent.finish_delegation.v1"
+ )
+ assert terminal["success"] is False
+ assert terminal["error"] == "Delegated task workflow cancelled"
+ assert any(
+ name == "agent.release_subagent_permit.v1"
+ for name, _payload in calls
+ )
+
+
+ "failed_release_activity_id",
+ [None, "release-permit-1-1"],
+)
+async def test_agent_cancellation_releases_all_started_delegation_permits(
+ monkeypatch,
+ failed_release_activity_id,
+):
+ from services.temporal import agent_workflow as agent_module
+
+ temporal_workflow = agent_module.workflow
+ monkeypatch.setattr(temporal_workflow, "logger", MagicMock())
+ monkeypatch.setattr(temporal_workflow, "patched", lambda _patch_id: True)
+ monkeypatch.setattr(
+ temporal_workflow,
+ "info",
+ lambda: SimpleNamespace(
+ workflow_id="agent-run",
+ run_id="run-id-12345678",
+ ),
+ )
+
+ children_started = asyncio.Event()
+ child_count = 0
+ releases: list[tuple[str, str]] = []
+
+ async def execute_activity(name, *, args, **kwargs):
+ if name == "agent.prepare_payload.v1":
+ return _two_delegation_agent_payload()
+ if name == "agent.broadcast_progress.v1":
+ return {"emitted": True}
+ if name == "agent.execute_llm_step.v1":
+ return {
+ "kind": "tool_calls",
+ "calls": [
+ {
+ "id": "delegate-a",
+ "name": "delegate_to_child_a",
+ "args": {"task": "research A"},
+ },
+ {
+ "id": "delegate-b",
+ "name": "delegate_to_child_b",
+ "args": {"task": "research B"},
+ },
+ ],
+ "usage": {},
+ }
+ if name == "agent.release_subagent_permit.v1":
+ releases.append(
+ (kwargs["activity_id"], args[0]["permit_id"])
+ )
+ if kwargs["activity_id"] == failed_release_activity_id:
+ raise RuntimeError("release failed")
+ return {"released": True}
+ if name in {
+ "agent.queue_delegation.v1",
+ "agent.acquire_subagent_permit.v1",
+ "agent.begin_delegation.v1",
+ "agent.register_task_execution.v1",
+ }:
+ return {}
+ raise AssertionError(f"Unexpected activity {name}")
+
+ async def start_child(*_args, **kwargs):
+ nonlocal child_count
+ child_count += 1
+ if child_count == 2:
+ children_started.set()
+ return _PendingChildHandle(kwargs["id"])
+
+ monkeypatch.setattr(temporal_workflow, "execute_activity", execute_activity)
+ monkeypatch.setattr(temporal_workflow, "start_child_workflow", start_child)
+
+ task = asyncio.create_task(
+ AgentWorkflow().run(
+ {
+ "node_id": "agent-1",
+ "execution_id": "root-run",
+ "nodes": [],
+ "edges": [],
+ }
+ )
+ )
+ await asyncio.wait_for(children_started.wait(), timeout=1)
+ task.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await asyncio.wait_for(task, timeout=1)
+
+ assert releases == [
+ (
+ "release-permit-1-1",
+ "task-root-run-agent-1-1-1",
+ ),
+ (
+ "release-permit-1-2",
+ "task-root-run-agent-1-1-2",
+ ),
+ ]
+
+
+ ("pause_activity", "blocked_command"),
+ [
+ ("agent.acquire_subagent_permit.v1", "agent.begin_delegation.v1"),
+ ("agent.begin_delegation.v1", "start_child_workflow"),
+ ],
+)
+async def test_agent_rechecks_pause_between_delegation_commands(
+ monkeypatch,
+ pause_activity,
+ blocked_command,
+):
+ from services.temporal import agent_workflow as agent_module
+
+ temporal_workflow = agent_module.workflow
+ monkeypatch.setattr(temporal_workflow, "logger", MagicMock())
+ monkeypatch.setattr(temporal_workflow, "patched", lambda _patch_id: True)
+ monkeypatch.setattr(
+ temporal_workflow,
+ "info",
+ lambda: SimpleNamespace(
+ workflow_id="agent-run",
+ run_id="run-id-12345678",
+ ),
+ )
+
+ instance = AgentWorkflow()
+ paused_gate = asyncio.Event()
+ child_started = asyncio.Event()
+ scheduled_commands: list[str] = []
+ llm_step = 0
+
+ async def execute_activity(name, *, args, **_kwargs):
+ nonlocal llm_step
+ if name == "agent.prepare_payload.v1":
+ return _delegation_agent_payload()
+ if name == "agent.broadcast_progress.v1":
+ return {"emitted": True}
+ if name == "agent.execute_llm_step.v1":
+ llm_step += 1
+ if llm_step == 1:
+ return {
+ "kind": "tool_calls",
+ "calls": [
+ {
+ "id": "delegate-1",
+ "name": "delegate_to_child",
+ "args": {"task": "research this"},
+ }
+ ],
+ "usage": {},
+ }
+ return {"kind": "final", "content": "done", "usage": {}}
+ if name in {
+ "agent.queue_delegation.v1",
+ "agent.acquire_subagent_permit.v1",
+ "agent.begin_delegation.v1",
+ "agent.register_task_execution.v1",
+ "agent.release_subagent_permit.v1",
+ "agent.finish_delegation.v1",
+ }:
+ scheduled_commands.append(name)
+ if name == pause_activity:
+ instance._control_paused = True
+ return {}
+ if name == "agent.store_output.v1":
+ return {"stored": True}
+ if name == "agent.skill.clear.v1":
+ return {"cleared": True}
+ raise AssertionError(f"Unexpected activity {name}")
+
+ async def start_child(*_args, **_kwargs):
+ scheduled_commands.append("start_child_workflow")
+ child_started.set()
+ return _CompletedChildHandle()
+
+ monkeypatch.setattr(temporal_workflow, "execute_activity", execute_activity)
+ monkeypatch.setattr(temporal_workflow, "start_child_workflow", start_child)
+ monkeypatch.setattr(
+ temporal_workflow,
+ "wait_condition",
+ lambda predicate: _wait_for_predicate(predicate, paused_gate),
+ )
+
+ task = asyncio.create_task(
+ instance.run(
+ {
+ "node_id": "agent-1",
+ "execution_id": "root-run",
+ "nodes": [],
+ "edges": [],
+ }
+ )
+ )
+ await asyncio.wait_for(paused_gate.wait(), timeout=1)
+ assert blocked_command not in scheduled_commands
+ assert not child_started.is_set()
+
+ await instance.resume()
+ result = await asyncio.wait_for(task, timeout=1)
+
+ assert child_started.is_set()
+ assert result["success"] is True
+ assert scheduled_commands == [
+ "agent.queue_delegation.v1",
+ "agent.acquire_subagent_permit.v1",
+ "agent.begin_delegation.v1",
+ "start_child_workflow",
+ "agent.register_task_execution.v1",
+ "agent.release_subagent_permit.v1",
+ "agent.finish_delegation.v1",
+ ]
+
+
+def _mixed_preflight_payload() -> dict:
+ payload = _delegation_agent_payload()
+ payload["tools"].append(
+ {
+ "name": "manage_tasks",
+ "definition": {
+ "name": "manage_tasks",
+ "description": "Manage team tasks",
+ "parameters": {"type": "object", "properties": {}},
+ },
+ "node_type": "taskManager",
+ "version": 1,
+ "task_queue": "test",
+ "tool_node_id": "task-manager",
+ "parameters": {},
+ "tool_info": {"label": "Task Manager"},
+ }
+ )
+ return payload
+
+
+async def test_agent_rechecks_pause_before_task_manager_preflight_batch(monkeypatch):
+ from services.temporal import agent_workflow as agent_module
+
+ temporal_workflow = agent_module.workflow
+ monkeypatch.setattr(temporal_workflow, "logger", MagicMock())
+ monkeypatch.setattr(temporal_workflow, "patched", lambda _patch_id: True)
+ monkeypatch.setattr(
+ temporal_workflow,
+ "info",
+ lambda: SimpleNamespace(
+ workflow_id="agent-run",
+ run_id="run-id-12345678",
+ ),
+ )
+
+ instance = AgentWorkflow()
+ paused_gate = asyncio.Event()
+ preflight_started = asyncio.Event()
+
+ async def execute_activity(name, *, args, **kwargs):
+ if name == "agent.prepare_payload.v1":
+ return _mixed_preflight_payload()
+ if name == "agent.broadcast_progress.v1":
+ return {"emitted": True}
+ if name == "agent.execute_llm_step.v1":
+ return {
+ "kind": "tool_calls",
+ "calls": [
+ {
+ "id": "delegate-1",
+ "name": "delegate_to_child",
+ "args": {"task": "research this"},
+ },
+ {
+ "id": "assign-1",
+ "name": "manage_tasks",
+ "args": {
+ "operation": "assign_task",
+ "task": "follow up",
+ },
+ },
+ ],
+ "usage": {},
+ }
+ if (
+ name == "agent.release_subagent_permit.v1"
+ and kwargs.get("activity_id") == "yield-own-permit-1"
+ ):
+ instance._control_paused = True
+ return {}
+ raise AssertionError(f"Unexpected activity {name}")
+
+ def start_activity(*_args, **_kwargs):
+ preflight_started.set()
+ raise asyncio.CancelledError
+
+ monkeypatch.setattr(temporal_workflow, "execute_activity", execute_activity)
+ monkeypatch.setattr(temporal_workflow, "start_activity", start_activity)
+ monkeypatch.setattr(
+ temporal_workflow,
+ "wait_condition",
+ lambda predicate: _wait_for_predicate(predicate, paused_gate),
+ )
+
+ task = asyncio.create_task(
+ instance.run(
+ {
+ "node_id": "agent-1",
+ "execution_id": "root-run",
+ "team_task_id": "own-permit",
+ "nodes": [],
+ "edges": [],
+ }
+ )
+ )
+ await asyncio.wait_for(paused_gate.wait(), timeout=1)
+ assert not preflight_started.is_set()
+
+ await instance.resume()
+ with pytest.raises(asyncio.CancelledError):
+ await asyncio.wait_for(task, timeout=1)
+ assert preflight_started.is_set()
+
+
+async def test_agent_rechecks_pause_before_task_manager_child_start(monkeypatch):
+ from services.temporal import agent_workflow as agent_module
+
+ temporal_workflow = agent_module.workflow
+ monkeypatch.setattr(temporal_workflow, "logger", MagicMock())
+ monkeypatch.setattr(temporal_workflow, "patched", lambda _patch_id: True)
+ monkeypatch.setattr(
+ temporal_workflow,
+ "info",
+ lambda: SimpleNamespace(
+ workflow_id="agent-run",
+ run_id="run-id-12345678",
+ ),
+ )
+ monkeypatch.setattr(
+ agent_module,
+ "get_node_class",
+ lambda _node_type: SimpleNamespace(needs_canvas=False),
+ )
+
+ instance = AgentWorkflow()
+ paused_gate = asyncio.Event()
+ child_started = asyncio.Event()
+ llm_step = 0
+
+ async def execute_activity(name, *, args, **_kwargs):
+ nonlocal llm_step
+ if name == "agent.prepare_payload.v1":
+ return _mixed_preflight_payload()
+ if name == "agent.broadcast_progress.v1":
+ return {"emitted": True}
+ if name == "agent.execute_llm_step.v1":
+ llm_step += 1
+ if llm_step == 1:
+ return {
+ "kind": "tool_calls",
+ "calls": [
+ {
+ "id": "assign-1",
+ "name": "manage_tasks",
+ "args": {
+ "operation": "assign_task",
+ "task": "follow up",
+ },
+ }
+ ],
+ "usage": {},
+ }
+ return {"kind": "final", "content": "done", "usage": {}}
+ if name == "agent.queue_delegation.v1":
+ instance._control_paused = True
+ return {"queued": True}
+ if name == "agent.store_output.v1":
+ return {"stored": True}
+ if name == "agent.skill.clear.v1":
+ return {"cleared": True}
+ raise AssertionError(f"Unexpected activity {name}")
+
+ def start_activity(*_args, **_kwargs):
+ handle = asyncio.get_running_loop().create_future()
+ handle.set_result(
+ {
+ "delegation_request": {
+ "delegate_name": "delegate_to_child",
+ "assignee_node_id": "child-agent",
+ "team_task_id": "task-1",
+ "task": "follow up",
+ "context": "",
+ }
+ }
+ )
+ return handle
+
+ async def start_child(*_args, **_kwargs):
+ child_started.set()
+ return MagicMock()
+
+ monkeypatch.setattr(temporal_workflow, "execute_activity", execute_activity)
+ monkeypatch.setattr(temporal_workflow, "start_activity", start_activity)
+ monkeypatch.setattr(temporal_workflow, "start_child_workflow", start_child)
+ monkeypatch.setattr(
+ temporal_workflow,
+ "wait_condition",
+ lambda predicate: _wait_for_predicate(predicate, paused_gate),
+ )
+
+ task = asyncio.create_task(
+ instance.run(
+ {
+ "node_id": "agent-1",
+ "execution_id": "root-run",
+ "nodes": [],
+ "edges": [],
+ }
+ )
+ )
+ await asyncio.wait_for(paused_gate.wait(), timeout=1)
+ assert not child_started.is_set()
+
+ await instance.resume()
+ result = await asyncio.wait_for(task, timeout=1)
+
+ assert child_started.is_set()
+ assert result["success"] is True
+
+
@pytest.mark.asyncio
async def test_controller_routes_push_events_without_listener_workflow():
controller = WorkflowControlWorkflow()
diff --git a/server/tests/test_cron_canary.py b/server/tests/test_cron_canary.py
index 78c8e6b0..3df23cea 100644
--- a/server/tests/test_cron_canary.py
+++ b/server/tests/test_cron_canary.py
@@ -27,7 +27,7 @@ from __future__ import annotations
import sys
import types
from typing import Any, Dict, List
-from unittest.mock import AsyncMock, MagicMock
+from unittest.mock import AsyncMock, MagicMock, sentinel
import pytest
@@ -159,6 +159,15 @@ class TestCreateCronSchedule:
assert passed_id == schedule_id
assert passed_schedule.spec.cron_expressions == ["*/5 * * * *"]
assert passed_schedule.spec.time_zone_name == "America/New_York"
+ action_attributes = {
+ pair.key.name: pair.value
+ for pair in passed_schedule.action.typed_search_attributes
+ }
+ assert action_attributes == {
+ "EventWorkflowId": "wf-1",
+ "TriggerNodeId": "cron-1",
+ "EventTriggerKind": "cron",
+ }
@pytest.mark.asyncio
async def test_create_schedule_idempotent_on_already_running(self):
@@ -171,6 +180,8 @@ class TestCreateCronSchedule:
client = MagicMock()
client.create_schedule = AsyncMock(side_effect=ScheduleAlreadyRunningError())
+ update = AsyncMock()
+ client.get_schedule_handle.return_value = MagicMock(update=update)
schedule_id = await create_cron_schedule(
client,
@@ -184,6 +195,79 @@ class TestCreateCronSchedule:
)
assert schedule_id == "wf-1-cron-1"
+ client.get_schedule_handle.assert_called_once_with(schedule_id)
+ update.assert_awaited_once()
+
+ updater = update.await_args.args[0]
+ existing_schedule = client.create_schedule.await_args.args[1]
+ existing_attributes = (
+ client.create_schedule.await_args.kwargs["search_attributes"]
+ )
+ update_result = updater(
+ types.SimpleNamespace(
+ description=types.SimpleNamespace(
+ schedule=types.SimpleNamespace(
+ state=sentinel.paused_state,
+ action=existing_schedule.action,
+ ),
+ typed_search_attributes=existing_attributes,
+ ),
+ )
+ )
+ assert update_result.schedule.state is sentinel.paused_state
+ action_attributes = {
+ pair.key.name: pair.value
+ for pair in update_result.schedule.action.typed_search_attributes
+ }
+ assert action_attributes["EventWorkflowId"] == "wf-1"
+ assert update_result.search_attributes == (
+ update_result.schedule.action.typed_search_attributes
+ )
+
+ with pytest.raises(
+ RuntimeError,
+ match="cron_schedule_ownership_conflict",
+ ):
+ updater(
+ types.SimpleNamespace(
+ description=types.SimpleNamespace(
+ schedule=types.SimpleNamespace(
+ state=sentinel.paused_state,
+ action=existing_schedule.action,
+ ),
+ typed_search_attributes=[
+ types.SimpleNamespace(
+ key=types.SimpleNamespace(
+ name="EventWorkflowId",
+ ),
+ value="different-workflow",
+ ),
+ ],
+ ),
+ )
+ )
+
+ @pytest.mark.asyncio
+ async def test_cron_workflow_pause_resume_gate(self, monkeypatch):
+ from nodes.scheduler.cron_scheduler import _workflow as cron_workflow
+
+ wait_condition = AsyncMock()
+ monkeypatch.setattr(
+ cron_workflow.workflow,
+ "wait_condition",
+ wait_condition,
+ )
+ instance = cron_workflow.CronTriggerWorkflow()
+
+ await instance.pause()
+ assert instance._control_paused is True
+ await instance._wait_until_resumed()
+ wait_condition.assert_awaited_once()
+ predicate = wait_condition.await_args.args[0]
+ assert predicate() is False
+
+ await instance.resume()
+ assert predicate() is True
class _FakeScheduleIterator:
@@ -291,6 +375,68 @@ class TestDeleteCronSchedulesForDeployment:
# + skipped.
assert count == 1
+ @pytest.mark.asyncio
+ async def test_strict_cleanup_raises_after_sweeping_all_schedules(self):
+ from services.temporal.schedules import (
+ delete_cron_schedules_for_deployment,
+ )
+
+ deleted_ids: List[str] = []
+
+ async def fake_list(query, **kwargs):
+ return _FakeScheduleIterator(
+ ["cron-schedule-wf-1-bad", "cron-schedule-wf-1-good"],
+ )
+
+ def fake_get_handle(sid):
+ handle = MagicMock()
+ if "bad" in sid:
+ handle.delete = AsyncMock(
+ side_effect=RuntimeError("simulated delete failure"),
+ )
+ else:
+ handle.delete = AsyncMock(
+ side_effect=lambda: deleted_ids.append(sid),
+ )
+ return handle
+
+ client = MagicMock()
+ client.list_schedules = fake_list
+ client.get_schedule_handle = fake_get_handle
+
+ with pytest.raises(
+ RuntimeError,
+ match="cron_schedule_cleanup_failed:1",
+ ):
+ await delete_cron_schedules_for_deployment(
+ client,
+ "wf-1",
+ strict=True,
+ )
+
+ assert deleted_ids == ["cron-schedule-wf-1-good"]
+
+ @pytest.mark.asyncio
+ async def test_strict_cleanup_raises_when_visibility_scan_fails(self):
+ from services.temporal.schedules import (
+ delete_cron_schedules_for_deployment,
+ )
+
+ client = MagicMock()
+ client.list_schedules = AsyncMock(
+ side_effect=RuntimeError("visibility unavailable"),
+ )
+
+ with pytest.raises(
+ RuntimeError,
+ match="cron_schedule_visibility_cleanup_failed",
+ ):
+ await delete_cron_schedules_for_deployment(
+ client,
+ "wf-1",
+ strict=True,
+ )
+
def test_source_awaits_list_schedules_before_async_for(self):
"""Regression: ``Client.list_schedules`` is ``async def`` in
the temporalio SDK. Bare ``async for ... in client.list_schedules(...)``
diff --git a/server/tests/test_deployment_canary_listener.py b/server/tests/test_deployment_canary_listener.py
index d2573aac..a6a94a5e 100644
--- a/server/tests/test_deployment_canary_listener.py
+++ b/server/tests/test_deployment_canary_listener.py
@@ -673,6 +673,8 @@ class TestCancelSweepsStuckNodeStatuses:
edges=[],
session_id="sess",
)
+ mgr._paused_workflows.add("wf-1")
+ mgr._paused_events["wf-1"] = [("trigger-1", {"event": "queued"})]
# Patch _cancel_canary_listeners / _cancel_canary_cron_schedules
# to be no-ops so the test focuses on the sweep + terminal
@@ -682,6 +684,8 @@ class TestCancelSweepsStuckNodeStatuses:
result = await mgr.cancel("wf-1")
assert result["success"] is True
+ assert "wf-1" not in mgr._paused_workflows
+ assert "wf-1" not in mgr._paused_events
# Sweep ran once with include_waiting=True (every indicator
# goes quiet on explicit user cancel).
@@ -696,6 +700,74 @@ class TestCancelSweepsStuckNodeStatuses:
f"Expected update_workflow_status(executing=False, " f"workflow_id='wf-1') on cancel; got {status_calls!r}"
)
+ @pytest.mark.asyncio
+ async def test_cancel_clears_pause_state_when_deployment_is_already_absent(self):
+ """Reset may cancel after another path has already torn deployment down."""
+ from services.deployment.manager import DeploymentManager
+
+ mgr = DeploymentManager(
+ database=MagicMock(),
+ execute_workflow_fn=AsyncMock(return_value={"success": True}),
+ store_output_fn=AsyncMock(),
+ broadcaster=MagicMock(),
+ )
+ mgr._paused_workflows.update({"wf-gone", "wf-other"})
+ mgr._paused_events.update(
+ {
+ "wf-gone": [("trigger-1", {"event": "stale"})],
+ "wf-other": [("trigger-2", {"event": "keep"})],
+ }
+ )
+
+ result = await mgr.cancel("wf-gone")
+
+ assert result == {
+ "success": False,
+ "error": "Workflow wf-gone is not deployed",
+ }
+ assert "wf-gone" not in mgr._paused_workflows
+ assert "wf-gone" not in mgr._paused_events
+ assert "wf-other" in mgr._paused_workflows
+ assert "wf-other" in mgr._paused_events
+
+ @pytest.mark.asyncio
+ async def test_cancel_failure_keeps_admission_closed_and_events_queued(self):
+ """A partial teardown must not reopen a deployment that still exists."""
+ from services.deployment.manager import DeploymentManager
+ from services.deployment.state import DeploymentState
+
+ broadcaster = MagicMock()
+ broadcaster.update_node_status = AsyncMock()
+ mgr = DeploymentManager(
+ database=MagicMock(),
+ execute_workflow_fn=AsyncMock(return_value={"success": True}),
+ store_output_fn=AsyncMock(),
+ broadcaster=broadcaster,
+ )
+ mgr._deployments["wf-1"] = DeploymentState(
+ deployment_id="deploy_wf-1",
+ workflow_id="wf-1",
+ is_running=True,
+ nodes=[],
+ edges=[],
+ session_id="sess",
+ )
+ mgr._paused_events["wf-1"] = [
+ ("trigger-1", {"event": "queued"}),
+ ]
+ mgr._cancel_canary_listeners = AsyncMock(
+ side_effect=RuntimeError("temporal cleanup failed"),
+ )
+
+ with pytest.raises(RuntimeError, match="temporal cleanup failed"):
+ await mgr.cancel("wf-1")
+
+ assert "wf-1" in mgr._paused_workflows
+ assert mgr._paused_events["wf-1"] == [
+ ("trigger-1", {"event": "queued"}),
+ ]
+ assert "wf-1" in mgr._deployments
+
class TestNoInstanceStateForCanaryListeners:
"""Lock the architectural decision: DeploymentManager does NOT keep