Self-improving AI Employees for everyone
Python ai-agentsai-employeeai-employeesassistantautonomous-agentsco-employeescompanygrok-botinstinctloop-agentsmuseown-your-datapersonalself-hosted
fix(workflow-control): make Start/Pause/Resume/Reset authoritative over Temporal
trohitg77dc7db98ba2
diff --git a/client/src/Dashboard.tsx b/client/src/Dashboard.tsxindex f74d111c..8d987487 100644--- a/client/src/Dashboard.tsx+++ b/client/src/Dashboard.tsx@@ -221,7 +221,7 @@ const DashboardContent: React.FC = () => {} = useWorkflowManagement();const { collapsedSections, searchQuery, setSearchQuery, toggleSection } = useComponentPalette();- const { saveNodeParameters, getAllNodeParameters, executeWorkflow, deployWorkflow, getWorkflowStatus, nodeStatuses, deploymentStatus, workflowControlStatuses, startWorkflow, pauseWorkflow, resumeWorkflow, resetWorkflow, getWorkflowControlStatus, workflowLock, isConnected, isReady, sendRequest, clearNodeStatus } = useWebSocket();+ const { saveNodeParameters, getAllNodeParameters, executeWorkflow, getWorkflowStatus, nodeStatuses, deploymentStatus, workflowControlStatuses, workflowControlPending, startWorkflow, pauseWorkflow, resumeWorkflow, resetWorkflow, getWorkflowControlStatus, workflowLock, isConnected, isReady, sendRequest, clearNodeStatus } = useWebSocket();const applyUIDefaults = useAppStore((state) => state.applyUIDefaults);// Workflows list: server-owned data, cached by TanStack Query.@@ -232,18 +232,21 @@ const DashboardContent: React.FC = () => {// Only show as "running" or "locked" if it applies to the currently viewed workflowconst isCurrentWorkflowDeployed = deploymentStatus.isRunning &&deploymentStatus.workflow_id === currentWorkflow?.id;- // Backend is the source of truth: per-workflow `isExecuting` is set by- // the `workflow_status` handler whenever ANY active run exists for the- // current workflow (ad-hoc node, whole-workflow run, deployed trigger- // run, or deployment registration). The toolbar Start/Stop button- // tracks this unified signal so it stays Stop while nodes glow.+ // Durable lifecycle capabilities are the only source of truth for toolbar+ // and command-palette actions. Ad-hoc `isExecuting` remains separate and+ // only drives canvas execution visuals.const workflowControl = currentWorkflow?.id? workflowControlStatuses[currentWorkflow.id] || {workflow_id: currentWorkflow.id, state: 'never_started' as const, revision: 0,active_count: 0, in_flight_count: 0, queued_count: 0,- can_start: true, can_pause: false, can_resume: false, can_reset: false,+ // Status is unresolved until the backend snapshot arrives. Keep every+ // lifecycle action disabled rather than briefly exposing a stale Start.+ can_start: false, can_pause: false, can_resume: false, can_reset: false,}: { state: 'never_started' as const, revision: 0, active_count: 0, in_flight_count: 0, queued_count: 0, can_start: false, can_pause: false, can_resume: false, can_reset: false };+ const workflowControlPendingMutation = currentWorkflow?.id+ ? workflowControlPending[currentWorkflow.id]+ : undefined;const isCurrentWorkflowLocked = workflowLock.locked &&workflowLock.workflow_id === currentWorkflow?.id;const [globalModelDefaults, setGlobalModelDefaults] = React.useState<{ provider: string; model: string } | null>(null);@@ -798,18 +801,20 @@ const DashboardContent: React.FC = () => {const workflow = await openExampleAndChat('AI Assistant');if (!workflow) return;- const alreadyDeployed = deploymentStatus.isRunning && deploymentStatus.workflow_id === workflow.id;- if (alreadyDeployed) return;-try {- const result = await deployWorkflow(workflow.id, workflow.nodes, workflow.edges, 'default');- if (!result.success) {- toast.error('Could not start the example — press Start on the toolbar.');- }+ const status = await getWorkflowControlStatus(workflow.id);+ if (!status.can_start) return;+ await startWorkflow(+ workflow.id,+ workflow.nodes,+ workflow.edges,+ 'default',+ status.revision,+ );} catch {toast.error('Could not start the example — press Start on the toolbar.');}- }, [openExampleAndChat, deployWorkflow, deploymentStatus.isRunning, deploymentStatus.workflow_id]);+ }, [getWorkflowControlStatus, openExampleAndChat, startWorkflow]);const handlePauseWorkflow = async () => {const workflowId = currentWorkflow?.id;@@ -1264,8 +1269,8 @@ const DashboardContent: React.FC = () => {onNew={handleNew}onOpen={handleOpen}onRun={withSound('run', handleRun)}- isRunning={isExecuting}workflowControl={workflowControl}+ workflowControlPending={workflowControlPendingMutation}onStartWorkflow={withSound('run', handleDeploy)}onPauseWorkflow={handlePauseWorkflow}onResumeWorkflow={withSound('run', handleResumeWorkflow)}@@ -1434,6 +1439,7 @@ const DashboardContent: React.FC = () => {resume: withSound('run', handleResumeWorkflow),reset: handleResetWorkflow,workflowControl,+ workflowControlPending: workflowControlPendingMutation,exportFile: handleExportFile,importJSON: handleImportJSON,openSettings: () => setSettingsOpen(true),diff --git a/client/src/components/ui/CommandPaletteHost.tsx b/client/src/components/ui/CommandPaletteHost.tsxindex a73649d1..da763025 100644--- a/client/src/components/ui/CommandPaletteHost.tsx+++ b/client/src/components/ui/CommandPaletteHost.tsx@@ -1,3 +1,4 @@+/* eslint-disable react-refresh/only-export-components -- lifecycle command builder is exported for focused capability tests. *//*** CommandPaletteHost — Dashboard-level command list registration.*@@ -31,9 +32,12 @@ import {} from 'lucide-react';import { CommandPalette, type CommandItem } from './CommandPalette';import { AVAILABLE_THEMES, useTheme, type ThemeName } from '../../contexts/ThemeContext';-import type { WorkflowControlStatus } from '../../contexts/WebSocketContext';+import {+ type WorkflowControlPendingMutation,+ type WorkflowControlStatus,+} from '../../contexts/WebSocketContext';-interface Handlers {+export interface CommandPaletteHandlers {save: () => void;newWorkflow: () => void;open: () => void;@@ -42,6 +46,7 @@ interface Handlers {resume: () => void;reset: () => void;workflowControl: WorkflowControlStatus;+ workflowControlPending?: WorkflowControlPendingMutation;exportFile: () => void;importJSON: () => void;openSettings: () => void;@@ -69,9 +74,63 @@ const THEME_LABEL: Record<ThemeName, string> = {interface Props {open: boolean;onOpenChange: (open: boolean) => void;- handlers: Handlers;+ handlers: CommandPaletteHandlers;}+export const buildWorkflowLifecycleCommands = (+ handlers: Pick<+ CommandPaletteHandlers,+ 'start' | 'pause' | 'resume' | 'reset' | 'workflowControl' | 'workflowControlPending'+ >,+): CommandItem[] => {+ if (handlers.workflowControlPending) {+ return [];+ }++ const { state } = handlers.workflowControl;+ const commands: CommandItem[] = [];+ if (handlers.workflowControl.can_start) {+ commands.push({+ id: 'run.start',+ label: 'Start Workflow',+ group: 'Run',+ icon: Play,+ onRun: handlers.start,+ });+ }+ if (handlers.workflowControl.can_pause || state === 'pausing') {+ commands.push({+ id: 'run.pause',+ label: state === 'pausing' ? 'Retry Pause Workflow' : 'Pause Workflow',+ group: 'Run',+ icon: Pause,+ onRun: handlers.pause,+ });+ }+ if (handlers.workflowControl.can_resume || state === 'resuming') {+ commands.push({+ id: 'run.resume',+ label: state === 'resuming' ? 'Retry Resume Workflow' : 'Resume Workflow',+ group: 'Run',+ icon: Play,+ onRun: handlers.resume,+ });+ }+ if (handlers.workflowControl.can_reset) {+ commands.push({+ id: 'run.reset',+ label: state === 'resetting'+ ? 'Retry Reset Workflow Execution'+ : 'Reset Workflow Execution',+ group: 'Run',+ icon: RotateCcw,+ hint: 'terminates active work',+ onRun: handlers.reset,+ });+ }+ return commands;+};+export const CommandPaletteHost: React.FC<Props> = ({ open, onOpenChange, handlers }) => {const { theme, setTheme } = useTheme();@@ -116,27 +175,7 @@ export const CommandPaletteHost: React.FC<Props> = ({ open, onOpenChange, handle},// ── Run ────────────────────────────────────────────────────────- handlers.workflowControl.state === 'running'- ? {- id: 'run.pause',- label: 'Pause Workflow',- group: 'Run',- icon: Pause,- onRun: handlers.pause,- }- : handlers.workflowControl.state === 'paused'- ? { id: 'run.resume', label: 'Resume Workflow', group: 'Run', icon: Play, onRun: handlers.resume }- : {- id: 'run.start',- label: 'Start Workflow',- group: 'Run',- icon: Play,- onRun: handlers.start,- },- ...(handlers.workflowControl.can_reset ? [{- id: 'run.reset', label: 'Reset Workflow Execution', group: 'Run', icon: RotateCcw,- hint: 'terminates active work', onRun: handlers.reset,- } satisfies CommandItem] : []),+ ...buildWorkflowLifecycleCommands(handlers),// ── Open panels ────────────────────────────────────────────────{diff --git a/client/src/components/ui/TopToolbar.tsx b/client/src/components/ui/TopToolbar.tsxindex 381ea9f0..7eb84551 100644--- a/client/src/components/ui/TopToolbar.tsx+++ b/client/src/components/ui/TopToolbar.tsx@@ -47,7 +47,11 @@ import {AlertDialog, AlertDialogAction, AlertDialogCancel, AlertDialogContent,AlertDialogDescription, AlertDialogFooter, AlertDialogHeader, AlertDialogTitle,} from '@/components/ui/alert-dialog';-import type { WorkflowControlStatus } from '../../contexts/WebSocketContext';+import {+ isWorkflowControlTransitioning,+ type WorkflowControlPendingMutation,+ type WorkflowControlStatus,+} from '../../contexts/WebSocketContext';import { ThemeSwitcher } from '@/components/ui/ThemeSwitcher';import { cn } from '@/lib/utils';import { useAuth } from '../../contexts/AuthContext';@@ -67,8 +71,8 @@ interface TopToolbarProps {onNew: () => void;onOpen: () => void;onRun: () => void;- isRunning?: boolean;workflowControl: WorkflowControlStatus;+ workflowControlPending?: WorkflowControlPendingMutation;onStartWorkflow: () => void;onPauseWorkflow: () => void;onResumeWorkflow: () => void;@@ -96,8 +100,8 @@ const TopToolbar: React.FC<TopToolbarProps> = ({onNew,onOpen,onRun: _onRun,- isRunning = false,workflowControl,+ workflowControlPending,onStartWorkflow,onPauseWorkflow,onResumeWorkflow,@@ -121,6 +125,14 @@ const TopToolbar: React.FC<TopToolbarProps> = ({const [resetOpen, setResetOpen] = useState(false);const [tempName, setTempName] = useState(workflowName);const { user, logout } = useAuth();+ const hasPendingControlMutation = Boolean(workflowControlPending);+ const isAuthoritativeTransition = isWorkflowControlTransitioning(workflowControl);+ const canPauseWorkflow = workflowControl.can_pause || workflowControl.state === 'pausing';+ const canResumeWorkflow = workflowControl.can_resume || workflowControl.state === 'resuming';+ const isRetryingReset = workflowControl.state === 'resetting';+ const controlTransitionLabel = (+ workflowControlPending?.state ?? workflowControl.state+ ).replace('_', ' ');// Global Model Selector stateconst { getValidatedAiProviders, saveGlobalModel, isConnected: apiKeysConnected } = useApiKeys();@@ -404,37 +416,76 @@ const TopToolbar: React.FC<TopToolbarProps> = ({<Divider />{/* Durable workflow lifecycle */}- {workflowControl.can_start ? (+ {hasPendingControlMutation ? (+ <ActionButton+ intent="run"+ disabled+ title={`Workflow is ${controlTransitionLabel}`}+ >+ <LoaderCircle className="h-3 w-3 animate-spin" />+ {controlTransitionLabel}+ </ActionButton>+ ) : workflowControl.can_start ? (<ActionButtonintent="run"onClick={onStartWorkflow}- disabled={isRunning}title="Start workflow"><Play className="h-3 w-3 fill-current" />Start</ActionButton>- ) : workflowControl.state === 'paused' ? (- <ActionButton intent="run" onClick={onResumeWorkflow} title="Resume this workflow execution">- <Play className="h-3 w-3 fill-current" /> Resume+ ) : canResumeWorkflow ? (+ <ActionButton+ intent="run"+ onClick={onResumeWorkflow}+ title={workflowControl.state === 'resuming'+ ? 'Retry the interrupted resume transition'+ : 'Resume this workflow execution'}+ >+ <Play className="h-3 w-3 fill-current" />+ {workflowControl.state === 'resuming' ? 'Retry Resume' : 'Resume'}</ActionButton>- ) : workflowControl.state === 'running' ? (+ ) : canPauseWorkflow ? (<ActionButtonintent="stop"onClick={onPauseWorkflow}- title="Pause new workflow scheduling after in-flight work finishes"+ title={workflowControl.state === 'pausing'+ ? 'Retry the interrupted pause transition'+ : 'Pause new workflow scheduling after in-flight work finishes'}>- <Pause className="h-3 w-3 fill-current" /> Pause+ <Pause className="h-3 w-3 fill-current" />+ {workflowControl.state === 'pausing' ? 'Retry Pause' : 'Pause'}+ </ActionButton>+ ) : isAuthoritativeTransition ? (+ <ActionButton+ intent="run"+ disabled+ title={`Workflow is ${controlTransitionLabel}`}+ >+ <LoaderCircle className="h-3 w-3 animate-spin" />+ {controlTransitionLabel}</ActionButton>) : (<ActionButton intent="run" disabled title={`Workflow is ${workflowControl.state}`}>- <LoaderCircle className="h-3 w-3 animate-spin" /> {workflowControl.state.replace('_', ' ')}+ {workflowControl.state.replace('_', ' ')}</ActionButton>)}- {workflowControl.can_reset && <ActionButton intent="stop" onClick={() => setResetOpen(true)} title="Terminate and archive this execution"><RotateCcw className="h-3 w-3" /> Reset</ActionButton>}+ {workflowControl.can_reset && (+ <ActionButton+ intent="stop"+ onClick={() => setResetOpen(true)}+ disabled={hasPendingControlMutation}+ title={isRetryingReset+ ? 'Retry the interrupted reset transition'+ : 'Terminate and archive this execution'}+ >+ <RotateCcw className="h-3 w-3" />+ {isRetryingReset ? 'Retry Reset' : 'Reset'}+ </ActionButton>+ )}- <AlertDialog open={resetOpen} onOpenChange={setResetOpen}><AlertDialogContent><AlertDialogHeader><AlertDialogTitle>Reset workflow execution?</AlertDialogTitle><AlertDialogDescription>This immediately terminates active work and archives the current generation. Task history remains available. The workflow will remain stopped until you press Start.</AlertDialogDescription></AlertDialogHeader><AlertDialogFooter><AlertDialogCancel>Keep current execution</AlertDialogCancel><AlertDialogAction onClick={onResetWorkflow}>Reset workflow</AlertDialogAction></AlertDialogFooter></AlertDialogContent></AlertDialog>+ <AlertDialog open={resetOpen} onOpenChange={setResetOpen}><AlertDialogContent><AlertDialogHeader><AlertDialogTitle>Reset workflow execution?</AlertDialogTitle><AlertDialogDescription>This immediately terminates active work and archives the current generation. Task history remains available. The workflow will remain stopped until you press Start.</AlertDialogDescription></AlertDialogHeader><AlertDialogFooter><AlertDialogCancel>Keep current execution</AlertDialogCancel><AlertDialogAction disabled={hasPendingControlMutation} onClick={onResetWorkflow}>{isRetryingReset ? 'Retry reset' : 'Reset workflow'}</AlertDialogAction></AlertDialogFooter></AlertDialogContent></AlertDialog><ActionButtonintent="save"diff --git a/client/src/components/ui/__tests__/CommandPaletteHost.workflowControl.test.ts b/client/src/components/ui/__tests__/CommandPaletteHost.workflowControl.test.tsnew file mode 100644index 00000000..74582e20--- /dev/null+++ b/client/src/components/ui/__tests__/CommandPaletteHost.workflowControl.test.ts@@ -0,0 +1,84 @@+import { describe, expect, it, vi } from 'vitest';+import type { WorkflowControlStatus } from '../../../contexts/WebSocketContext';+import {+ buildWorkflowLifecycleCommands,+ type CommandPaletteHandlers,+} from '../CommandPaletteHost';++const control = (+ overrides: Partial<WorkflowControlStatus> = {},+): WorkflowControlStatus => ({+ workflow_id: 'workflow-1',+ generation: 1,+ state: 'running',+ revision: 1,+ active_count: 0,+ in_flight_count: 0,+ queued_count: 0,+ can_start: false,+ can_pause: false,+ can_resume: false,+ can_reset: false,+ ...overrides,+});++const handlers = (+ workflowControl: WorkflowControlStatus,+ overrides: Partial<CommandPaletteHandlers> = {},+) => ({+ start: vi.fn(),+ pause: vi.fn(),+ resume: vi.fn(),+ reset: vi.fn(),+ workflowControl,+ ...overrides,+});++describe('command palette workflow lifecycle capabilities', () => {+ it.each([+ ['run.start', { state: 'failed' as const, can_start: true }],+ ['run.pause', { state: 'paused' as const, can_pause: true }],+ ['run.resume', { state: 'running' as const, can_resume: true }],+ ])('exposes %s from its capability rather than the generic state', (id, overrides) => {+ const commands = buildWorkflowLifecycleCommands(handlers(control(overrides)));+ expect(commands.map((command) => command.id)).toContain(id);+ });++ it('does not infer a start action when no lifecycle capability allows one', () => {+ const commands = buildWorkflowLifecycleCommands(handlers(control({+ state: 'never_started',+ can_start: false,+ })));+ expect(commands.map((command) => command.id)).not.toContain('run.start');+ });++ it('suppresses lifecycle actions while a local mutation is pending', () => {+ const commands = buildWorkflowLifecycleCommands(handlers(+ control({ can_pause: true, can_reset: true }),+ { workflowControlPending: { action: 'pause', state: 'pausing' } },+ ));+ expect(commands).toEqual([]);+ });++ it.each([+ ['pausing' as const, 'run.pause', 'Retry Pause Workflow'],+ ['resuming' as const, 'run.resume', 'Retry Resume Workflow'],+ ['resetting' as const, 'run.reset', 'Retry Reset Workflow Execution'],+ ])('makes an authoritative %s transition retryable', (state, id, label) => {+ const commands = buildWorkflowLifecycleCommands(handlers(control({+ state,+ can_reset: state === 'resetting',+ })));+ expect(commands).toEqual(expect.arrayContaining([+ expect.objectContaining({ id, label }),+ ]));+ });++ it('allows Reset to recover an authoritative Start transition when permitted', () => {+ const commands = buildWorkflowLifecycleCommands(handlers(control({+ state: 'starting',+ can_reset: true,+ })));+ expect(commands.map((command) => command.id)).toEqual(['run.reset']);+ });+});diff --git a/client/src/contexts/WebSocketContext.tsx b/client/src/contexts/WebSocketContext.tsxindex c2be66ec..d700abc9 100644--- a/client/src/contexts/WebSocketContext.tsx+++ b/client/src/contexts/WebSocketContext.tsx@@ -63,6 +63,11 @@ interface QueuedSend {// Request timeout (30 seconds)const REQUEST_TIMEOUT = 30000;+// Lifecycle handlers wait for durable deployment setup, Temporal acknowledgements,+// and generation teardown. Keep their request alive longer than ordinary reads+// while retaining a finite bound so a lost response cannot lock controls forever.+export const WORKFLOW_CONTROL_REQUEST_TIMEOUT = 5 * 60 * 1000;+// Maximum queued sends before backpressure kicks in (FIFO eviction of oldest)const QUEUE_MAX_SIZE = 200;@@ -142,6 +147,55 @@ export interface WorkflowControlStatus {error?: string | null;}+export type WorkflowControlMutationAction = 'start' | 'pause' | 'resume' | 'reset';+export type WorkflowControlTransitionState = Extract<+ WorkflowControlState,+ 'starting' | 'pausing' | 'resuming' | 'resetting'+>;++export interface WorkflowControlPendingMutation {+ action: WorkflowControlMutationAction;+ state: WorkflowControlTransitionState;+}++const WORKFLOW_CONTROL_STATES = new Set<WorkflowControlState>([+ 'never_started',+ 'ready',+ 'starting',+ 'running',+ 'pausing',+ 'paused',+ 'resuming',+ 'resetting',+ 'failed',+]);++const WORKFLOW_CONTROL_TRANSITION_STATES = new Set<WorkflowControlState>([+ 'starting',+ 'pausing',+ 'resuming',+ 'resetting',+]);++export const isWorkflowControlTransitioning = (+ status: Pick<WorkflowControlStatus, 'state'>,+): boolean => WORKFLOW_CONTROL_TRANSITION_STATES.has(status.state);++const WORKFLOW_CONTROL_STABLE_STATE_BY_ACTION: Record<+ WorkflowControlMutationAction,+ WorkflowControlState+> = {+ start: 'running',+ pause: 'paused',+ resume: 'running',+ reset: 'ready',+};++export const isWorkflowControlMutationConfirmed = (+ action: WorkflowControlMutationAction,+ status: Pick<WorkflowControlStatus, 'state'>,+): boolean => status.state === WORKFLOW_CONTROL_STABLE_STATE_BY_ACTION[action];+export interface TeamTaskTraceEvent {event_id: string | number;event_type: string;@@ -361,6 +415,7 @@ interface WebSocketContextValue {workflowStatus: WorkflowStatus;deploymentStatus: DeploymentStatus;workflowControlStatuses: Record<string, WorkflowControlStatus>;+ workflowControlPending: Record<string, WorkflowControlPendingMutation>;workflowLock: WorkflowLock;compactionStats: Record<string, CompactionStats>; // session_id -> stats (current workflow)@@ -476,8 +531,9 @@ const defaultDeploymentStatus: DeploymentStatus = {status: 'idle'};-const emptyWorkflowControlStatus = (workflowId?: string): WorkflowControlStatus => ({+export const emptyWorkflowControlStatus = (workflowId?: string): WorkflowControlStatus => ({workflow_id: workflowId || null,+ generation: 0,state: 'never_started',revision: 0,active_count: 0,@@ -489,7 +545,7 @@ const emptyWorkflowControlStatus = (workflowId?: string): WorkflowControlStatuscan_reset: false,});-const normalizeWorkflowControlStatus = (value: any, workflowId?: string): WorkflowControlStatus => {+export const normalizeWorkflowControlStatus = (value: any, workflowId?: string): WorkflowControlStatus => {const source = value?.status && typeof value.status === 'object' ? value.status : value || {};const state = source.state || source.control_state || (source.is_running ? 'running' : 'never_started');return {@@ -497,7 +553,8 @@ const normalizeWorkflowControlStatus = (value: any, workflowId?: string): Workfl...source,workflow_id: source.workflow_id ?? workflowId ?? null,state,- revision: Number(source.revision || 0),+ generation: Number(source.generation ?? 0),+ revision: Number(source.revision ?? 0),active_count: Number(source.active_count ?? source.active_runs ?? 0),in_flight_count: Number(source.in_flight_count ?? source.active_count ?? source.active_runs ?? 0),queued_count: Number(source.queued_count || 0),@@ -508,6 +565,83 @@ const normalizeWorkflowControlStatus = (value: any, workflowId?: string): Workfl};};+/**+ * Mutation failures may still carry the newest authoritative control-plane+ * snapshot (for example, an in-progress retry). Only recognize payloads with+ * an explicit valid lifecycle state so a generic error cannot be projected as+ * a synthetic `never_started` status.+ */+export const extractWorkflowControlStatusSnapshot = (+ value: unknown,+ workflowId?: string,+): WorkflowControlStatus | undefined => {+ if (!value || typeof value !== 'object') return undefined;+ const candidate = value as Record<string, any>;+ const source = candidate.status && typeof candidate.status === 'object'+ ? candidate.status as Record<string, any>+ : candidate;+ const state = source.state ?? source.control_state;+ if (typeof state !== 'string' || !WORKFLOW_CONTROL_STATES.has(state as WorkflowControlState)) {+ return undefined;+ }+ return normalizeWorkflowControlStatus(value, workflowId);+};++/**+ * Resolve two snapshots from the workflow control plane without allowing a+ * delayed response or broadcast to move the UI to an older generation or+ * revision. Equal versions are accepted because runtime counts can change+ * without a control-plane transition.+ */+export const mergeWorkflowControlStatus = (+ current: WorkflowControlStatus | undefined,+ incoming: WorkflowControlStatus,+): WorkflowControlStatus => {+ if (!current) return incoming;++ const currentGeneration = Number(current.generation ?? 0);+ const incomingGeneration = Number(incoming.generation ?? 0);+ if (incomingGeneration < currentGeneration) return current;+ if (incomingGeneration > currentGeneration) return incoming;++ const currentRevision = Number(current.revision ?? 0);+ const incomingRevision = Number(incoming.revision ?? 0);+ return incomingRevision < currentRevision ? current : incoming;+};++export const assertWorkflowControlMutationSucceeded = <T,>(response: T): T => {+ if ((response as any)?.success === false) {+ const message = (response as any)?.error || (response as any)?.message || 'Workflow lifecycle mutation failed';+ throw new Error(String(message));+ }+ return response;+};++export const shouldRetryResetWorkflowAfterConflict = (+ error: unknown,+ status: Pick<WorkflowControlStatus, 'can_reset'>,+): boolean => (+ status.can_reset+ && error instanceof Error+ && error.message === 'control_revision_conflict'+);++type WorkflowControlMutationRequest =+ | 'start_workflow'+ | 'pause_workflow'+ | 'resume_workflow'+ | 'reset_workflow';++const WORKFLOW_CONTROL_PENDING_BY_REQUEST: Record<+ WorkflowControlMutationRequest,+ WorkflowControlPendingMutation+> = {+ start_workflow: { action: 'start', state: 'starting' },+ pause_workflow: { action: 'pause', state: 'pausing' },+ resume_workflow: { action: 'resume', state: 'resuming' },+ reset_workflow: { action: 'reset', state: 'resetting' },+};+const defaultWorkflowLock: WorkflowLock = {locked: false,workflow_id: null,@@ -604,6 +738,8 @@ export const WebSocketProvider: React.FC<{ children: React.ReactNode }> = ({ chiconst [deploymentStatus, setDeploymentStatus] = useState<DeploymentStatus>(defaultDeploymentStatus);const [workflowControlStatuses, setWorkflowControlStatuses] = useState<Record<string, WorkflowControlStatus>>({});const workflowControlStatusesRef = useRef<Record<string, WorkflowControlStatus>>({});+ const [workflowControlPending, setWorkflowControlPending] = useState<Record<string, WorkflowControlPendingMutation>>({});+ const workflowControlPendingRef = useRef<Map<string, WorkflowControlPendingMutation>>(new Map());const [workflowLock, setWorkflowLock] = useState<WorkflowLock>(defaultWorkflowLock);// Per-workflow compaction stats: workflow_id -> session_id -> CompactionStats (n8n pattern)const [allCompactionStats, setAllCompactionStats] = useState<Record<string, Record<string, CompactionStats>>>({});@@ -636,9 +772,25 @@ export const WebSocketProvider: React.FC<{ children: React.ReactNode }> = ({ chi// do not read from elsewhere.const previousWorkflowIdForSwitchRef = useRef<string | undefined>(currentWorkflowId);- useEffect(() => {- workflowControlStatusesRef.current = workflowControlStatuses;- }, [workflowControlStatuses]);+ const applyWorkflowControlStatus = useCallback((+ workflowId: string,+ value: any,+ ): WorkflowControlStatus => {+ const incoming = normalizeWorkflowControlStatus(value, workflowId);+ const current = workflowControlStatusesRef.current[workflowId];+ const merged = mergeWorkflowControlStatus(current, incoming);+ if (current && merged === current) return current;++ const next = {+ ...workflowControlStatusesRef.current,+ [workflowId]: merged,+ };+ // Update the ref synchronously so multiple responses/broadcasts received+ // in one React batch still compare against the latest accepted version.+ workflowControlStatusesRef.current = next;+ setWorkflowControlStatuses(next);+ return merged;+ }, []);// Detect workflow switches and refresh deployment status from the backend.// The single source of truth for currentWorkflowId is `useAppStore`; the@@ -1384,10 +1536,7 @@ export const WebSocketProvider: React.FC<{ children: React.ReactNode }> = ({ chiconst payload = message.data || message;const workflowId = payload.workflow_id;if (workflowId) {- setWorkflowControlStatuses((previous) => ({- ...previous,- [workflowId]: normalizeWorkflowControlStatus(payload, workflowId),- }));+ applyWorkflowControlStatus(workflowId, payload);}break;}@@ -1686,7 +1835,7 @@ export const WebSocketProvider: React.FC<{ children: React.ReactNode }> = ({ chi} catch (error) {console.error('[WebSocket] Failed to parse message:', error);}- }, []); // Empty deps - reads workflow id via useAppStore.getState() escape hatch+ }, [applyWorkflowControlStatus]);// Drain queued sends after a successful reconnect. Each queued send gets a// fresh request_id and a reset timeout budget; responses correlate via the@@ -2551,20 +2700,102 @@ export const WebSocketProvider: React.FC<{ children: React.ReactNode }> = ({ chi}}, [sendRequest]);+ const requestWorkflowControlStatus = useCallback(async (+ workflowId: string,+ ): Promise<WorkflowControlStatus> => {+ const response = assertWorkflowControlMutationSucceeded(+ await sendRequest<any>('get_workflow_control_status', { workflow_id: workflowId }),+ );+ return applyWorkflowControlStatus(workflowId, response);+ }, [applyWorkflowControlStatus, sendRequest]);+const controlMutation = useCallback(async (- type: 'start_workflow' | 'pause_workflow' | 'resume_workflow' | 'reset_workflow',+ type: WorkflowControlMutationRequest,workflowId: string,data: Record<string, any>,): Promise<WorkflowControlStatus> => {- const response = await sendRequest<any>(type, {- workflow_id: workflowId,- idempotency_key: crypto.randomUUID(),- ...data,- });- const normalized = normalizeWorkflowControlStatus(response, workflowId);- setWorkflowControlStatuses((previous) => ({ ...previous, [workflowId]: normalized }));- return normalized;- }, [sendRequest]);+ if (workflowControlPendingRef.current.has(workflowId)) {+ throw new Error('A workflow lifecycle change is already in progress');+ }++ const pending = WORKFLOW_CONTROL_PENDING_BY_REQUEST[type];+ workflowControlPendingRef.current.set(workflowId, pending);+ setWorkflowControlPending((previous) => ({ ...previous, [workflowId]: pending }));++ const sendMutationAttempt = async (+ attemptData: Record<string, any>,+ ): Promise<WorkflowControlStatus> => {+ const response = await sendRequest<any>(type, {+ ...attemptData,+ workflow_id: workflowId,+ // Every bounded attempt has its own idempotency identity. In+ // particular, a conflict retry must not replay the rejected request.+ idempotency_key: crypto.randomUUID(),+ }, WORKFLOW_CONTROL_REQUEST_TIMEOUT);++ // A success:false response can still be the newest authoritative+ // snapshot. Merge it before assertion so the UI is correct even if the+ // subsequent reconciliation request is unavailable.+ const snapshot = extractWorkflowControlStatusSnapshot(response, workflowId);+ const acceptedSnapshot = snapshot+ ? applyWorkflowControlStatus(workflowId, snapshot)+ : undefined;+ assertWorkflowControlMutationSucceeded(response);+ return acceptedSnapshot ?? applyWorkflowControlStatus(workflowId, response);+ };++ try {+ return await sendMutationAttempt(data);+ } catch (error) {+ let reconciled: WorkflowControlStatus | undefined;+ try {+ reconciled = await requestWorkflowControlStatus(workflowId);+ if (isWorkflowControlMutationConfirmed(pending.action, reconciled)) {+ return reconciled;+ }+ } catch (resyncError) {+ console.error('[WebSocket] Failed to resync workflow control status:', resyncError);+ }++ // Start can legitimately advance `starting -> starting` while attaching+ // its controller run id, which increments the revision. If Reset raced+ // that CAS, retry exactly once with the newly read revision. Other+ // mutations and other errors never retry automatically.+ if (+ type === 'reset_workflow'+ && reconciled+ && shouldRetryResetWorkflowAfterConflict(error, reconciled)+ ) {+ try {+ return await sendMutationAttempt({+ ...data,+ expected_revision: reconciled.revision,+ });+ } catch (retryError) {+ // Preserve timeout/idempotency recovery for the single retry without+ // issuing a third mutation attempt.+ try {+ const retryReconciled = await requestWorkflowControlStatus(workflowId);+ if (isWorkflowControlMutationConfirmed(pending.action, retryReconciled)) {+ return retryReconciled;+ }+ } catch (resyncError) {+ console.error('[WebSocket] Failed to resync workflow control status after reset retry:', resyncError);+ }+ throw retryError;+ }+ }+ throw error;+ } finally {+ workflowControlPendingRef.current.delete(workflowId);+ setWorkflowControlPending((previous) => {+ if (!(workflowId in previous)) return previous;+ const next = { ...previous };+ delete next[workflowId];+ return next;+ });+ }+ }, [applyWorkflowControlStatus, requestWorkflowControlStatus, sendRequest]);const graphEnvelope = (nodes: any[], edges: any[]) => ({nodes: nodes.map((node) => ({ id: node.id, type: node.type || '', data: node.data || {} })),@@ -2613,15 +2844,12 @@ export const WebSocketProvider: React.FC<{ children: React.ReactNode }> = ({ chiconst getWorkflowControlStatusAsync = useCallback(async (workflowId: string) => {try {- const response = await sendRequest<any>('get_workflow_control_status', { workflow_id: workflowId });- const normalized = normalizeWorkflowControlStatus(response, workflowId);- setWorkflowControlStatuses((previous) => ({ ...previous, [workflowId]: normalized }));- return normalized;+ return await requestWorkflowControlStatus(workflowId);} catch (error) {console.error('[WebSocket] Failed to get workflow control status:', error);- return emptyWorkflowControlStatus(workflowId);+ throw error;}- }, [sendRequest]);+ }, [requestWorkflowControlStatus]);const getTeamTaskTraceAsync = useCallback(async (scope: Record<string, any>): Promise<TeamTaskTraceResponse> => {const response = await sendRequest<any>('get_team_task_trace', scope);@@ -3249,6 +3477,7 @@ export const WebSocketProvider: React.FC<{ children: React.ReactNode }> = ({ chiworkflowStatus,deploymentStatus,workflowControlStatuses,+ workflowControlPending,workflowLock,// Compaction stats (real-time via broadcasts)@@ -3358,7 +3587,7 @@ export const WebSocketProvider: React.FC<{ children: React.ReactNode }> = ({ chiapiKeyStatuses,consoleLogs, terminalLogs, chatMessages,nodeStatuses, nodeParameters,- variables, workflowStatus, deploymentStatus, workflowControlStatuses, workflowLock,+ variables, workflowStatus, deploymentStatus, workflowControlStatuses, workflowControlPending, workflowLock,compactionStats, updateCompactionStats,getNodeStatus, getApiKeyStatus, getVariable,requestStatus, clearNodeStatus,diff --git a/client/src/contexts/__tests__/workflowControlStatus.test.ts b/client/src/contexts/__tests__/workflowControlStatus.test.tsnew file mode 100644index 00000000..a63c222b--- /dev/null+++ b/client/src/contexts/__tests__/workflowControlStatus.test.ts@@ -0,0 +1,180 @@+import { describe, expect, it } from 'vitest';+import {+ WORKFLOW_CONTROL_REQUEST_TIMEOUT,+ assertWorkflowControlMutationSucceeded,+ extractWorkflowControlStatusSnapshot,+ isWorkflowControlMutationConfirmed,+ mergeWorkflowControlStatus,+ normalizeWorkflowControlStatus,+ shouldRetryResetWorkflowAfterConflict,+ type WorkflowControlMutationAction,+ type WorkflowControlState,+ type WorkflowControlStatus,+} from '../WebSocketContext';++const status = (+ overrides: Partial<WorkflowControlStatus> = {},+): WorkflowControlStatus => ({+ workflow_id: 'workflow-1',+ generation: 1,+ state: 'running',+ revision: 1,+ active_count: 0,+ in_flight_count: 0,+ queued_count: 0,+ can_start: false,+ can_pause: true,+ can_resume: false,+ can_reset: true,+ ...overrides,+});++describe('workflow control status ordering', () => {+ it('rejects an older generation even when it carries a larger revision', () => {+ const current = status({ generation: 3, revision: 2, state: 'running' });+ const stale = status({ generation: 2, revision: 99, state: 'paused' });++ expect(mergeWorkflowControlStatus(current, stale)).toBe(current);+ });++ it('accepts a newer pre-migration generation whose revision restarted', () => {+ const current = status({ generation: 2, revision: 8, state: 'ready' });+ const incoming = status({ generation: 3, revision: 1, state: 'starting' });++ expect(mergeWorkflowControlStatus(current, incoming)).toBe(incoming);+ });++ it('rejects a lower revision in the same generation', () => {+ const current = status({ generation: 4, revision: 7, state: 'paused' });+ const stale = status({ generation: 4, revision: 6, state: 'running' });++ expect(mergeWorkflowControlStatus(current, stale)).toBe(current);+ });++ it('accepts equal-version runtime count refreshes', () => {+ const current = status({ generation: 4, revision: 7, active_count: 1 });+ const incoming = status({ generation: 4, revision: 7, active_count: 2 });++ expect(mergeWorkflowControlStatus(current, incoming)).toBe(incoming);+ });++ it('normalizes nested snapshots and numeric version strings', () => {+ const normalized = normalizeWorkflowControlStatus({+ status: {+ workflow_id: 'workflow-1',+ generation: '5',+ revision: '11',+ state: 'paused',+ active_runs: '2',+ },+ });++ expect(normalized).toMatchObject({+ workflow_id: 'workflow-1',+ generation: 5,+ revision: 11,+ state: 'paused',+ active_count: 2,+ can_resume: true,+ });+ });+});++describe('workflow control mutation responses', () => {+ it('uses a lifecycle timeout long enough for durable setup and teardown', () => {+ expect(WORKFLOW_CONTROL_REQUEST_TIMEOUT).toBe(5 * 60 * 1000);+ });++ it('rejects explicit success:false responses with the backend error', () => {+ expect(() => assertWorkflowControlMutationSucceeded({+ success: false,+ error: 'control_revision_conflict',+ state: 'never_started',+ })).toThrow('control_revision_conflict');+ });++ it('returns successful status payloads unchanged', () => {+ const response = { success: true, generation: 2, revision: 3 };+ expect(assertWorkflowControlMutationSucceeded(response)).toBe(response);+ });++ it('extracts the authoritative snapshot carried by a success:false response', () => {+ const current = status({ state: 'running', revision: 3 });+ const response = {+ success: false,+ error: 'workflow_control_transition_pending',+ workflow_id: 'workflow-1',+ generation: 1,+ state: 'pausing',+ revision: 4,+ can_start: false,+ can_pause: false,+ can_resume: false,+ can_reset: true,+ };++ const snapshot = extractWorkflowControlStatusSnapshot(response);++ expect(snapshot).toMatchObject({+ workflow_id: 'workflow-1',+ state: 'pausing',+ revision: 4,+ can_reset: true,+ });+ expect(mergeWorkflowControlStatus(current, snapshot!)).toBe(snapshot);+ expect(() => assertWorkflowControlMutationSucceeded(response))+ .toThrow('workflow_control_transition_pending');+ });++ it('does not mistake a generic mutation error for a control snapshot', () => {+ expect(extractWorkflowControlStatusSnapshot({+ success: false,+ error: 'control_revision_conflict',+ workflow_id: 'workflow-1',+ })).toBeUndefined();+ });++ it('only retries Reset conflicts when the resynced snapshot still allows Reset', () => {+ const conflict = new Error('control_revision_conflict');+ expect(shouldRetryResetWorkflowAfterConflict(+ conflict,+ status({ state: 'starting', revision: 2, can_reset: true }),+ )).toBe(true);+ expect(shouldRetryResetWorkflowAfterConflict(+ conflict,+ status({ state: 'ready', revision: 3, can_reset: false }),+ )).toBe(false);+ expect(shouldRetryResetWorkflowAfterConflict(+ new Error('workflow_control_transition_pending'),+ status({ state: 'starting', revision: 2, can_reset: true }),+ )).toBe(false);+ });++ it.each([+ ['start', 'running'],+ ['pause', 'paused'],+ ['resume', 'running'],+ ['reset', 'ready'],+ ] satisfies Array<[WorkflowControlMutationAction, WorkflowControlState]>)(+ 'accepts an authoritative %s resync only after it reaches %s',+ (action, stableState) => {+ expect(isWorkflowControlMutationConfirmed(action, status({+ state: stableState,+ }))).toBe(true);+ },+ );++ it.each([+ ['start', 'starting'],+ ['pause', 'pausing'],+ ['resume', 'resuming'],+ ['reset', 'resetting'],+ ] satisfies Array<[WorkflowControlMutationAction, WorkflowControlState]>)(+ 'does not treat the transitional %s state as confirmed',+ (action, transitionalState) => {+ expect(isWorkflowControlMutationConfirmed(action, status({+ state: transitionalState,+ }))).toBe(false);+ },+ );+});diff --git a/docs-internal/frontend_architecture.md b/docs-internal/frontend_architecture.mdindex 796462ae..6b62b4af 100644--- a/docs-internal/frontend_architecture.md+++ b/docs-internal/frontend_architecture.md@@ -393,6 +393,25 @@ See [ui_migration_plan.md](./ui_migration_plan.md) Phase 6.persisted generation record is reconciled with the Temporal controller ratherthan inferred from process-local tasks. `workflow_control_status` broadcastsupdate transitions in real time; reconnect performs an authoritative read.+ Reads, broadcasts, and mutation responses merge monotonically by generation+ and database revision, while per-workflow pending mutation state disables+ duplicate clicks. A `{success: false}` response carrying a recognizable+ control snapshot merges that snapshot before surfacing the error; every+ failure is followed by an authoritative resync, and a generic error is never+ normalized into `never_started`.+ Lifecycle mutations use a five-minute acknowledgement timeout because Start+ and Reset may wait on durable setup/cleanup. If a timeout races a successful+ operation, the resync-confirmed stable state is returned as success.+ Authoritative `pausing`, `resuming`, and `resetting` states remain retryable+ after the local request finishes. The UI treats stable `paused`/`running` as+ confirmation that strict cron-Schedule and running-execution fan-out+ completed; a fan-out failure must remain transitional after resync.+ Likewise, `ready` means Reset first quiesced controller, cron, and local+ producers and then completed its final execution sweep. A duplicate Reset in+ `ready` is a read-only idempotent response, so an old request cannot clean up+ resources belonging to a later Start. Standalone cron and detached child+ executions carry `EventWorkflowId`, allowing the server to uphold that+ boundary through Temporal Visibility.The older `deployment_snapshot` and binary deployment status remain migrationadapters for legacy deployments and must not override a resolved controllergeneration. See [Temporal Execution Engine RFC](temporal-execution-engine-rfc.md).diff --git a/docs-internal/temporal-execution-engine-rfc.md b/docs-internal/temporal-execution-engine-rfc.mdindex 15cec5cd..f3c1e471 100644--- a/docs-internal/temporal-execution-engine-rfc.md+++ b/docs-internal/temporal-execution-engine-rfc.md@@ -51,8 +51,8 @@ path.### 2.2 Non-goals-- Native Temporal Workflow Pause. The engine uses cooperative Signals and- deterministic wait conditions.+- Native Temporal Workflow Pause. The engine uses an acknowledged controller+ Update, cooperative descendant Signals, and deterministic wait conditions.- Temporal's history-rewind Reset. Product Reset terminates and archives thecurrent application generation.- Shared parent/child LLM transcripts. Delegated agents start with isolated@@ -243,20 +243,29 @@ sequenceDiagramUser->>UI: PauseUI->>API: pause_workflow(expected_revision)API->>DB: running -> pausing (CAS)- API->>C: pause signal- API->>T: pause cron schedules- API->>DM: gate legacy admission + disarm trigger UI+ API-->>UI: workflow_control_status(pausing)+ API->>DM: close local admission+ API->>C: set_control_state(paused) Update+ C-->>API: acknowledged paused state+ API->>T: strictly pause cron schedules+ API->>T: strictly signal running descendants+ T-->>API: schedule + descendant fan-out complete+ API->>DM: disarm trigger UIAPI->>DB: pausing -> paused- API-->>UI: paused+ API-->>UI: workflow_control_status(paused)User->>UI: ResumeUI->>API: resume_workflow(expected_revision)API->>DB: paused -> resuming (CAS)- API->>C: resume signal- API->>T: unpause cron schedules+ API-->>UI: workflow_control_status(resuming)+ API->>C: set_control_state(running) Update+ C-->>API: acknowledged running state+ API->>T: strictly unpause cron schedules+ API->>T: strictly signal running descendants+ T-->>API: schedule + descendant fan-out completeAPI->>DM: drain buffered legacy events FIFO + rearm UIAPI->>DB: resuming -> running- API-->>UI: running+ API-->>UI: workflow_control_status(running)```### 6.4 Cooperative pause semantics@@ -268,19 +277,35 @@ Pause is an admission barrier, not suspension of operating-system threads:- Poll cycles do not start new graph runs; a poll activity already in flight mayfinish and its events wait for Resume.- Temporal cron schedules are explicitly paused.-- `MachinaWorkflow`, `AgentWorkflow`, and `DelegatedTaskWorkflow` gate new- scheduling at deterministic wait conditions.+- `WorkflowControlWorkflow`, `TriggerListenerWorkflow`,+ `PollingTriggerWorkflow`, `CronTriggerWorkflow`, `MachinaWorkflow`,+ `AgentWorkflow`, and `DelegatedTaskWorkflow` gate new scheduling at+ deterministic, replay-patched wait conditions. Admission is rechecked after+ awaited trigger broadcasts, polling cycles, LLM, preflight, permit, and+ delegation operations, immediately before the next activity or child+ workflow is scheduled.- Activities already started are allowed to finish. Their results remain inhistory and are consumed after the gate opens.- Trigger nodes broadcast an idle/paused projection, stop their listeninganimation, and rearm on Resume.+- Stable `paused`/`running` is published only after strict cron-Schedule and+ running-execution fan-out completes. Visibility or per-target failures leave+ the generation transitional and retryable.### 6.5 Reset semantics-Reset transitions the generation to `resetting`, rejects new admission,-signals the controller to close, terminates visible descendants carrying the-application workflow search attribute, cancels local compatibility resources,-and archives the record as `reset`. It does **not** start the next generation.+Reset transitions the generation to `resetting` and quiesces producers before+its final execution sweep: it synchronously closes local admission, signals the+controller to close, deletes generation cron schedules, and cancels local+compatibility resources. It then strictly terminates controller descendants and+standalone roots carrying the application workflow search attribute before+archiving the data scope. `EventWorkflowId` is propagated to Schedule action+executions and detached/abandoned graph children so those standalone runs remain+discoverable. Runtime/node cleanup completes before the database publishes+`reset`; a failed step leaves `resetting` retryable. A duplicate command against+an already-reset generation returns idempotently without repeating cleanup,+which prevents an old retry from affecting a later generation. Reset does+**not** start the next generation.Task history and historical Temporal histories are preserved. Existinghistorical rows continue to appear in Temporal Web until namespace retention ordiff --git a/docs-internal/temporal-workflow-control.md b/docs-internal/temporal-workflow-control.mdindex e1ca9def..618f1a2f 100644--- a/docs-internal/temporal-workflow-control.md+++ b/docs-internal/temporal-workflow-control.md@@ -16,15 +16,54 @@ is authoritative for execution history.- **Resume** signals the same running Temporal executions and drains bufferedtrigger events in FIFO order, unpauses cron schedules, and rearms triggernodes.-- **Reset** revision-guards the old generation, terminates visible executions- carrying the workflow search attribute, cancels local deployment resources,- and archives the old generation. It leaves the control state `ready`; the- user must press **Start** to create the next generation.--All mutations require an expected revision and support an idempotency key.+- **Reset** revision-guards the old generation, closes controller and local+ admission, removes Temporal cron schedules, cancels local compatibility+ resources, and then performs a final strict execution sweep before archiving+ the old generation. It leaves the control state `ready`; the user must press+ **Start** to create the next generation.++Clients send an expected revision and idempotency key with every mutation.+New state transitions compare-and-swap the expected revision; Start persists+its request key, controller Updates receive unique request identities, and+stable/retry states are idempotent from their durable state.+Revisions increase across generation boundaries instead of restarting at zero,+so a delayed request from an archived generation cannot satisfy a later+generation's compare-and-swap check.The toolbar and command palette derive available actions from the server's`can_start`, `can_pause`, `can_resume`, and `can_reset` fields.+Start does not publish `running` merely because deployment setup was placed on+an asyncio task. The control handler waits for trigger/listener setup to finish;+setup failure moves the generation to `failed` and closes its controller.+Once the `running` compare-and-swap commits, a status-broadcast/projection+failure does not tear down that live generation; the failed request is recovered+by the client's authoritative status resync.++Pause and Resume use the controller's acknowledged `set_control_state` Temporal+Update. The database first publishes `pausing`/`resuming`, then publishes the+stable state only after the Update result confirms `paused`/`running`. If the+Update is known to have been rejected before admission, the database and local+admission gate return to the prior stable state. An unknown outcome, such as a+transport timeout after the server may have accepted the Update, remains+transitional. Status reads and explicit UI retries reconcile that state by+idempotently retrying the desired Update, reapplying schedule/local admission+gates, and completing the database CAS.+Legacy pause/resume Signals remain registered for history compatibility and+fan out to already-running descendant workflows. That fan-out and every cron+Schedule mutation are strict lifecycle barriers: a visibility or per-target+failure leaves the database in `pausing`/`resuming` for reconciliation instead+of falsely publishing a stable state.++Every accepted control transition emits `workflow_control_status`. Clients+merge reads, mutation responses, and broadcasts monotonically by generation+then database revision; an older response cannot move another tab backward.+Only one lifecycle mutation per workflow may be pending in a browser tab.+Lifecycle requests have a five-minute acknowledgement window, and a timed-out+request is treated as successful when the immediate authoritative resync+already reports its requested stable state. Transitional Pause, Resume, and+Reset states expose an explicit retry rather than trapping the toolbar in a+permanent spinner.+## Generation-scoped workflow dataThe editable `workflows` row is the stable canvas definition; it is not runtime@@ -39,13 +78,32 @@ state. Every successful **Start** atomically creates a- remains immutable as an execution snapshot while runtime records accumulateunder its scope ID.-**Reset** terminates the generation and marks its data scope `archived`; it does-not delete outputs, tasks, traces, or the graph snapshot. The toolbar returns to+**Reset** reaches `ready` only after durable executions and schedules are gone,+the generation data scope is archived, and node/runtime state has been reset.+If any cleanup step fails, the control remains `resetting`; retrying Reset+resumes cleanup instead of falsely completing or skipping it. Reset does not+delete outputs, tasks, traces, or the graph snapshot. The toolbar returns to**Start**. The next Start creates a new generation, execution ID, Temporalcontroller, and empty runtime namespace from the then-current saved canvas.Historical scopes therefore remain queryable without leaking state into thenew run.+Reset quiesces every producer before its final strict Visibility sweep: local+admission closes synchronously, the controller is told to close, cron Schedules+are deleted, and legacy local resources are cancelled. `EventWorkflowId` is+propagated to cron action executions and detached/abandoned graph children, so+the first Visibility pass identifies both the controller tree and standalone+execution roots. Because Temporal does not inherit custom Search Attributes+onto child workflows, a second batched `RootWorkflowId` pass expands every+tagged root to its active Agent and delegated descendants before signal or+termination fan-out.+Idempotent cron redeploys refresh the Schedule's frozen action metadata while+preserving its current paused state; an ownership check prevents a same-label+Schedule from being overwritten by a different application workflow.+Once `reset` is persisted, duplicate Reset requests return idempotently without+repeating generation-wide cleanup; this prevents an old retry from racing and+terminating a newly started generation.+The editor's live projection follows the same boundary. Reset broadcasts`workflow_runtime_reset`, clears node statuses, variables, and console/chatprojections, and remounts the parameter panel so local outputdiff --git a/server/core/database.py b/server/core/database.pyindex 30c5bcf5..63464bdd 100644--- a/server/core/database.py+++ b/server/core/database.py@@ -3,7 +3,7 @@import jsonimport inspectfrom copy import deepcopy-from datetime import datetime, timezone+from datetime import datetime, timedelta, timezonefrom typing import Awaitable, Callable, Dict, Any, List, Optional, Tuple, Unionfrom pydantic_core import to_jsonable_pythonfrom sqlmodel import SQLModel, select@@ -3078,7 +3078,7 @@ class Database:update(TeamTask).where(TeamTask.id == task_id,- TeamTask.status == "queued",+ TeamTask.status.in_(["queued", "pending"]),or_(TeamTask.assigned_to.is_(None), TeamTask.assigned_to == agent_node_id),).values(@@ -3230,6 +3230,128 @@ class Database:logger.error(f"Failed to fail task: {e}")return False+ async def cancel_team_task(+ self,+ team_id: str,+ task_id: str,+ reason: str,+ ) -> Optional[Dict[str, Any]]:+ """Terminally cancel active delegated work without applying retry policy.++ The state guard makes retries idempotent and prevents cancellation+ from overwriting a result that won the race to a terminal state.+ Active attempts receive the same terminal reason as the task row;+ queued work gets an attempt record as well so every cancellation has+ a durable terminal audit entry.+ """+ active_statuses = ["blocked", "queued", "pending", "running"]+ terminal_statuses = {+ "submitted",+ "accepted",+ "failed",+ "cancelled",+ "skipped",+ "completed",+ }+ cancellation_reason = reason or "Delegated agent cancelled"+ now = datetime.now(timezone.utc)++ async def _cancel(session: AsyncSession) -> Dict[str, Any]:+ changed = await session.execute(+ update(TeamTask)+ .where(+ TeamTask.id == task_id,+ TeamTask.team_id == team_id,+ TeamTask.status.in_(active_statuses),+ )+ .values(+ status="cancelled",+ cancellation_requested=True,+ cancellation_reason=cancellation_reason,+ completed_at=now,+ revision=TeamTask.revision + 1,+ )+ )+ task = (+ await session.execute(+ select(TeamTask).where(+ TeamTask.id == task_id,+ TeamTask.team_id == team_id,+ )+ )+ ).scalar_one_or_none()+ if task is None:+ return {"found": False}++ cancellation_applied = changed.rowcount == 1+ if cancellation_applied:+ attempt_id = f"{task_id}:attempt:{task.current_attempt}"+ attempt = (+ await session.execute(+ select(TeamTaskAttempt).where(+ TeamTaskAttempt.id == attempt_id+ )+ )+ ).scalar_one_or_none()+ if attempt is None:+ session.add(+ TeamTaskAttempt(+ id=attempt_id,+ task_id=task_id,+ team_id=team_id,+ attempt_number=task.current_attempt,+ assignee_node_id=task.assigned_to,+ status="cancelled",+ child_workflow_id=task.child_workflow_id,+ child_run_id=task.child_run_id,+ runner_workflow_id=task.runner_workflow_id,+ runner_run_id=task.runner_run_id,+ parent_workflow_id=task.parent_workflow_id,+ parent_run_id=task.parent_run_id,+ error=cancellation_reason,+ created_at=task.created_at or now,+ started_at=task.started_at,+ completed_at=now,+ )+ )+ else:+ attempt.status = "cancelled"+ attempt.error = cancellation_reason+ attempt.completed_at = now++ if not cancellation_applied and task.status not in terminal_statuses:+ return {+ "found": True,+ "status": task.status,+ "cancellation_applied": False,+ "valid_state": False,+ }++ snapshot = self._team_task_dict(task)+ snapshot["found"] = True+ snapshot["cancellation_applied"] = cancellation_applied+ snapshot["valid_state"] = True+ return snapshot++ try:+ result, _ = await self.run_runtime_mutation(+ resource_type="team_task",+ resource_id=task_id,+ operation="cancel",+ mutate=_cancel,+ )+ if not result.get("found") or not result.get("valid_state", True):+ return None+ return result+ except Exception as e:+ logger.error(+ "Failed to cancel team task",+ team_id=team_id,+ task_id=task_id,+ error=str(e),+ )+ return None+async def get_claimable_tasks(self, team_id: str) -> List[Dict[str, Any]]:"""Get pending tasks with resolved dependencies."""try:@@ -3407,76 +3529,160 @@ class Database:return {"error": str(e)}async def acquire_subagent_permit(self, root_execution_id: str, permit_id: str, limit: int = 3) -> bool:- """Atomically acquire a root-global permit; retries by the owner succeed."""+ """Atomically acquire a root-global permit; retries by the owner succeed.++ Active leases older than the maximum delegated-workflow duration are+ opportunistically reconciled in the same write transaction. This is a+ last-resort capacity repair for workers that died before compensation.+ """if limit < 1:return False+ now = datetime.now(timezone.utc)+ stale_before = now - timedelta(hours=3)+try:- async with self.get_session() as session:+ async def _acquire(session: AsyncSession) -> Dict[str, Any]:+ stale = await session.execute(+ update(SubagentConcurrencyPermit)+ .where(+ SubagentConcurrencyPermit.root_execution_id+ == root_execution_id,+ SubagentConcurrencyPermit.status == "active",+ SubagentConcurrencyPermit.acquired_at < stale_before,+ )+ .values(status="released", released_at=now)+ )+ stale_count = max(int(stale.rowcount or 0), 0)+ if stale_count:+ await session.execute(+ update(SubagentConcurrencyCounter)+ .where(+ SubagentConcurrencyCounter.root_execution_id+ == root_execution_id+ )+ .values(+ active_count=case(+ (+ SubagentConcurrencyCounter.active_count+ >= stale_count,+ SubagentConcurrencyCounter.active_count+ - stale_count,+ ),+ else_=0,+ ),+ updated_at=now,+ )+ )+existing = await session.execute(- select(SubagentConcurrencyPermit).where(SubagentConcurrencyPermit.permit_id == permit_id)+ select(SubagentConcurrencyPermit).where(+ SubagentConcurrencyPermit.permit_id == permit_id+ ))permit = existing.scalar_one_or_none()if permit:- return permit.root_execution_id == root_execution_id and permit.status == "active"+ return {+ "acquired": (+ permit.root_execution_id == root_execution_id+ and permit.status == "active"+ )+ }- await session.execute(text(- "INSERT OR IGNORE INTO subagent_concurrency_counters "- "(root_execution_id, active_count, updated_at) VALUES (:root, 0, CURRENT_TIMESTAMP)"- ), {"root": root_execution_id})+ await session.execute(+ text(+ "INSERT OR IGNORE INTO subagent_concurrency_counters "+ "(root_execution_id, active_count, updated_at) "+ "VALUES (:root, 0, CURRENT_TIMESTAMP)"+ ),+ {"root": root_execution_id},+ )claimed = await session.execute(update(SubagentConcurrencyCounter).where(- SubagentConcurrencyCounter.root_execution_id == root_execution_id,+ SubagentConcurrencyCounter.root_execution_id+ == root_execution_id,SubagentConcurrencyCounter.active_count < limit,).values(active_count=SubagentConcurrencyCounter.active_count + 1,- updated_at=datetime.now(timezone.utc),+ updated_at=now,))if claimed.rowcount != 1:- await session.rollback()- return False- session.add(SubagentConcurrencyPermit(- permit_id=permit_id, root_execution_id=root_execution_id, status="active"- ))- await session.commit()- return True+ return {"acquired": False}+ session.add(+ SubagentConcurrencyPermit(+ permit_id=permit_id,+ root_execution_id=root_execution_id,+ status="active",+ acquired_at=now,+ )+ )+ return {"acquired": True}++ result, _ = await self.run_runtime_mutation(+ resource_type="subagent_concurrency",+ resource_id=root_execution_id,+ operation="acquire",+ mutate=_acquire,+ )+ return bool(result.get("acquired"))except IntegrityError:# A concurrent retry inserted the same permit. Resolve ownership# from durable state rather than incrementing the counter again.async with self.get_session() as session:result = await session.execute(- select(SubagentConcurrencyPermit).where(SubagentConcurrencyPermit.permit_id == permit_id)+ select(SubagentConcurrencyPermit).where(+ SubagentConcurrencyPermit.permit_id == permit_id+ ))permit = result.scalar_one_or_none()- return bool(permit and permit.root_execution_id == root_execution_id and permit.status == "active")+ return bool(+ permit+ and permit.root_execution_id == root_execution_id+ and permit.status == "active"+ )except Exception as e:- logger.error("Failed to acquire subagent permit", root_execution_id=root_execution_id, permit_id=permit_id, error=str(e))+ logger.error(+ "Failed to acquire subagent permit",+ root_execution_id=root_execution_id,+ permit_id=permit_id,+ error=str(e),+ )return Falseasync def release_subagent_permit(self, root_execution_id: str, permit_id: str) -> bool:- """Idempotently release a permit and decrement its root counter once."""+ """Idempotently release a permit and decrement its root counter once.++ A lease that was never created is an expected no-op (not an ownership+ error). A lease owned by another root remains a hard failure.+ """try:async with self.get_session() as session:released = await session.execute(update(SubagentConcurrencyPermit).where(SubagentConcurrencyPermit.permit_id == permit_id,- SubagentConcurrencyPermit.root_execution_id == root_execution_id,+ SubagentConcurrencyPermit.root_execution_id+ == root_execution_id,SubagentConcurrencyPermit.status == "active",)- .values(status="released", released_at=datetime.now(timezone.utc))+ .values(+ status="released",+ released_at=datetime.now(timezone.utc),+ ))if released.rowcount == 1:await session.execute(update(SubagentConcurrencyCounter).where(- SubagentConcurrencyCounter.root_execution_id == root_execution_id,+ SubagentConcurrencyCounter.root_execution_id+ == root_execution_id,SubagentConcurrencyCounter.active_count > 0,).values(- active_count=SubagentConcurrencyCounter.active_count - 1,+ active_count=SubagentConcurrencyCounter.active_count+ - 1,updated_at=datetime.now(timezone.utc),))@@ -3484,16 +3690,26 @@ class Database:return Trueexisting = await session.execute(select(SubagentConcurrencyPermit).where(- SubagentConcurrencyPermit.permit_id == permit_id,- SubagentConcurrencyPermit.root_execution_id == root_execution_id,+ SubagentConcurrencyPermit.permit_id == permit_id))permit = existing.scalar_one_or_none()- already_released = bool(permit and permit.status == "released")+ if permit is None:+ await session.rollback()+ return True+ already_released = bool(+ permit.root_execution_id == root_execution_id+ and permit.status == "released"+ )await session.rollback()return already_releasedexcept Exception as e:- logger.error("Failed to release subagent permit", root_execution_id=root_execution_id, permit_id=permit_id, error=str(e))+ logger.error(+ "Failed to release subagent permit",+ root_execution_id=root_execution_id,+ permit_id=permit_id,+ error=str(e),+ )return False# ============================================================================diff --git a/server/nodes/scheduler/cron_scheduler/_workflow.py b/server/nodes/scheduler/cron_scheduler/_workflow.pyindex b018b29e..605e250e 100644--- a/server/nodes/scheduler/cron_scheduler/_workflow.py+++ b/server/nodes/scheduler/cron_scheduler/_workflow.py@@ -37,6 +37,12 @@ from temporalio.common import WorkflowIDReusePolicyfrom temporalio.workflow import ParentClosePolicy+CHILD_SEARCH_ATTRIBUTES_PATCH = "cron-child-search-attributes-v1"+COOPERATIVE_PAUSE_SCHEDULING_PATCH = (+ "cron-cooperative-pause-scheduling-v1"+)++@workflow.defn(name="CronTriggerWorkflow", sandboxed=False)class CronTriggerWorkflow:"""One-shot per-firing workflow that spawns a child MachinaWorkflow.@@ -46,6 +52,21 @@ class CronTriggerWorkflow:everything else is computed from the static action args."""+ def __init__(self) -> None:+ self._control_paused = False++ @workflow.signal+ async def pause(self) -> None:+ self._control_paused = True++ @workflow.signal+ async def resume(self) -> None:+ self._control_paused = False++ async def _wait_until_resumed(self) -> None:+ if self._control_paused:+ await workflow.wait_condition(lambda: not self._control_paused)+@workflow.runasync def run(self, listener_data: Dict[str, Any]) -> Dict[str, Any]:"""Spawn one MachinaWorkflow child per cron firing.@@ -108,6 +129,36 @@ class CronTriggerWorkflow:firing_iso = trigger_output["timestamp"]child_id = f"{workflow_slug}-{trigger_label}-{firing_iso}"+ try:+ use_child_search_attributes = workflow.patched(+ CHILD_SEARCH_ATTRIBUTES_PATCH+ )+ use_cooperative_pause_schedule_gate = workflow.patched(+ COOPERATIVE_PAUSE_SCHEDULING_PATCH+ )+ except RuntimeError: # direct unit invocation outside Temporal runtime+ use_child_search_attributes = False+ use_cooperative_pause_schedule_gate = False+ from services.temporal.trigger_listener_workflow import (+ event_workflow_search_attributes,+ )++ child_options = {+ "id": child_id,+ "parent_close_policy": ParentClosePolicy.ABANDON,+ "id_reuse_policy": (+ WorkflowIDReusePolicy.ALLOW_DUPLICATE_FAILED_ONLY+ ),+ "execution_timeout": timedelta(hours=1),+ "run_timeout": timedelta(hours=1),+ }+ if use_child_search_attributes:+ child_options["search_attributes"] = (+ event_workflow_search_attributes(deployment_workflow_id)+ )++ if use_cooperative_pause_schedule_gate:+ await self._wait_until_resumed()await workflow.start_child_workflow("MachinaWorkflow",args=[@@ -120,11 +171,7 @@ class CronTriggerWorkflow:"tenant_id": tenant_id,}],- id=child_id,- parent_close_policy=ParentClosePolicy.ABANDON,- id_reuse_policy=WorkflowIDReusePolicy.ALLOW_DUPLICATE_FAILED_ONLY,- execution_timeout=timedelta(hours=1),- run_timeout=timedelta(hours=1),+ **child_options,)workflow.logger.info(f"CronTriggerWorkflow spawned child run: child_id={child_id} " f"timestamp={firing_iso}")@@ -161,4 +208,8 @@ def _build_trigger_output(listener_data: Dict[str, Any]) -> Dict[str, Any]:}-__all__ = ["CronTriggerWorkflow"]+__all__ = [+ "CHILD_SEARCH_ATTRIBUTES_PATCH",+ "COOPERATIVE_PAUSE_SCHEDULING_PATCH",+ "CronTriggerWorkflow",+]diff --git a/server/services/agent_team.py b/server/services/agent_team.pyindex ab34c21f..3889de64 100644--- a/server/services/agent_team.py+++ b/server/services/agent_team.py@@ -359,6 +359,32 @@ class AgentTeamService:return success+ async def cancel_delegation(+ self,+ team_id: str,+ task_id: str,+ reason: str,+ ) -> Optional[Dict[str, Any]]:+ """Terminally cancel delegated work without failing or requeueing it."""+ task = await self.database.cancel_team_task(team_id, task_id, reason)+ if not task:+ return None++ if task.get("status") == "cancelled":+ if task.get("assigned_to"):+ await self.database.update_member_status(+ team_id,+ task["assigned_to"],+ "idle",+ )+ if task.get("cancellation_applied") and self.broadcaster:+ await self.broadcaster.broadcast_team_event(+ team_id,+ "team.task.cancelled",+ task,+ )+ return task+async def get_claimable_tasks(self, team_id: str) -> List[Dict[str, Any]]:"""Get tasks ready to be claimed."""return await self.database.get_claimable_tasks(team_id)diff --git a/server/services/deployment/control.py b/server/services/deployment/control.pyindex 2cea0500..2d00adf9 100644--- a/server/services/deployment/control.py+++ b/server/services/deployment/control.py@@ -72,6 +72,10 @@ class WorkflowControlService:controller_workflow_id=f"workflow-control-{workflow_id}-g{generation}",session_id=session_id, graph_hash=_graph_hash(nodes, edges),graph_snapshot={"nodes": nodes, "edges": edges}, idempotency_key=idempotency_key,+ # Revisions are monotonic across generations so a delayed request+ # from an archived generation cannot pass CAS against a newer one+ # that happens to be at the same lifecycle step.+ revision=(latest.revision + 1) if latest else 0,)try:return await self.database.create_workflow_control(control), Truediff --git a/server/services/deployment/handlers.py b/server/services/deployment/handlers.pyindex 619071f7..8f5acf96 100644--- a/server/services/deployment/handlers.py+++ b/server/services/deployment/handlers.py@@ -42,6 +42,14 @@ from services.deployment.control import WorkflowControlService, serialize_contrologger = get_logger(__name__)+class TemporalControlUnavailable(RuntimeError):+ """A lifecycle Update could not start because no Temporal client exists."""+++class TemporalControlAckMismatch(RuntimeError):+ """Temporal completed an Update without returning the requested state."""++# Per-workflow deployment tasks for proper cancellation (Temporal/n8n pattern).# Maps workflow_id -> asyncio.Task for parallel workflow deployments._deployment_tasks: Dict[str, asyncio.Task] = {}@@ -137,7 +145,12 @@ async def handle_deploy_workflow(data: Dict[str, Any], websocket: WebSocket) ->"locked_at": lock_info.get("locked_at"),}- await broadcaster.update_workflow_status(executing=True, current_node=None, progress=0)+ await broadcaster.update_workflow_status(+ executing=True,+ current_node=None,+ progress=0,+ workflow_id=workflow_id,+ )await broadcaster.update_deployment_status(is_running=True,status="starting",@@ -161,7 +174,12 @@ async def handle_deploy_workflow(data: Dict[str, Any], websocket: WebSocket) ->position = node_data.get("position", 0) if node_data else 0total = node_data.get("total", 1) if node_data else 1progress = int((position / total) * 100) if total > 0 else 0- await broadcaster.update_workflow_status(executing=True, current_node=node_id, progress=progress)+ await broadcaster.update_workflow_status(+ executing=True,+ current_node=node_id,+ progress=progress,+ workflow_id=workflow_id,+ )async def run_deployment():try:@@ -184,6 +202,7 @@ async def handle_deploy_workflow(data: Dict[str, Any], websocket: WebSocket) ->)await broadcaster.unlock_workflow(workflow_id)_deployment_tasks.pop(workflow_id, None)+ return resultelse:await broadcaster.update_deployment_status(is_running=True,@@ -201,6 +220,7 @@ async def handle_deploy_workflow(data: Dict[str, Any], websocket: WebSocket) ->workflow_id=workflow_id,triggers=len(result.get("triggers_setup", [])),)+ return resultexcept Exception as e:logger.error("Deployment task error", workflow_id=workflow_id, error=str(e))@@ -213,6 +233,7 @@ async def handle_deploy_workflow(data: Dict[str, Any], websocket: WebSocket) ->)await broadcaster.unlock_workflow(workflow_id)_deployment_tasks.pop(workflow_id, None)+ return {"success": False, "error": str(e), "workflow_id": workflow_id}_deployment_tasks[workflow_id] = asyncio.create_task(run_deployment())@@ -271,7 +292,12 @@ async def handle_cancel_deployment(data: Dict[str, Any], websocket: WebSocket) -for node_id in result.get("cancelled_listener_node_ids", []):await broadcaster.clear_node_status(node_id)- await broadcaster.update_workflow_status(executing=False, current_node=None, progress=0)+ await broadcaster.update_workflow_status(+ executing=False,+ current_node=None,+ progress=0,+ workflow_id=workflow_id,+ )await broadcaster.update_deployment_status(is_running=False,status="cancelled",@@ -380,12 +406,14 @@ def _control_service():async def _start_controller(control) -> Optional[str]:- """Start the durable Temporal controller when Temporal is available."""+ """Start the durable controller, or use local mode when Temporal is disabled."""from core.container import containerfrom temporalio.common import SearchAttributeKey, SearchAttributePair, TypedSearchAttributeswrapper = container.temporal_client()if wrapper is None or wrapper.client is None:+ if container.settings().temporal_enabled:+ raise RuntimeError("temporal_control_unavailable")return Nonehandle = await wrapper.client.start_workflow("WorkflowControlWorkflow",@@ -398,7 +426,7 @@ async def _start_controller(control) -> Optional[str]:"state": "running",}],id=control.controller_workflow_id,- task_queue="machina-tasks",+ task_queue=container.settings().temporal_task_queue,search_attributes=TypedSearchAttributes([SearchAttributePair(SearchAttributeKey.for_keyword("EventWorkflowId"), control.workflow_id,@@ -408,17 +436,151 @@ async def _start_controller(control) -> Optional[str]:return getattr(handle, "result_run_id", None) or getattr(handle, "first_execution_run_id", None)-async def _signal_controller(control, signal_name: str) -> None:+def _controller_handle(control):from core.container import containerwrapper = container.temporal_client()- if wrapper is not None and wrapper.client is not None and control.controller_workflow_id:- await wrapper.client.get_workflow_handle(- control.controller_workflow_id, run_id=control.controller_run_id- ).signal(signal_name)+ if wrapper is None or wrapper.client is None or not control.controller_workflow_id:+ return None+ return wrapper.client.get_workflow_handle(+ control.controller_workflow_id,+ run_id=control.controller_run_id,+ )++async def _signal_controller(control, signal_name: str) -> None:+ handle = _controller_handle(control)+ if handle is not None:+ await handle.signal(signal_name)+++async def _update_controller_state(+ control,+ requested_state: str,+ *,+ update_id: str,+) -> Optional[Dict[str, Any]]:+ """Apply and await a durable controller state change.++ Temporal-disabled installations retain the local deployment path. When+ Temporal is enabled, losing its client is a failed mutation rather than a+ false-positive pause/resume acknowledgement.+ """+ from core.container import container++ handle = _controller_handle(control)+ if handle is None:+ if container.settings().temporal_enabled:+ raise TemporalControlUnavailable("temporal_control_unavailable")+ return None+ result = await handle.execute_update(+ "set_control_state",+ requested_state,+ id=update_id,+ )+ expected_state = "paused" if requested_state in {"pause", "paused"} else "running"+ if not isinstance(result, dict) or result.get("state") != expected_state:+ raise TemporalControlAckMismatch(+ f"temporal_control_ack_mismatch:{result.get('state') if isinstance(result, dict) else 'missing'}"+ )+ return result-async def _signal_generation_workflows(workflow_id: str, signal_name: str) -> int:++async def _query_controller_state(control) -> Optional[Dict[str, Any]]:+ """Return controller state when reachable; status reads remain resilient."""+ handle = _controller_handle(control)+ if handle is None:+ return None+ try:+ result = await handle.query("status")+ return result if isinstance(result, dict) else None+ except Exception as exc:+ logger.warning(+ "Workflow controller status query failed",+ workflow_id=control.workflow_id,+ controller_workflow_id=control.controller_workflow_id,+ error=str(exc),+ )+ return None+++def _generation_visibility_query(control) -> str:+ """Visibility query for controller descendants and standalone trigger roots."""+ if control.controller_workflow_id:+ root_id = str(control.controller_workflow_id).replace("'", "''")+ workflow_id = str(control.workflow_id).replace("'", "''")+ return (+ f"(RootWorkflowId='{root_id}' OR "+ f"EventWorkflowId='{workflow_id}') "+ "AND ExecutionStatus='Running'"+ )+ workflow_id = str(control.workflow_id).replace("'", "''")+ return f"EventWorkflowId='{workflow_id}' AND ExecutionStatus='Running'"+++def _visibility_literal(value: Any) -> str:+ return str(value).replace("'", "''")+++async def _list_generation_workflows(client, control) -> list[Any]:+ """Resolve tagged roots and every running descendant in those trees.++ Temporal Search Attributes are not inherited by child workflows. The+ first query therefore discovers the controller tree plus tagged standalone+ trigger/graph roots; a second batched RootWorkflowId query expands those+ roots to active Agent/DelegatedTask descendants.+ """+ targets: Dict[tuple[str, str], Any] = {}+ root_ids: set[str] = set()++ async for execution in client.list_workflows(+ query=_generation_visibility_query(control)+ ):+ execution_id = str(execution.id)+ run_id = str(getattr(execution, "run_id", "") or "")+ targets[(execution_id, run_id)] = execution+ root_ids.add(+ str(getattr(execution, "root_id", None) or execution_id)+ )++ ordered_roots = sorted(root_ids)+ batch_size = 40+ for offset in range(0, len(ordered_roots), batch_size):+ batch = ordered_roots[offset : offset + batch_size]+ root_values = ", ".join(+ f"'{_visibility_literal(root_id)}'"+ for root_id in batch+ )+ query = (+ f"RootWorkflowId IN ({root_values}) "+ "AND ExecutionStatus='Running'"+ )+ async for execution in client.list_workflows(query=query):+ execution_id = str(execution.id)+ run_id = str(getattr(execution, "run_id", "") or "")+ targets[(execution_id, run_id)] = execution++ return list(targets.values())+++def _temporal_target_already_gone(exc: Exception) -> bool:+ try:+ from temporalio.service import RPCError, RPCStatusCode++ if isinstance(exc, RPCError) and exc.status == RPCStatusCode.NOT_FOUND:+ return True+ except Exception:+ pass+ message = str(exc).lower()+ return "not found" in message or "already completed" in message+++async def _signal_generation_workflows(+ control,+ signal_name: str,+ *,+ strict: bool = False,+) -> int:"""Best-effort cooperative fan-out to this deployment's live executions.Visibility is discovery only; durable control state remains authoritative.@@ -428,23 +590,28 @@ async def _signal_generation_workflows(workflow_id: str, signal_name: str) -> inwrapper = container.temporal_client()if wrapper is None or wrapper.client is None:+ if strict and container.settings().temporal_enabled:+ raise TemporalControlUnavailable("temporal_control_unavailable")return 0- query = (- f"EventWorkflowId='{workflow_id}' "- "AND ExecutionStatus='Running'"- )signalled = 0+ failures: list[Exception] = []try:- async for execution in wrapper.client.list_workflows(query=query):+ executions = await _list_generation_workflows(+ wrapper.client,+ control,+ )+ for execution in executions:try:await wrapper.client.get_workflow_handle(execution.id, run_id=execution.run_id).signal(signal_name)signalled += 1except Exception as exc:+ if not _temporal_target_already_gone(exc):+ failures.append(exc)logger.warning("Workflow control signal failed",- workflow_id=workflow_id,+ workflow_id=control.workflow_id,temporal_workflow_id=execution.id,signal=signal_name,error=str(exc),@@ -452,59 +619,116 @@ async def _signal_generation_workflows(workflow_id: str, signal_name: str) -> inexcept Exception as exc:logger.warning("Workflow control visibility fan-out failed",- workflow_id=workflow_id,+ workflow_id=control.workflow_id,signal=signal_name,error=str(exc),)+ if strict:+ raise RuntimeError(+ "workflow_signal_visibility_failed"+ ) from exc+ if strict and failures:+ raise RuntimeError(+ f"workflow_signal_failed:{len(failures)}"+ ) from failures[0]return signalled-async def _terminate_generation_workflows(workflow_id: str) -> int:+async def _terminate_generation_workflows(control, *, strict: bool = False) -> int:"""Immediately terminate every visible execution in one application tree."""from core.container import containerwrapper = container.temporal_client()if wrapper is None or wrapper.client is None:+ if strict and container.settings().temporal_enabled:+ raise TemporalControlUnavailable("temporal_control_unavailable")return 0terminated = 0+ failures: list[Exception] = []try:- async for execution in wrapper.client.list_workflows(- query=f"EventWorkflowId='{workflow_id}' AND ExecutionStatus='Running'"- ):+ executions = await _list_generation_workflows(+ wrapper.client,+ control,+ )+ for execution in executions:try:await wrapper.client.get_workflow_handle(execution.id, run_id=execution.run_id).terminate(reason="workflow_reset")terminated += 1except Exception as exc:+ if not _temporal_target_already_gone(exc):+ failures.append(exc)logger.warning("Workflow reset termination failed",- workflow_id=workflow_id,+ workflow_id=control.workflow_id,temporal_workflow_id=execution.id,error=str(exc),)except Exception as exc:logger.warning("Workflow reset visibility scan failed",- workflow_id=workflow_id,+ workflow_id=control.workflow_id,error=str(exc),)+ if strict:+ raise RuntimeError("workflow_visibility_cleanup_failed") from exc+ if strict and failures:+ raise RuntimeError(+ f"workflow_termination_failed:{len(failures)}"+ ) from failures[0]return terminateddef _expected_revision(data: Dict[str, Any], control) -> int:supplied = data.get("expected_revision")- return control.revision if supplied is None else int(supplied)+ if supplied is None:+ raise ValueError("expected_revision_required")+ return int(supplied)-async def _set_cron_pause(workflow_id: str, *, paused: bool) -> int:+async def _set_cron_pause(+ workflow_id: str,+ *,+ paused: bool,+ strict: bool = False,+) -> int:from core.container import containerfrom services.temporal.schedules import set_cron_schedules_pausedwrapper = container.temporal_client()if wrapper is None or wrapper.client is None:+ if strict and container.settings().temporal_enabled:+ raise TemporalControlUnavailable("temporal_control_unavailable")return 0- return await set_cron_schedules_paused(wrapper.client, workflow_id, paused=paused)+ return await set_cron_schedules_paused(+ wrapper.client,+ workflow_id,+ paused=paused,+ strict=strict,+ )+++async def _delete_cron_schedules(+ workflow_id: str,+ *,+ strict: bool = False,+) -> int:+ from core.container import container+ from services.temporal.schedules import (+ delete_cron_schedules_for_deployment,+ )++ wrapper = container.temporal_client()+ if wrapper is None or wrapper.client is None:+ if strict and container.settings().temporal_enabled:+ raise TemporalControlUnavailable("temporal_control_unavailable")+ return 0+ return await delete_cron_schedules_for_deployment(+ wrapper.client,+ workflow_id,+ strict=strict,+ )async def _with_runtime_counts(payload: Dict[str, Any], workflow_id: str) -> Dict[str, Any]:@@ -515,13 +739,240 @@ async def _with_runtime_counts(payload: Dict[str, Any], workflow_id: str) -> Dic**payload,"active_count": status.get("active_runs", 0),"in_flight_count": status.get("active_runs", 0),- "queued_count": 0,+ "queued_count": (+ int(payload.get("queued_count", 0) or 0)+ + int(status.get("queued_events", 0) or 0)+ ),+ }+++def _close_local_admission(workflow_id: str) -> None:+ """Synchronously gate legacy trigger callbacks before durable cleanup."""+ from core.container import container++ container.workflow_service().pause_deployment(workflow_id)+++async def _control_payload(+ control,+ *,+ extra: Optional[Dict[str, Any]] = None,+ controller_status: Optional[Dict[str, Any]] = None,+) -> Dict[str, Any]:+ payload = serialize_control(control)+ if controller_status is not None:+ payload.update({+ "temporal_state": controller_status.get("state"),+ "temporal_revision": controller_status.get("revision"),+ "queued_count": controller_status.get("queued_events", 0),+ "temporal_available": True,+ })+ payload.update(extra or {})+ return await _with_runtime_counts(payload, control.workflow_id)+++async def _broadcast_control(+ control,+ *,+ extra: Optional[Dict[str, Any]] = None,+ controller_status: Optional[Dict[str, Any]] = None,+) -> Dict[str, Any]:+ from services.status_broadcaster import get_status_broadcaster++ payload = await _control_payload(+ control,+ extra=extra,+ controller_status=controller_status,+ )+ await get_status_broadcaster().broadcast({+ "type": "workflow_control_status",+ "workflow_id": control.workflow_id,+ "data": payload,+ })+ return payload+++async def _reconcile_control(service: WorkflowControlService, control):+ """Finish an interrupted DB transition from acknowledged Temporal state."""+ from core.container import container++ controller_status = await _query_controller_state(control)+ transition_target = {+ "pausing": ("paused", "paused"),+ "resuming": ("running", "running"),+ }.get(control.status)+ if transition_target is None:+ return control, controller_status+ requested_state, stable_state = transition_target++ if controller_status is None:+ if container.settings().temporal_enabled:+ return control, None+ if control.status == "pausing":+ container.workflow_service().pause_deployment(control.workflow_id)+ if controller_status is not None and controller_status.get("state") != stable_state:+ controller_status = await _update_controller_state(+ control,+ requested_state,+ update_id=(+ f"reconcile:{control.id}:{control.revision}:{stable_state}"+ ),+ )++ if control.status == "pausing":+ paused_schedules = await _set_cron_pause(+ control.workflow_id,+ paused=True,+ strict=True,+ )+ paused_triggers = await container.workflow_service().update_trigger_pause_status(+ control.workflow_id,+ paused=True,+ )+ signalled = await _signal_generation_workflows(+ control,+ "pause",+ strict=True,+ )+ transition_details = {+ "signalled_executions": signalled,+ "paused_schedules": paused_schedules,+ "paused_triggers": paused_triggers,+ }+ else:+ resumed_schedules = await _set_cron_pause(+ control.workflow_id,+ paused=False,+ strict=True,+ )+ signalled = await _signal_generation_workflows(+ control,+ "resume",+ strict=True,+ )+ queued = await container.workflow_service().resume_deployment(+ control.workflow_id,+ )+ resumed_triggers = await container.workflow_service().update_trigger_pause_status(+ control.workflow_id,+ paused=False,+ )+ transition_details = {+ "resumed_queued_events": queued,+ "signalled_executions": signalled,+ "resumed_schedules": resumed_schedules,+ "resumed_triggers": resumed_triggers,+ }+ try:+ control = await service.transition(+ control,+ expected_revision=control.revision,+ from_statuses={control.status},+ status=stable_state,+ )+ await _broadcast_control(+ control,+ controller_status=controller_status,+ extra=transition_details,+ )+ except ValueError as exc:+ if str(exc) != "control_revision_conflict":+ raise+ latest = await service.database.get_latest_workflow_control(control.workflow_id)+ if latest is not None and latest.generation == control.generation:+ control = latest+ return control, controller_status+++async def _await_deployment_setup(workflow_id: str) -> Dict[str, Any]:+ """Wait for trigger/listener setup started by the legacy deploy handler."""+ task = _deployment_tasks.get(workflow_id)+ if task is None:+ return {+ "success": False,+ "error": "deployment_setup_task_missing",+ "workflow_id": workflow_id,+ }+ result = await asyncio.shield(task)+ if isinstance(result, dict):+ return result+ return {+ "success": False,+ "error": "deployment_setup_did_not_return_status",+ "workflow_id": workflow_id,}+async def _restore_control_after_failed_update(+ service: WorkflowControlService,+ control,+ *,+ transitional_state: str,+ stable_state: str,+):+ """Undo the DB projection when Temporal rejected a lifecycle update."""+ try:+ restored = await service.transition(+ control,+ expected_revision=control.revision,+ from_statuses={transitional_state},+ status=stable_state,+ )+ except ValueError:+ latest = await service.database.get_latest_workflow_control(control.workflow_id)+ restored = latest if latest is not None else control+ await _broadcast_control(restored)+ return restored+++async def _duplicate_start_response(+ control,+ *,+ controller_status: Optional[Dict[str, Any]] = None,+) -> Dict[str, Any]:+ """Report the durable outcome of a retried Start idempotency key."""+ payload = await _control_payload(+ control,+ controller_status=controller_status,+ )+ if control.status == "starting":+ return {+ "success": False,+ "error": "workflow_start_pending",+ "idempotent": True,+ **payload,+ }+ if control.status == "failed":+ return {+ "success": False,+ "error": control.terminal_reason or "workflow_start_failed",+ "idempotent": True,+ **payload,+ }+ return {"success": True, "idempotent": True, **payload}++@ws_handler("workflow_id")async def handle_get_workflow_control_status(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:- return await _with_runtime_counts(await _control_service().get_status(data["workflow_id"]), data["workflow_id"])+ service = _control_service()+ control = await service.database.get_latest_workflow_control(data["workflow_id"])+ if control is None:+ return await _with_runtime_counts(+ serialize_control(None),+ data["workflow_id"],+ )+ control, controller_status = await _reconcile_control(service, control)+ return await _control_payload(+ control,+ controller_status=controller_status,+ extra={+ "temporal_available": (+ controller_status is not None+ if control.controller_run_id+ else False+ ),+ },+ )@ws_handler("workflow_id")@@ -530,12 +981,35 @@ async def handle_start_workflow(data: Dict[str, Any], websocket: WebSocket) -> Dworkflow_id = data["workflow_id"]key = data.get("idempotency_key") or f"start:{workflow_id}:{uuid.uuid4().hex}"service = _control_service()+ existing = await service.database.get_workflow_control_by_idempotency_key(+ workflow_id,+ key,+ )+ if existing is not None:+ existing, controller_status = await _reconcile_control(service, existing)+ return await _duplicate_start_response(+ existing,+ controller_status=controller_status,+ )++ latest = await service.database.get_latest_workflow_control(workflow_id)+ if data.get("expected_revision") is None:+ raise ValueError("expected_revision_required")+ expected_revision = int(data["expected_revision"])+ if expected_revision != (latest.revision if latest else 0):+ raise ValueError("control_revision_conflict")+control, created = await service.begin_generation(workflow_id=workflow_id, nodes=data.get("nodes", []), edges=data.get("edges", []),session_id=data.get("session_id", "default"), idempotency_key=key,)if not created:- return await _with_runtime_counts({"success": True, "idempotent": True, **serialize_control(control)}, workflow_id)+ control, controller_status = await _reconcile_control(service, control)+ return await _duplicate_start_response(+ control,+ controller_status=controller_status,+ )+ await _broadcast_control(control)try:run_id = await _start_controller(control)if run_id:@@ -546,6 +1020,7 @@ async def handle_start_workflow(data: Dict[str, Any], websocket: WebSocket) -> Dawait service.database.update_workflow_run_data_scope(control.data_scope_id or control.execution_id, temporal_run_id=run_id,)+ await _broadcast_control(control)# Runtime persistence is generation-scoped. The caller's session is# retained on the scope for provenance, but must never namespace node# outputs for a controlled run.@@ -557,14 +1032,39 @@ async def handle_start_workflow(data: Dict[str, Any], websocket: WebSocket) -> D}deployed = await handle_deploy_workflow(deploy_data, websocket)if not deployed.get("success"):- await service.fail(control, str(deployed.get("error", "deployment_failed")))- return deployed+ raise RuntimeError(str(deployed.get("error", "deployment_failed")))+ deployed = await _await_deployment_setup(workflow_id)+ if not deployed.get("success"):+ raise RuntimeError(str(deployed.get("error", "deployment_failed")))control = await service.transition(control, expected_revision=control.revision, from_statuses={"starting"}, status="running")- return await _with_runtime_counts({"success": True, **serialize_control(control)}, workflow_id)except Exception as exc:- await service.fail(control, str(exc))+ latest = await service.database.get_latest_workflow_control(workflow_id)+ if latest is not None and latest.generation == control.generation:+ control = latest+ if control.status == "starting":+ control = await service.fail(control, str(exc))+ try:+ await _signal_controller(control, "reset")+ except Exception as reset_exc:+ logger.warning(+ "Failed to reset controller after deployment setup error",+ workflow_id=workflow_id,+ error=str(reset_exc),+ )+ await _terminate_generation_workflows(control)+ await _broadcast_control(control)raise+ # Reaching ``running`` commits successful deployment setup. Projection+ # failures after that point must not tear down a live durable generation;+ # the client can recover the committed state through its status resync.+ controller_status = await _query_controller_state(control)+ payload = await _broadcast_control(+ control,+ controller_status=controller_status,+ )+ return {"success": True, **payload}+@ws_handler("workflow_id")async def handle_pause_workflow(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:@@ -575,20 +1075,64 @@ async def handle_pause_workflow(data: Dict[str, Any], websocket: WebSocket) -> Dcontrol = await service.database.get_latest_workflow_control(workflow_id)if control is None:return {"success": False, "error": "workflow_never_started"}+ control, controller_status = await _reconcile_control(service, control)+ if control.status == "paused":+ return {+ "success": True,+ "idempotent": True,+ **await _control_payload(control, controller_status=controller_status),+ }+ if control.status == "pausing":+ return {+ "success": False,+ "error": "workflow_control_transition_pending",+ **await _control_payload(control, controller_status=controller_status),+ }control = await service.transition(control, expected_revision=_expected_revision(data, control), from_statuses={"running"}, status="pausing")+ await _broadcast_control(control)container.workflow_service().pause_deployment(workflow_id)- await _signal_controller(control, "pause")- paused_schedules = await _set_cron_pause(workflow_id, paused=True)+ try:+ controller_status = await _update_controller_state(+ control,+ "paused",+ update_id=(+ f"{control.id}:{control.revision}:"+ f"{data.get('idempotency_key') or uuid.uuid4().hex}:paused"+ ),+ )+ except (TemporalControlUnavailable, TemporalControlAckMismatch):+ await container.workflow_service().resume_deployment(workflow_id)+ await _restore_control_after_failed_update(+ service,+ control,+ transitional_state="pausing",+ stable_state="running",+ )+ raise+ paused_schedules = await _set_cron_pause(+ workflow_id,+ paused=True,+ strict=True,+ )paused_triggers = await container.workflow_service().update_trigger_pause_status(workflow_id, paused=True)- signalled = await _signal_generation_workflows(workflow_id, "pause")+ signalled = await _signal_generation_workflows(+ control,+ "pause",+ strict=True,+ )control = await service.transition(control, expected_revision=control.revision, from_statuses={"pausing"}, status="paused")- return await _with_runtime_counts({+ payload = await _broadcast_control(control, controller_status=controller_status, extra={+ "signalled_executions": signalled,+ "paused_schedules": paused_schedules,+ "paused_triggers": paused_triggers,+ })+ return {"success": True, "signalled_executions": signalled,"paused_schedules": paused_schedules, "paused_triggers": paused_triggers,- **serialize_control(control),- }, workflow_id)+ **payload,+ }@ws_handler("workflow_id")@@ -600,20 +1144,64 @@ async def handle_resume_workflow(data: Dict[str, Any], websocket: WebSocket) ->control = await service.database.get_latest_workflow_control(workflow_id)if control is None:return {"success": False, "error": "workflow_never_started"}+ control, controller_status = await _reconcile_control(service, control)+ if control.status == "running":+ return {+ "success": True,+ "idempotent": True,+ **await _control_payload(control, controller_status=controller_status),+ }+ if control.status == "resuming":+ return {+ "success": False,+ "error": "workflow_control_transition_pending",+ **await _control_payload(control, controller_status=controller_status),+ }control = await service.transition(control, expected_revision=_expected_revision(data, control), from_statuses={"paused"}, status="resuming")- await _signal_controller(control, "resume")- resumed_schedules = await _set_cron_pause(workflow_id, paused=False)- signalled = await _signal_generation_workflows(workflow_id, "resume")+ await _broadcast_control(control)+ try:+ controller_status = await _update_controller_state(+ control,+ "running",+ update_id=(+ f"{control.id}:{control.revision}:"+ f"{data.get('idempotency_key') or uuid.uuid4().hex}:running"+ ),+ )+ except (TemporalControlUnavailable, TemporalControlAckMismatch):+ await _restore_control_after_failed_update(+ service,+ control,+ transitional_state="resuming",+ stable_state="paused",+ )+ raise+ resumed_schedules = await _set_cron_pause(+ workflow_id,+ paused=False,+ strict=True,+ )+ signalled = await _signal_generation_workflows(+ control,+ "resume",+ strict=True,+ )queued = await container.workflow_service().resume_deployment(workflow_id)resumed_triggers = await container.workflow_service().update_trigger_pause_status(workflow_id, paused=False)control = await service.transition(control, expected_revision=control.revision, from_statuses={"resuming"}, status="running")- return await _with_runtime_counts({+ payload = await _broadcast_control(control, controller_status=controller_status, extra={+ "resumed_queued_events": queued,+ "signalled_executions": signalled,+ "resumed_schedules": resumed_schedules,+ "resumed_triggers": resumed_triggers,+ })+ return {"success": True, "resumed_queued_events": queued, "signalled_executions": signalled,"resumed_schedules": resumed_schedules, "resumed_triggers": resumed_triggers,- **serialize_control(control),- }, workflow_id)+ **payload,+ }@ws_handler("workflow_id")@@ -621,35 +1209,97 @@ async def handle_reset_workflow(data: Dict[str, Any], websocket: WebSocket) -> Dworkflow_id = data["workflow_id"]service = _control_service()current = await service.database.get_latest_workflow_control(workflow_id)- if current is not None:+ if current is None:+ return await _with_runtime_counts(+ await service.get_status(workflow_id),+ workflow_id,+ )++ # ``reset`` is a completed cleanup barrier. Re-running generation-wide+ # sweeps here would race a concurrent Start and could terminate resources+ # from the next generation because standalone triggers use the stable+ # application workflow id.+ if current.status == "reset":+ return {+ "success": True,+ "idempotent": True,+ **await _control_payload(current),+ }++ if current.status != "resetting":current = await service.transition(current, expected_revision=_expected_revision(data, current),from_statuses={"starting", "running", "pausing", "paused", "resuming", "failed"}, status="resetting",values={"terminal_reason": "workflow_reset", "completed_at": datetime.now(timezone.utc)},)- try:- await _signal_controller(current, "reset")- finally:- terminated = await _terminate_generation_workflows(workflow_id)- await handle_cancel_deployment({"workflow_id": workflow_id}, websocket)- else:- terminated = 0- if current is None:- return await _with_runtime_counts(await service.get_status(workflow_id), workflow_id)- current = await service.transition(- current, expected_revision=current.revision, from_statuses={"resetting"}, status="reset",- values={"terminal_reason": "workflow_reset", "completed_at": datetime.now(timezone.utc)},++ # Quiesce every producer before the final execution sweep. The local gate+ # is synchronous, so no callback can be admitted while broadcasts or+ # Temporal cleanup yield control.+ _close_local_admission(workflow_id)+ await _broadcast_control(current)++ try:+ await _signal_controller(current, "reset")+ except Exception as exc:+ logger.warning(+ "Workflow reset signal failed; continuing with termination",+ workflow_id=workflow_id,+ error=str(exc),+ )++ # Remove cron producers before terminating executions; otherwise a firing+ # between the execution scan and schedule deletion could survive Reset.+ deleted_schedules = await _delete_cron_schedules(+ workflow_id,+ strict=True,)- await service.database.update_workflow_run_data_scope(+ cancelled = await handle_cancel_deployment(+ {"workflow_id": workflow_id},+ websocket,+ )+ # Durable Temporal cleanup above is authoritative. The process-local+ # deployment may legitimately be absent after an API-server restart.+ # Any other local teardown failure can leave a listener/admission path+ # alive, so keep the durable control in ``resetting`` for a safe retry.+ local_cleanup_completed = bool(cancelled.get("success"))+ if not local_cleanup_completed:+ # ``handle_cancel_deployment`` retains its historical envelope and+ # exposes manager errors as ``message``. Accept ``error`` as well for+ # direct/internal callers and future wire compatibility.+ local_error = str(+ cancelled.get("error")+ or cancelled.get("message")+ or "unknown"+ )+ expected_absent_error = f"Workflow {workflow_id} is not deployed"+ if local_error != expected_absent_error:+ raise RuntimeError(+ f"workflow_local_cleanup_failed:{local_error}"+ )++ # Controller, cron, and legacy local admission paths are now closed. This+ # final strict sweep therefore observes a fixed set of generation runs.+ terminated = await _terminate_generation_workflows(current, strict=True)++ archived = await service.database.update_workflow_run_data_scope(current.data_scope_id or current.execution_id,status="archived", archived_at=datetime.now(timezone.utc),)+ if not archived:+ raise RuntimeError("workflow_data_scope_archive_failed")+from services.status_broadcaster import get_status_broadcasterfrom services.deployment.runtime_state import archive_and_reset_node_statebroadcaster = get_status_broadcaster()node_state = await archive_and_reset_node_state(current, service.database, broadcaster,)++ current = await service.transition(+ current, expected_revision=current.revision, from_statuses={"resetting"}, status="reset",+ values={"terminal_reason": "workflow_reset", "completed_at": datetime.now(timezone.utc)},+ )await broadcaster.broadcast({"type": "workflow_runtime_reset","workflow_id": workflow_id,@@ -658,16 +1308,23 @@ async def handle_reset_workflow(data: Dict[str, Any], websocket: WebSocket) -> D"archived_nodes": node_state["archived_nodes"],"reset_nodes": node_state["reset_nodes"],})- return await _with_runtime_counts(- {- "success": True,- "terminated_executions": terminated,- "archived_nodes": node_state["archived_nodes"],- "reset_nodes": node_state["reset_nodes"],- **serialize_control(current),- },- workflow_id,- )+ payload = await _broadcast_control(current, extra={+ "terminated_executions": terminated,+ "deleted_schedules": deleted_schedules,+ "local_cleanup_completed": local_cleanup_completed,+ "archived_nodes": node_state["archived_nodes"],+ "reset_nodes": node_state["reset_nodes"],+ })+ return {+ "success": True,+ "idempotent": False,+ "terminated_executions": terminated,+ "deleted_schedules": deleted_schedules,+ "local_cleanup_completed": local_cleanup_completed,+ "archived_nodes": node_state["archived_nodes"],+ "reset_nodes": node_state["reset_nodes"],+ **payload,+ }WS_HANDLERS: Dict[str, Any] = {diff --git a/server/services/deployment/manager.py b/server/services/deployment/manager.pyindex 0df370fd..a2eada8b 100644--- a/server/services/deployment/manager.py+++ b/server/services/deployment/manager.py@@ -117,6 +117,11 @@ class DeploymentManager:await self._spawn_run(trigger_node_id, trigger_data, workflow_id=workflow_id)return len(queued)+ def _clear_paused_state(self, workflow_id: str) -> None:+ """Drop admission and queued-event state for a cancelled generation."""+ self._paused_workflows.discard(workflow_id)+ self._paused_events.pop(workflow_id, None)+async def update_trigger_pause_status(self, workflow_id: str, *, paused: bool) -> int:"""Reflect deployment admission state on every armed trigger node."""state = self._deployments.get(workflow_id)@@ -263,11 +268,18 @@ class DeploymentManager:# Find workflow to cancelif workflow_id:if not self.is_workflow_deployed(workflow_id):+ # Reset/cancel callers may arrive after deployment teardown.+ # With no live admission path, stale paused events are safe to+ # discard immediately.+ self._clear_paused_state(workflow_id)return {"success": False, "error": f"Workflow {workflow_id} is not deployed"}else:# Backward compatibility: cancel first running deploymentdeployed = self.get_deployed_workflows()if not deployed:+ # Any pause entries without a live deployment are stale.+ self._paused_workflows.clear()+ self._paused_events.clear()return {"success": False, "error": "No deployment running"}workflow_id = deployed[0]@@ -278,6 +290,11 @@ class DeploymentManager:deployment_id = state.deployment_idlogger.info("Cancelling deployment", deployment_id=deployment_id, workflow_id=workflow_id)+ # Close admission before taking the active-run snapshot. Keep this+ # barrier set until every teardown step succeeds; otherwise a callback+ # racing cancellation could launch an untracked run.+ self._paused_workflows.add(workflow_id)+# Get trigger manager for this workflowtrigger_manager = self._trigger_managers.get(workflow_id)if trigger_manager:@@ -355,6 +372,7 @@ class DeploymentManager:self._active_runs.pop(workflow_id, None)self._run_counters.pop(workflow_id, None)self._status_callbacks.pop(workflow_id, None)+ self._clear_paused_state(workflow_id)return {"success": True,@@ -389,6 +407,7 @@ class DeploymentManager:"active_runs": len(execution_runs),"active_listeners": len(workflow_runs) - len(execution_runs),"run_counter": self._run_counters.get(workflow_id, 0),+ "queued_events": len(self._paused_events.get(workflow_id, [])),"deployed_at": state.deployed_at,}diff --git a/server/services/temporal/agent_activities.py b/server/services/temporal/agent_activities.pyindex 3bda5366..498eafbb 100644--- a/server/services/temporal/agent_activities.py+++ b/server/services/temporal/agent_activities.py@@ -1456,6 +1456,7 @@ async def clear_agent_skills(payload: Dict[str, Any]) -> Dict[str, Any]:async def begin_agent_delegation(payload: Dict[str, Any]) -> Dict[str, Any]:"""Idempotently persist and claim a delegation before child startup."""from services.agent_team import get_agent_team_service+ from temporalio.exceptions import ApplicationErrorteam_id = str(payload.get("team_id") or "")task_id = str(payload.get("team_task_id") or "")@@ -1476,14 +1477,39 @@ async def begin_agent_delegation(payload: Dict[str, Any]) -> Dict[str, Any]:if not task:raise RuntimeError("Failed to persist delegated team task")- if str(task.get("status") or "") in {"pending", "queued", ""}:+ terminal_statuses = {+ "submitted",+ "accepted",+ "failed",+ "cancelled",+ "skipped",+ "completed",+ }++ def raise_terminal(status: str) -> None:+ raise ApplicationError(+ f"Delegated team task {task_id} is already terminal ({status})",+ type="DelegationTaskTerminal",+ non_retryable=True,+ )++ status = str(task.get("status") or "")+ if status in {"pending", "queued"}:claimed = await service.claim_task(team_id, task_id, child_id)- if not claimed:- tasks = await service.database.get_team_tasks(team_id)- task = next((item for item in tasks if item.get("id") == task_id), None)- if not task or task.get("assigned_to") != child_id:- raise RuntimeError("Failed to claim delegated team task")- elif task.get("assigned_to") and task.get("assigned_to") != child_id:+ tasks = await service.database.get_team_tasks(team_id)+ task = next((item for item in tasks if item.get("id") == task_id), None)+ status = str((task or {}).get("status") or "")+ if status in terminal_statuses:+ raise_terminal(status)+ if (+ not claimed+ or status != "running"+ or task.get("assigned_to") != child_id+ ):+ raise RuntimeError("Failed to claim delegated team task")+ elif status in terminal_statuses:+ raise_terminal(status)+ elif status != "running" or task.get("assigned_to") != child_id:raise RuntimeError("Delegated team task is claimed by another agent")message = await service.send_message(@@ -1499,6 +1525,21 @@ async def begin_agent_delegation(payload: Dict[str, Any]) -> Dict[str, Any]:)if not message:raise RuntimeError("Failed to persist delegation assignment event")++ # Cancellation may commit while the idempotent assignment message is+ # being persisted. Re-read immediately before returning permission to+ # start the child so a terminal task is never reported as claimed.+ tasks = await service.database.get_team_tasks(team_id)+ task = next((item for item in tasks if item.get("id") == task_id), None)+ status = str((task or {}).get("status") or "")+ if status in terminal_statuses:+ raise_terminal(status)+ if (+ not task+ or status != "running"+ or task.get("assigned_to") != child_id+ ):+ raise RuntimeError("Delegated team task lost its claim before child startup")return {"team_id": team_id, "team_task_id": task_id, "claimed": True}@@ -1535,6 +1576,74 @@ async def queue_agent_delegation(payload: Dict[str, Any]) -> Dict[str, Any]:return {"team_id": team_id, "team_task_id": task_id, "status": "queued"}[email protected](name="agent.cancel_delegation.v1")+async def cancel_agent_delegation(payload: Dict[str, Any]) -> Dict[str, Any]:+ """Persist cancellation as a terminal state without failure requeueing."""+ from services.agent_team import get_agent_team_service++ team_id = str(payload.get("team_id") or "")+ task_id = str(payload.get("team_task_id") or "")+ child_id = str(payload.get("child_agent_node_id") or "")+ parent_id = str(payload.get("parent_agent_node_id") or "")+ reason = str(+ payload.get("reason")+ or payload.get("error")+ or "Delegated agent cancelled"+ )+ if not all((team_id, task_id, child_id, parent_id)):+ raise ValueError("Incomplete delegation cancellation payload")++ service = get_agent_team_service()+ task = await service.cancel_delegation(team_id, task_id, reason)+ if task is None:+ raise RuntimeError("Failed to cancel delegated team task")++ status = str(task.get("status") or "")+ cancellation_applied = bool(task.get("cancellation_applied"))+ if status == "cancelled":+ persisted_reason = str(task.get("cancellation_reason") or reason)+ message = await service.send_message(+ team_id,+ child_id,+ f"Task {task_id} cancelled: {persisted_reason}",+ to_agent=parent_id,+ message_type="error",+ event_id=str(+ payload.get("terminal_event_id") or f"{task_id}:cancelled"+ ),+ extra_data={+ "status": "cancelled",+ "task_id": task_id,+ "reason": persisted_reason,+ "root_execution_id": payload.get("root_execution_id"),+ "trace_id": payload.get("trace_id"),+ },+ )+ if not message:+ raise RuntimeError("Failed to persist delegation cancellation event")+ else:+ persisted_reason = None++ return {+ "team_id": team_id,+ "team_task_id": task_id,+ "status": status,+ "cancelled": status == "cancelled",+ "cancellation_applied": cancellation_applied,+ "reason": persisted_reason,+ }+++def _subagent_lease_id(+ permit_id: str,+ temporal_activity_id: str,+ attempt: int,+) -> str:+ """Return the bounded physical lease identity for one activity attempt."""+ identity = f"{permit_id}\0{temporal_activity_id}\0{attempt}".encode()+ return f"subagent-lease-v2-{hashlib.sha256(identity).hexdigest()}"++@activity.defn(name="agent.acquire_subagent_permit.v1")async def acquire_subagent_permit(payload: Dict[str, Any]) -> Dict[str, Any]:"""Poll the durable root coordinator until this delegation is admitted."""@@ -1543,14 +1652,91 @@ async def acquire_subagent_permit(payload: Dict[str, Any]) -> Dict[str, Any]:root_id = str(payload.get("root_execution_id") or "")permit_id = str(payload.get("permit_id") or "")limit = max(1, int(payload.get("limit") or 3))+ lease_version = int(payload.get("lease_version") or 1)if not root_id or not permit_id:raise ValueError("root_execution_id and permit_id are required")+ if lease_version not in {1, 2}:+ raise ValueError("lease_version must be 1 or 2")+service = get_agent_team_service()- while True:- if await service.acquire_subagent_permit(root_id, permit_id, limit):- return {"root_execution_id": root_id, "permit_id": permit_id, "acquired": True}- activity.heartbeat({"root_execution_id": root_id, "permit_id": permit_id, "status": "queued"})- await asyncio.sleep(1)+ lease_id = permit_id+ attempt = 1+ if lease_version == 2:+ info = activity.info()+ temporal_activity_id = str(info.activity_id)+ attempt = max(1, int(info.attempt))+ lease_id = _subagent_lease_id(+ permit_id,+ temporal_activity_id,+ attempt,+ )+ for prior_attempt in range(1, attempt):+ prior_lease_id = _subagent_lease_id(+ permit_id,+ temporal_activity_id,+ prior_attempt,+ )+ if not await service.release_subagent_permit(+ root_id,+ prior_lease_id,+ ):+ raise RuntimeError(+ "Failed to reconcile prior subagent permit lease "+ f"{prior_lease_id}"+ )++ try:+ while True:+ if await service.acquire_subagent_permit(root_id, lease_id, limit):+ result = {+ "root_execution_id": root_id,+ "permit_id": permit_id,+ "acquired": True,+ }+ if lease_version == 2:+ result.update({+ "lease_version": 2,+ "lease_id": lease_id,+ "attempt": attempt,+ })+ return result++ heartbeat = {+ "root_execution_id": root_id,+ "permit_id": permit_id,+ "status": "queued",+ }+ if lease_version == 2:+ heartbeat.update({+ "lease_version": 2,+ "lease_id": lease_id,+ "attempt": attempt,+ })+ activity.heartbeat(heartbeat)+ await asyncio.sleep(1)+ except asyncio.CancelledError:+ # The database commit may have completed even when cancellation wins+ # the await race and the activity result never reaches workflow+ # history. Always compensate the attempted permit locally. Releasing+ # a permit that was never acquired is an expected no-op here.+ try:+ compensated = await service.release_subagent_permit(+ root_id,+ lease_id,+ )+ if not compensated:+ raise RuntimeError("Permit compensation was rejected")+ except Exception as release_exc: # noqa: BLE001+ logger.error(+ "Failed to compensate cancelled subagent permit acquisition",+ extra={+ "root_execution_id": root_id,+ "permit_id": permit_id,+ "lease_id": lease_id,+ "error": str(release_exc),+ },+ )+ raise@activity.defn(name="agent.release_subagent_permit.v1")@@ -1560,12 +1746,24 @@ async def release_subagent_permit(payload: Dict[str, Any]) -> Dict[str, Any]:root_id = str(payload.get("root_execution_id") or "")permit_id = str(payload.get("permit_id") or "")- if not root_id or not permit_id:+ lease_id = str(payload.get("lease_id") or permit_id)+ lease_version = int(payload.get("lease_version") or (2 if payload.get("lease_id") else 1))+ if not root_id or not permit_id or not lease_id:raise ValueError("root_execution_id and permit_id are required")- released = await get_agent_team_service().release_subagent_permit(root_id, permit_id)+ released = await get_agent_team_service().release_subagent_permit(+ root_id,+ lease_id,+ )if not released:raise RuntimeError("Failed to release subagent permit")- return {"root_execution_id": root_id, "permit_id": permit_id, "released": True}+ result = {+ "root_execution_id": root_id,+ "permit_id": permit_id,+ "released": True,+ }+ if lease_version == 2:+ result.update({"lease_version": 2, "lease_id": lease_id})+ return result@activity.defn(name="agent.finish_delegation.v1")diff --git a/server/services/temporal/agent_workflow.py b/server/services/temporal/agent_workflow.pyindex 7a73bb20..c8ed3744 100644--- a/server/services/temporal/agent_workflow.py+++ b/server/services/temporal/agent_workflow.py@@ -52,7 +52,7 @@ from typing import Any, Dict, List, Optionalfrom temporalio import workflowfrom temporalio.common import RetryPolicy # kept for type hints-from temporalio.workflow import ParentClosePolicy+from temporalio.workflow import ActivityCancellationType, ParentClosePolicyfrom services.node_registry import get_node_class@@ -105,12 +105,33 @@ def _default_max_iterations() -> int:# non_retryable_error_types include ``NodeUserError`` — user-correctable# failures inside the LLM step fail fast instead of burning 3 retries.AGENT_ACTIVITY_RETRY: RetryPolicy = DEFAULT_ACTIVITY_RETRY+DELEGATION_CLEANUP_RETRY: RetryPolicy = RetryPolicy(+ initial_interval=timedelta(seconds=1),+ maximum_interval=timedelta(seconds=30),+ maximum_attempts=10,+)# Temporal patch marker for per-call command identity and duplicate-name# validation. Existing histories must retain their recorded activity/child# ids and last-wins tool index; new histories take the isolated path.TOOL_CALL_IDENTITY_V2_PATCH = "agent-tool-call-identity-v2"TASK_MANAGER_DELEGATION_PATCH = "agent-task-manager-delegation-v1"+COOPERATIVE_PAUSE_SCHEDULING_PATCH = (+ "agent-cooperative-pause-scheduling-v1"+)+DELEGATED_TASK_COOPERATIVE_PAUSE_SCHEDULING_PATCH = (+ "delegated-task-cooperative-pause-scheduling-v1"+)+DELEGATION_PERMIT_CLEANUP_PATCH = "agent-delegation-permit-cleanup-v1"+DELEGATION_ACQUIRE_CANCELLATION_PATCH = (+ "agent-delegation-acquire-cancellation-v1"+)+DELEGATED_TASK_CANCELLATION_TERMINAL_PATCH = (+ "delegated-task-cancellation-terminal-v1"+)+DELEGATED_TASK_SEARCH_ATTRIBUTES_PATCH = (+ "agent-delegated-task-search-attributes-v1"+)DUPLICATE_TOOL_NAME_ERROR_TYPE = "DuplicateToolNameError"@@ -298,10 +319,28 @@ class DelegatedTaskWorkflow:@workflow.runasync def run(self, request: Dict[str, Any]) -> Dict[str, Any]:+ use_cooperative_pause_schedule_gate = workflow.patched(+ DELEGATED_TASK_COOPERATIVE_PAUSE_SCHEDULING_PATCH+ )+ use_acquire_cancellation_completion = workflow.patched(+ DELEGATION_ACQUIRE_CANCELLATION_PATCH+ )+ use_cancellation_terminal = workflow.patched(+ DELEGATED_TASK_CANCELLATION_TERMINAL_PATCH+ )+ acquire_cancellation_options = (+ {+ "cancellation_type": (+ ActivityCancellationType.WAIT_CANCELLATION_COMPLETED+ )+ }+ if use_acquire_cancellation_completion+ else {}+ )lifecycle = request["lifecycle"]task_id = lifecycle["team_task_id"]root_id = lifecycle["root_execution_id"]- acquired = False+ acquired_permit_id: Optional[str] = Nonebegan = Falsetry:await self._wait_until_resumed()@@ -313,23 +352,43 @@ class DelegatedTaskWorkflow:"child_workflow_id": request["child_workflow_id"]}],start_to_close_timeout=PERSIST_TURN_TIMEOUT,retry_policy=AGENT_ACTIVITY_RETRY,+ **acquire_cancellation_options,)- await workflow.execute_activity(+ if use_cooperative_pause_schedule_gate:+ await self._wait_until_resumed()+ acquire_payload = {+ "root_execution_id": root_id,+ "permit_id": task_id,+ "limit": request.get("limit", 3),+ }+ if use_acquire_cancellation_completion:+ acquire_payload["lease_version"] = 2+ acquire_result = await workflow.execute_activity("agent.acquire_subagent_permit.v1",- args=[{"root_execution_id": root_id, "permit_id": task_id,- "limit": request.get("limit", 3)}],+ args=[acquire_payload],start_to_close_timeout=timedelta(hours=1),heartbeat_timeout=timedelta(seconds=10),retry_policy=AGENT_ACTIVITY_RETRY,+ **acquire_cancellation_options,+ )+ acquired_permit_id = str(+ (acquire_result or {}).get("lease_id")+ or task_id)- acquired = True+ if use_cooperative_pause_schedule_gate:+ await self._wait_until_resumed()await workflow.execute_activity("agent.begin_delegation.v1", args=[lifecycle],start_to_close_timeout=PERSIST_TURN_TIMEOUT,retry_policy=AGENT_ACTIVITY_RETRY,+ **acquire_cancellation_options,)began = Trueawait self._wait_until_resumed()+ if use_acquire_cancellation_completion:+ request["child_context"]["team_permit_id"] = (+ acquired_permit_id+ )child_handle = await workflow.start_child_workflow("AgentWorkflow", args=[request["child_context"]],id=request["child_workflow_id"],@@ -355,6 +414,74 @@ class DelegatedTaskWorkflow:start_to_close_timeout=PERSIST_TURN_TIMEOUT,retry_policy=AGENT_ACTIVITY_RETRY,)+ except asyncio.CancelledError:+ if use_cancellation_terminal:+ if began:+ try:+ await workflow.execute_activity(+ "agent.cancel_delegation.v1",+ args=[{+ **lifecycle,+ "reason": (+ "Delegated task workflow cancelled"+ ),+ "terminal_event_id": f"{task_id}:terminal",+ }],+ activity_id="cancel-delegation",+ start_to_close_timeout=PERSIST_TURN_TIMEOUT,+ retry_policy=DELEGATION_CLEANUP_RETRY,+ **acquire_cancellation_options,+ )+ except Exception as cleanup_exc: # noqa: BLE001+ workflow.logger.error(+ "DelegatedTaskWorkflow failed to persist "+ f"cancellation for {task_id}: {cleanup_exc}"+ )+ else:+ # Queue/acquire/begin may have completed even when their+ # result lost the cancellation race. The dedicated+ # transition is idempotent for absent/already-terminal+ # tasks and never invokes the normal failure/requeue path.+ try:+ await workflow.execute_activity(+ "agent.cancel_delegation.v1",+ args=[{+ **lifecycle,+ "reason": (+ "Delegated task workflow cancelled"+ ),+ "terminal_event_id": f"{task_id}:terminal",+ }],+ activity_id="cancel-delegation-before-begin",+ start_to_close_timeout=PERSIST_TURN_TIMEOUT,+ retry_policy=DELEGATION_CLEANUP_RETRY,+ **acquire_cancellation_options,+ )+ except Exception as cleanup_exc: # noqa: BLE001+ workflow.logger.error(+ "DelegatedTaskWorkflow failed to persist early "+ f"cancellation for {task_id}: {cleanup_exc}"+ )+ if acquired_permit_id:+ try:+ await workflow.execute_activity(+ "agent.release_subagent_permit.v1",+ args=[{+ "root_execution_id": root_id,+ "permit_id": acquired_permit_id,+ }],+ activity_id="release-permit-cancelled",+ start_to_close_timeout=PERSIST_TURN_TIMEOUT,+ retry_policy=DELEGATION_CLEANUP_RETRY,+ **acquire_cancellation_options,+ )+ acquired_permit_id = None+ except Exception as cleanup_exc: # noqa: BLE001+ workflow.logger.error(+ "DelegatedTaskWorkflow failed to release "+ f"cancelled permit {task_id}: {cleanup_exc}"+ )+ raiseexcept Exception as exc:if began:await workflow.execute_activity(@@ -367,13 +494,33 @@ class DelegatedTaskWorkflow:)raisefinally:- if acquired:- await workflow.execute_activity(- "agent.release_subagent_permit.v1",- args=[{"root_execution_id": root_id, "permit_id": task_id}],- start_to_close_timeout=PERSIST_TURN_TIMEOUT,- retry_policy=AGENT_ACTIVITY_RETRY,+ if acquired_permit_id:+ release_options = (+ {+ "activity_id": "release-permit-final",+ "retry_policy": DELEGATION_CLEANUP_RETRY,+ }+ if use_cancellation_terminal+ else {"retry_policy": AGENT_ACTIVITY_RETRY})+ try:+ await workflow.execute_activity(+ "agent.release_subagent_permit.v1",+ args=[{+ "root_execution_id": root_id,+ "permit_id": acquired_permit_id,+ }],+ start_to_close_timeout=PERSIST_TURN_TIMEOUT,+ **acquire_cancellation_options,+ **release_options,+ )+ except Exception as release_exc:+ if not use_cancellation_terminal:+ raise+ workflow.logger.error(+ "DelegatedTaskWorkflow final permit release failed "+ f"for {task_id}: {release_exc}"+ )@workflow.defn(sandboxed=False, name="AgentWorkflow")@@ -477,6 +624,24 @@ class AgentWorkflow:# deterministic duplicate-name validation.use_tool_call_identity_v2 = workflow.patched(TOOL_CALL_IDENTITY_V2_PATCH)use_task_manager_delegation = workflow.patched(TASK_MANAGER_DELEGATION_PATCH)+ use_cooperative_pause_schedule_gate = workflow.patched(+ COOPERATIVE_PAUSE_SCHEDULING_PATCH+ )+ use_delegation_permit_cleanup = workflow.patched(+ DELEGATION_PERMIT_CLEANUP_PATCH+ )+ use_acquire_cancellation_completion = workflow.patched(+ DELEGATION_ACQUIRE_CANCELLATION_PATCH+ )+ acquire_cancellation_options = (+ {+ "cancellation_type": (+ ActivityCancellationType.WAIT_CANCELLATION_COMPLETED+ )+ }+ if use_acquire_cancellation_completion+ else {}+ )# ---- Step 0: Resolve payload via the prep activity --------------# DB lookups + edge walking + tool schema build happen here, NOT@@ -648,6 +813,8 @@ class AgentWorkflow:max_iterations,phase="llm_step",)+ if use_cooperative_pause_schedule_gate:+ await self._wait_until_resumed()# Strip per-turn fields the activity doesn't need. Native turns# receive only provider-neutral JSON tool definitions. Legacy@@ -862,6 +1029,11 @@ class AgentWorkflow:break# ---- Schedule tool activities -------------------------------+ if use_cooperative_pause_schedule_gate:+ # The LLM activity and phase broadcasts yield long enough for+ # a pause signal to arrive after the iteration-level check.+ # Admit the resulting tool/delegation command batch afresh.+ await self._wait_until_resumed()calls = step_result.get("calls") or []workflow.logger.info(f"AgentWorkflow scheduling {len(calls)} tool call(s)")@@ -891,13 +1063,120 @@ class AgentWorkflow:]delegation_handles: Dict[int, Any] = {}delegation_permits: Dict[int, str] = {}+ delegation_lifecycles: Dict[int, Dict[str, Any]] = {}+ delegation_release_attempts: Dict[int, int] = {}yielded_own_permit = False+ async def _release_delegation_permit(call_index: int) -> None:+ """Idempotently release one confirmed permit exactly once locally."""+ permit_id = delegation_permits.get(call_index)+ if not permit_id:+ return+ release_attempt = delegation_release_attempts.get(+ call_index,+ 0,+ )+ base_activity_id = (+ f"release-permit-{iteration + 1}-{call_index + 1}"+ )+ release_activity_id = (+ base_activity_id+ if release_attempt == 0+ else f"{base_activity_id}-recovery-{release_attempt}"+ )+ delegation_release_attempts[call_index] = (+ release_attempt + 1+ )+ await workflow.execute_activity(+ "agent.release_subagent_permit.v1",+ args=[{+ "root_execution_id": root_execution_id,+ "permit_id": permit_id,+ }],+ activity_id=release_activity_id,+ start_to_close_timeout=PERSIST_TURN_TIMEOUT,+ retry_policy=(+ DELEGATION_CLEANUP_RETRY+ if use_delegation_permit_cleanup+ else AGENT_ACTIVITY_RETRY+ ),+ )+ delegation_permits.pop(call_index, None)++ async def _release_delegation_permit_for_cleanup(+ call_index: int,+ ) -> None:+ """Best-effort release that cannot replace an active failure."""+ permit_id = delegation_permits.get(call_index)+ try:+ await _release_delegation_permit(call_index)+ except Exception as release_exc: # noqa: BLE001+ workflow.logger.error(+ "AgentWorkflow failed to release delegation permit "+ f"{permit_id or call_index}: {release_exc}"+ )++ async def _release_remaining_delegation_permits() -> None:+ """Best-effort drain all confirmed permits in deterministic order."""+ for pending_index in sorted(list(delegation_permits)):+ await _release_delegation_permit_for_cleanup(+ pending_index+ )++ async def _cancel_remaining_delegations() -> None:+ """Terminalize every task whose runner this workflow owns."""+ for pending_index in sorted(+ list(delegation_lifecycles)+ ):+ lifecycle = delegation_lifecycles[pending_index]+ try:+ await workflow.execute_activity(+ "agent.cancel_delegation.v1",+ args=[{+ **lifecycle,+ "reason": (+ "Parent agent workflow cancelled"+ ),+ "terminal_event_id": (+ f"{lifecycle['team_task_id']}:terminal"+ ),+ }],+ activity_id=(+ "cancel-delegation-"+ f"{iteration + 1}-{pending_index + 1}"+ ),+ start_to_close_timeout=PERSIST_TURN_TIMEOUT,+ retry_policy=DELEGATION_CLEANUP_RETRY,+ )+ delegation_lifecycles.pop(+ pending_index,+ None,+ )+ except Exception as cancellation_exc: # noqa: BLE001+ workflow.logger.error(+ "AgentWorkflow failed to persist delegation "+ f"cancellation for call {pending_index}: "+ f"{cancellation_exc}"+ )++ async def _cleanup_cancelled_delegations() -> None:+ # Terminal state and capacity are independent cleanup+ # obligations. Always attempt both and preserve the original+ # cancellation/error at the caller.+ await _cancel_remaining_delegations()+ await _release_remaining_delegation_permits()+# A child that waits for grandchildren must not retain a slot,# otherwise N admitted children can all block forever trying to# acquire the N+1th permit. Yield while orchestrating descendants# and reacquire before resuming this agent's next LLM turn.- own_permit_id = str(context.get("team_task_id") or "")+ own_logical_permit_id = str(+ context.get("team_task_id") or ""+ )+ own_permit_id = str(+ context.get("team_permit_id")+ or own_logical_permit_id+ )if delegation_call_indices and own_permit_id:await workflow.execute_activity("agent.release_subagent_permit.v1",@@ -979,6 +1258,11 @@ class AgentWorkflow:f"{child_context['team_task_id']}:assigned"),}+ delegation_lifecycles[call_index] = (+ lifecycle_payload+ )+ if use_cooperative_pause_schedule_gate:+ await self._wait_until_resumed()await workflow.execute_activity("agent.queue_delegation.v1",args=[{@@ -992,26 +1276,42 @@ class AgentWorkflow:),start_to_close_timeout=PERSIST_TURN_TIMEOUT,retry_policy=AGENT_ACTIVITY_RETRY,+ **acquire_cancellation_options,)- await workflow.execute_activity(+ if use_cooperative_pause_schedule_gate:+ await self._wait_until_resumed()+ acquire_payload = {+ "root_execution_id": root_execution_id,+ "permit_id": permit_id,+ "limit": max_concurrent_subagents,+ }+ if use_acquire_cancellation_completion:+ acquire_payload["lease_version"] = 2+ acquire_result = await workflow.execute_activity("agent.acquire_subagent_permit.v1",- args=[{- "root_execution_id": root_execution_id,- "permit_id": permit_id,- "limit": max_concurrent_subagents,- }],+ args=[acquire_payload],activity_id=(f"acquire-permit-{iteration + 1}-{call_index + 1}"),start_to_close_timeout=timedelta(hours=1),heartbeat_timeout=timedelta(seconds=10),retry_policy=AGENT_ACTIVITY_RETRY,+ **acquire_cancellation_options,+ )+ delegation_permits[call_index] = str(+ (acquire_result or {}).get("lease_id")+ or permit_id)- delegation_permits[call_index] = permit_id+ if use_acquire_cancellation_completion:+ child_context["team_permit_id"] = (+ delegation_permits[call_index]+ )# Claim only after admission. Persistence failure still# prevents child startup and the task remains pending# while the coordinator queues it.try:+ if use_cooperative_pause_schedule_gate:+ await self._wait_until_resumed()await workflow.execute_activity("agent.begin_delegation.v1",args=[lifecycle_payload],@@ -1020,20 +1320,14 @@ class AgentWorkflow:),start_to_close_timeout=PERSIST_TURN_TIMEOUT,retry_policy=AGENT_ACTIVITY_RETRY,+ **acquire_cancellation_options,)+ except asyncio.CancelledError:+ if not use_delegation_permit_cleanup:+ await _release_delegation_permit(call_index)+ raiseexcept BaseException:- await workflow.execute_activity(- "agent.release_subagent_permit.v1",- args=[{- "root_execution_id": root_execution_id,- "permit_id": permit_id,- }],- activity_id=(- f"release-permit-{iteration + 1}-{call_index + 1}"- ),- start_to_close_timeout=PERSIST_TURN_TIMEOUT,- retry_policy=AGENT_ACTIVITY_RETRY,- )+ await _release_delegation_permit(call_index)raisechild_id = (_delegation_child_id_v2(@@ -1046,6 +1340,8 @@ class AgentWorkflow:else f"{workflow.info().workflow_id}-delegate-{candidate_tool['tool_node_id']}-{iteration}")try:+ if use_cooperative_pause_schedule_gate:+ await self._wait_until_resumed()delegation_handles[call_index] = await workflow.start_child_workflow("AgentWorkflow",args=[child_context],@@ -1063,21 +1359,19 @@ class AgentWorkflow:activity_id=f"register-delegation-{iteration + 1}-{call_index + 1}",start_to_close_timeout=PERSIST_TURN_TIMEOUT,retry_policy=AGENT_ACTIVITY_RETRY,+ **acquire_cancellation_options,)+ except asyncio.CancelledError:+ if (+ not use_delegation_permit_cleanup+ and team_id+ and call_index in delegation_permits+ ):+ await _release_delegation_permit(call_index)+ raiseexcept BaseException:if team_id and call_index in delegation_permits:- await workflow.execute_activity(- "agent.release_subagent_permit.v1",- args=[{- "root_execution_id": root_execution_id,- "permit_id": delegation_permits[call_index],- }],- activity_id=(- f"release-permit-{iteration + 1}-{call_index + 1}"- ),- start_to_close_timeout=PERSIST_TURN_TIMEOUT,- retry_policy=AGENT_ACTIVITY_RETRY,- )+ await _release_delegation_permit(call_index)raiseasync def _run_task_manager_delegation(@@ -1155,6 +1449,8 @@ class AgentWorkflow:# queue_delegation is intentionally retained: it is# idempotent for the pre-created task and records the same# lifecycle event as direct delegation without duplicating it.+ if use_cooperative_pause_schedule_gate:+ await self._wait_until_resumed()await workflow.execute_activity("agent.queue_delegation.v1",args=[{**lifecycle, "queued_event_id": f"{task_id}:queued"}],@@ -1179,13 +1475,34 @@ class AgentWorkflow:workflow.info().workflow_id, assignee_id, iteration, call_index) + f"-{task_id}"runner_id = f"{child_id}-runner"+ if use_cooperative_pause_schedule_gate:+ await self._wait_until_resumed()+ delegated_task_options: Dict[str, Any] = {+ "id": runner_id,+ "parent_close_policy": ParentClosePolicy.ABANDON,+ "execution_timeout": timedelta(hours=2),+ "run_timeout": timedelta(hours=2),+ }+ use_delegated_task_search_attributes = workflow.patched(+ DELEGATED_TASK_SEARCH_ATTRIBUTES_PATCH+ )+ if use_delegated_task_search_attributes:+ from services.temporal.trigger_listener_workflow import (+ event_workflow_search_attributes,+ )++ search_attributes = event_workflow_search_attributes(+ payload.get("workflow_id")+ )+ if search_attributes is not None:+ delegated_task_options["search_attributes"] = (+ search_attributes+ )await workflow.start_child_workflow("DelegatedTaskWorkflow",args=[{"lifecycle": lifecycle, "child_context": child_context,"child_workflow_id": child_id, "limit": max_concurrent_subagents}],- id=runner_id,- parent_close_policy=ParentClosePolicy.ABANDON,- execution_timeout=timedelta(hours=2), run_timeout=timedelta(hours=2),+ **delegated_task_options,)return {"status": "queued", "result": None, "runner_workflow_id": runner_id}@@ -1233,6 +1550,13 @@ class AgentWorkflow:**preflight_metadata,}task_manager_preflight_indices.append(preflight_index)+ if (+ not task_manager_preflight_handles+ and use_cooperative_pause_schedule_gate+ ):+ # ``start_activity`` does not yield; one admission+ # check protects this whole concurrent batch.+ await self._wait_until_resumed()task_manager_preflight_handles.append(workflow.start_activity(f"node.taskManager.v{preflight_tool['version']}",@@ -1287,9 +1611,16 @@ class AgentWorkflow:)next_delegation_to_start = 0- for delegation_index in delegation_call_indices[:max_concurrent_subagents]:- await _start_delegation(delegation_index)- next_delegation_to_start += 1+ try:+ for delegation_index in delegation_call_indices[+ :max_concurrent_subagents+ ]:+ await _start_delegation(delegation_index)+ next_delegation_to_start += 1+ except BaseException:+ if use_delegation_permit_cleanup:+ await _cleanup_cancelled_delegations()+ raisefor call_index, call in enumerate(calls):if llm_engine == NATIVE_LLM_ENGINE and call.get("parse_error"):@@ -1459,41 +1790,41 @@ class AgentWorkflow:else f"node.{tool_info['node_type']}.v{tool_info['version']}")- await self._emit_phase(- agent_node_id,- agent_workflow_id,- iteration,- max_iterations,- phase="executing_tool",- extra={- "tool_name": call.get("name", ""),- "tool_node_id": tool_info["tool_node_id"],- "tool_call_id": str(- call.get("id") or f"{iteration + 1}:{call_index + 1}"- ),- },- )+ try:+ await self._emit_phase(+ agent_node_id,+ agent_workflow_id,+ iteration,+ max_iterations,+ phase="executing_tool",+ extra={+ "tool_name": call.get("name", ""),+ "tool_node_id": tool_info["tool_node_id"],+ "tool_call_id": str(+ call.get("id")+ or f"{iteration + 1}:{call_index + 1}"+ ),+ },+ )+ except asyncio.CancelledError:+ if use_delegation_permit_cleanup:+ await _cleanup_cancelled_delegations()+ raisetry:if is_delegation and tool_info["node_type"] in AGENT_WORKFLOW_TYPES:handle = delegation_handles[call_index]try:tool_result = await handle- finally:- permit_id = delegation_permits.get(call_index)- if permit_id:- await workflow.execute_activity(- "agent.release_subagent_permit.v1",- args=[{- "root_execution_id": root_execution_id,- "permit_id": permit_id,- }],- activity_id=(- f"release-permit-{iteration + 1}-{call_index + 1}"- ),- start_to_close_timeout=PERSIST_TURN_TIMEOUT,- retry_policy=AGENT_ACTIVITY_RETRY,- )+ except asyncio.CancelledError:+ if not use_delegation_permit_cleanup:+ await _release_delegation_permit(call_index)+ raise+ except BaseException:+ await _release_delegation_permit(call_index)+ raise+ else:+ await _release_delegation_permit(call_index)child_succeeded = (bool(tool_result.get("success", True))if isinstance(tool_result, dict) else True@@ -1538,6 +1869,10 @@ class AgentWorkflow:start_to_close_timeout=PERSIST_TURN_TIMEOUT,retry_policy=AGENT_ACTIVITY_RETRY,)+ delegation_lifecycles.pop(+ call_index,+ None,+ )tool_result = {"success": child_succeeded,"status": "submitted" if child_succeeded else "failed",@@ -1565,6 +1900,8 @@ class AgentWorkflow:if isinstance(tool_result, BaseException):raise tool_resultelse:+ if use_cooperative_pause_schedule_gate:+ await self._wait_until_resumed()tool_result = await workflow.execute_activity(tool_activity_name,args=[tool_payload],@@ -1639,6 +1976,8 @@ class AgentWorkflow:"agent_node_type": payload.get("node_type") or context.get("node_type"),**call_metadata,}+ if use_cooperative_pause_schedule_gate:+ await self._wait_until_resumed()refresh_result = await workflow.execute_activity("agent.refresh_tools.v1",args=[refresh_payload],@@ -1692,6 +2031,10 @@ class AgentWorkflow:len(added_tools),len(tools),)+ except asyncio.CancelledError:+ if use_delegation_permit_cleanup:+ await _cleanup_cancelled_delegations()+ raiseexcept Exception as e: # noqa: BLE001 — Temporal handles retries# After all retries exhausted, surface the error to# the LLM (per user decision: LLM sees error and@@ -1703,52 +2046,72 @@ class AgentWorkflow:f"task-{root_execution_id}-{agent_node_id}-"f"{iteration + 1}-{call_index + 1}")- await workflow.execute_activity(- "agent.finish_delegation.v1",- args=[{- "team_id": team_id,- "team_task_id": task_id,- "parent_agent_node_id": agent_node_id,- "child_agent_node_id": tool_info["tool_node_id"],- "child_agent_name": str(- (tool_info.get("tool_info") or {}).get("label")- or tool_info.get("node_type")- or "agent"+ try:+ await workflow.execute_activity(+ "agent.finish_delegation.v1",+ args=[{+ "team_id": team_id,+ "team_task_id": task_id,+ "parent_agent_node_id": agent_node_id,+ "child_agent_node_id": tool_info["tool_node_id"],+ "child_agent_name": str(+ (tool_info.get("tool_info") or {}).get(+ "label"+ )+ or tool_info.get("node_type")+ or "agent"+ ),+ "workflow_id": payload.get("workflow_id"),+ "parent_agent_workflow_id": (+ workflow.info().workflow_id+ ),+ "root_execution_id": root_execution_id,+ "trace_id": str(call.get("id", "") or ""),+ "success": False,+ "error": f"{type(e).__name__}: {e}",+ "terminal_event_id": f"{task_id}:terminal",+ }],+ activity_id=(+ "finish-delegation-"+ f"{iteration + 1}-{call_index + 1}"),- "workflow_id": payload.get("workflow_id"),- "parent_agent_workflow_id": workflow.info().workflow_id,- "root_execution_id": root_execution_id,- "trace_id": str(call.get("id", "") or ""),- "success": False,- "error": f"{type(e).__name__}: {e}",- "terminal_event_id": f"{task_id}:terminal",- }],- activity_id=(- f"finish-delegation-{iteration + 1}-{call_index + 1}"- ),- start_to_close_timeout=PERSIST_TURN_TIMEOUT,- retry_policy=AGENT_ACTIVITY_RETRY,- )+ start_to_close_timeout=PERSIST_TURN_TIMEOUT,+ retry_policy=AGENT_ACTIVITY_RETRY,+ )+ delegation_lifecycles.pop(+ call_index,+ None,+ )+ except asyncio.CancelledError:+ if use_delegation_permit_cleanup:+ await _cleanup_cancelled_delegations()+ raisetool_content = f'{{"error": "{type(e).__name__}: {e}"}}'- await self._emit_phase(- agent_node_id,- agent_workflow_id,- iteration,- max_iterations,- phase="tool_completed",- extra={- "tool_name": call.get("name", ""),- "tool_node_id": tool_info["tool_node_id"],- "tool_call_id": str(- call.get("id") or f"{iteration + 1}:{call_index + 1}"- ),- # Do not put raw failures into public status- # events. This safe flag is enough for the- # broadcaster to retain ``tool <name>`` with a- # failed capability state.- "tool_failed": True,- },- )+ try:+ await self._emit_phase(+ agent_node_id,+ agent_workflow_id,+ iteration,+ max_iterations,+ phase="tool_completed",+ extra={+ "tool_name": call.get("name", ""),+ "tool_node_id": tool_info["tool_node_id"],+ "tool_call_id": str(+ call.get("id")+ or f"{iteration + 1}:{call_index + 1}"+ ),+ # Do not put raw failures into public status+ # events. This safe flag is enough for the+ # broadcaster to retain ``tool <name>`` with a+ # failed capability state.+ "tool_failed": True,+ },+ )+ except asyncio.CancelledError:+ if use_delegation_permit_cleanup:+ await _cleanup_cancelled_delegations()+ raise_append_tool_result_message(messages,@@ -1759,6 +2122,8 @@ class AgentWorkflow:)if yielded_own_permit:+ if use_cooperative_pause_schedule_gate:+ await self._wait_until_resumed()await workflow.execute_activity("agent.acquire_subagent_permit.v1",args=[{@@ -1770,6 +2135,7 @@ class AgentWorkflow:start_to_close_timeout=timedelta(hours=1),heartbeat_timeout=timedelta(seconds=10),retry_policy=AGENT_ACTIVITY_RETRY,+ **acquire_cancellation_options,)# ---- Persist this turn (append-per-turn) -------------------@@ -1828,6 +2194,8 @@ class AgentWorkflow:"api_key": payload["api_key"],"model": payload["model"],}+ if use_cooperative_pause_schedule_gate:+ await self._wait_until_resumed()compact_result = await workflow.execute_activity("agent.compact_memory.v1",args=[compact_payload],diff --git a/server/services/temporal/polling_trigger_workflow.py b/server/services/temporal/polling_trigger_workflow.pyindex 64961c93..2fa18be2 100644--- a/server/services/temporal/polling_trigger_workflow.py+++ b/server/services/temporal/polling_trigger_workflow.py@@ -57,6 +57,9 @@ _DEFAULT_POLL_INTERVAL_S = 60# headroom for slow Gmail / Twitter responses without hanging forever.# Workflow ``RetryPolicy`` (default) handles transient failures._ACTIVITY_TIMEOUT_MULT = 4+COOPERATIVE_PAUSE_SCHEDULING_PATCH = (+ "polling-trigger-cooperative-pause-scheduling-v1"+)@workflow.defn(name="PollingTriggerWorkflow", sandboxed=False)@@ -84,6 +87,10 @@ class PollingTriggerWorkflow:async def resume(self) -> None:self._control_paused = False+ async def _wait_until_resumed(self) -> None:+ if self._control_paused:+ await workflow.wait_condition(lambda: not self._control_paused)+@workflow.runasync def run(self, listener_data: Dict[str, Any]) -> Dict[str, Any]:"""Poll loop body.@@ -110,6 +117,14 @@ class PollingTriggerWorkflow:node_type = listener_data["node_type"]version = listener_data.get("version", 1)activity_name = f"poll.{node_type}.v{version}"+ try:+ use_cooperative_pause_schedule_gate = workflow.patched(+ COOPERATIVE_PAUSE_SCHEDULING_PATCH+ )+ except Exception: # direct unit invocation outside Temporal runtime+ if workflow.in_workflow():+ raise+ use_cooperative_pause_schedule_gate = Falseparams = listener_data.get("filter_params", {}) or {}poll_interval = int(params.get("poll_interval") or _DEFAULT_POLL_INTERVAL_S)@@ -141,6 +156,8 @@ class PollingTriggerWorkflow:}else:await workflow.sleep(timedelta(seconds=poll_interval))+ if use_cooperative_pause_schedule_gate:+ await self._wait_until_resumed()cycle_payload = {"node_id": listener_data["trigger_node_id"],"params": params,@@ -177,7 +194,28 @@ class PollingTriggerWorkflow:continueself._seen_event_ids.add(event_id)try:- await self._spawn_child_run(event, listener_data)+ if use_cooperative_pause_schedule_gate:+ await self._wait_until_resumed()+ from services.temporal.trigger_listener_workflow import (+ event_workflow_search_attributes,+ )++ await self._spawn_child_run(+ event,+ listener_data,+ admission_check=(+ self._wait_until_resumed+ if use_cooperative_pause_schedule_gate+ else None+ ),+ search_attributes=(+ event_workflow_search_attributes(+ listener_data.get("workflow_id")+ )+ if use_cooperative_pause_schedule_gate+ else None+ ),+ )except Exception as spawn_exc: # noqa: BLE001# Per-event spawn failure logged; subsequent events# still try. Same isolation contract as the push@@ -187,6 +225,11 @@ class PollingTriggerWorkflow:if self._processed_count >= _MAX_EVENTS_BEFORE_CONTINUE_AS_NEW:workflow.logger.info(f"PollingTriggerWorkflow continue_as_new: processed={self._processed_count}")+ if use_cooperative_pause_schedule_gate:+ # A pause may arrive during the final waiting-status+ # broadcast. Continue only after Resume so the next run+ # cannot reset a still-active pause flag.+ await self._wait_until_resumed()# Carry seen_ids forward so the new run doesn't re-emit# what's already been seen by the provider.listener_data["seen_ids"] = list(seen_ids)@@ -196,6 +239,8 @@ class PollingTriggerWorkflow:self,event: Dict[str, Any],listener_data: Dict[str, Any],+ admission_check=None,+ search_attributes=None,) -> None:"""Start a child :class:`MachinaWorkflow` with the triggerpre-executed against this event payload.@@ -275,6 +320,21 @@ class PollingTriggerWorkflow:event_type=listener_data.get("event_type", ""),)+ if admission_check is not None:+ await admission_check()++ child_options = {+ "id": child_id,+ "parent_close_policy": ParentClosePolicy.ABANDON,+ "id_reuse_policy": (+ WorkflowIDReusePolicy.ALLOW_DUPLICATE_FAILED_ONLY+ ),+ "execution_timeout": timedelta(hours=1),+ "run_timeout": timedelta(hours=1),+ }+ if search_attributes is not None:+ child_options["search_attributes"] = search_attributes+await workflow.start_child_workflow("MachinaWorkflow",args=[@@ -287,11 +347,7 @@ class PollingTriggerWorkflow:"tenant_id": tenant_id,}],- id=child_id,- parent_close_policy=ParentClosePolicy.ABANDON,- id_reuse_policy=WorkflowIDReusePolicy.ALLOW_DUPLICATE_FAILED_ONLY,- execution_timeout=timedelta(hours=1),- run_timeout=timedelta(hours=1),+ **child_options,)workflow.logger.info(f"PollingTriggerWorkflow spawned child run: child_id={child_id} " f"event.id={event.get('id')}")diff --git a/server/services/temporal/schedules.py b/server/services/temporal/schedules.pyindex cdcedf4a..6ef40cf9 100644--- a/server/services/temporal/schedules.py+++ b/server/services/temporal/schedules.py@@ -20,6 +20,7 @@ Refs:from __future__ import annotations+from dataclasses import replacefrom datetime import timedeltafrom typing import Any, Dict@@ -32,6 +33,7 @@ from temporalio.client import (ScheduleOverlapPolicy,SchedulePolicy,ScheduleSpec,+ ScheduleUpdate,)from temporalio.common import (SearchAttributeKey,@@ -111,6 +113,22 @@ async def create_cron_schedule("""schedule_id = cron_schedule_id(workflow_slug, trigger_label)action_workflow_id = cron_action_workflow_id(workflow_slug, trigger_label)+ execution_search_attributes = TypedSearchAttributes(+ [+ SearchAttributePair(+ SearchAttributeKey.for_keyword("EventWorkflowId"),+ workflow_id,+ ),+ SearchAttributePair(+ SearchAttributeKey.for_keyword("TriggerNodeId"),+ node_id,+ ),+ SearchAttributePair(+ SearchAttributeKey.for_keyword("EventTriggerKind"),+ "cron",+ ),+ ]+ )schedule = Schedule(action=ScheduleActionStartWorkflow(@@ -118,6 +136,7 @@ async def create_cron_schedule(args=[listener_data],id=action_workflow_id,task_queue=task_queue,+ typed_search_attributes=execution_search_attributes,),spec=ScheduleSpec(cron_expressions=[cron_expression],@@ -141,22 +160,7 @@ async def create_cron_schedule(# ``search_attributes`` is a ``client.create_schedule`` kwarg, not# a field on the ``Schedule`` dataclass itself (per the Temporal# SDK API).- schedule_search_attributes = TypedSearchAttributes(- [- SearchAttributePair(- SearchAttributeKey.for_keyword("EventWorkflowId"),- workflow_id,- ),- SearchAttributePair(- SearchAttributeKey.for_keyword("TriggerNodeId"),- node_id,- ),- SearchAttributePair(- SearchAttributeKey.for_keyword("EventTriggerKind"),- "cron",- ),- ]- )+ schedule_search_attributes = execution_search_attributestry:await client.create_schedule(@@ -171,10 +175,51 @@ async def create_cron_schedule(timezone=timezone,)except ScheduleAlreadyRunningError:- # Idempotent re-deploy. The pre-existing Schedule keeps its- # state; the caller's create-or-reuse contract is satisfied.+ # Idempotent re-deploy must still refresh the frozen action args and+ # execution Search Attributes. Otherwise a Schedule created before+ # the lifecycle-control release would keep spawning untagged roots+ # that Reset cannot discover. Preserve the server-owned paused state+ # while replacing the deployment snapshot and metadata.+ handle = client.get_schedule_handle(schedule_id)++ def update_existing(update_input):+ description = update_input.description+ existing_attributes = {+ pair.key.name: pair.value+ for pair in description.typed_search_attributes+ }+ existing_owner = existing_attributes.get("EventWorkflowId")+ if existing_owner is None:+ # Compatibility with Schedules created before execution+ # Search Attributes were copied onto the action. Only claim a+ # legacy id when its frozen payload proves the same owner.+ existing_args = getattr(+ description.schedule.action,+ "args",+ (),+ )+ if (+ existing_args+ and isinstance(existing_args[0], dict)+ ):+ existing_owner = existing_args[0].get("workflow_id")+ if str(existing_owner or "") != str(workflow_id):+ raise RuntimeError(+ "cron_schedule_ownership_conflict:"+ f"{schedule_id}"+ )+ updated_schedule = replace(+ schedule,+ state=description.schedule.state,+ )+ return ScheduleUpdate(+ schedule=updated_schedule,+ search_attributes=schedule_search_attributes,+ )++ await handle.update(update_existing)logger.info(- "Temporal cron Schedule already exists (re-deploy idempotency)",+ "Updated existing Temporal cron Schedule",schedule_id=schedule_id,)@@ -184,19 +229,23 @@ async def create_cron_schedule(async def delete_cron_schedules_for_deployment(client: Client,deployment_workflow_id: str,+ *,+ strict: bool = False,) -> int:"""Delete every cron Schedule for ``deployment_workflow_id``.Visibility-equivalent sweep: ``client.list_schedules(query=...)``is the registry, no local handle dict. Per-Schedule failures don't- block the sweep (mirror of- :meth:`DeploymentManager._cancel_canary_listeners` semantics).+ block the sweep. With ``strict=True``, the sweep still visits every+ target and then raises if any deletion failed, so Reset cannot publish+ completion while a durable Schedule survives.Returns count of Schedules deleted."""query = f"EventWorkflowId='{deployment_workflow_id}' " f"AND EventTriggerKind='cron'"deleted = 0+ failures: list[Exception] = []try:# ``Client.list_schedules`` is ``async def`` (returns a coroutine# that resolves to ``ScheduleAsyncIterator``), unlike@@ -213,6 +262,7 @@ async def delete_cron_schedules_for_deployment(await handle.delete()deleted += 1except Exception as exc: # noqa: BLE001+ failures.append(exc)logger.warning(f"Failed to delete cron Schedule {sched_id!r}: {exc}",deployment_workflow_id=deployment_workflow_id,@@ -222,6 +272,13 @@ async def delete_cron_schedules_for_deployment(f"Visibility query for cron Schedules failed: {exc} " f"(query={query!r})",deployment_workflow_id=deployment_workflow_id,)+ if strict:+ raise RuntimeError("cron_schedule_visibility_cleanup_failed") from exc++ if strict and failures:+ raise RuntimeError(+ f"cron_schedule_cleanup_failed:{len(failures)}"+ ) from failures[0]if deleted:logger.info(@@ -233,11 +290,22 @@ async def delete_cron_schedules_for_deployment(async def set_cron_schedules_paused(- client: Client, deployment_workflow_id: str, *, paused: bool,+ client: Client,+ deployment_workflow_id: str,+ *,+ paused: bool,+ strict: bool = False,) -> int:- """Pause or unpause every cron Schedule owned by one deployment."""+ """Pause or unpause every cron Schedule owned by one deployment.++ Strict lifecycle transitions still sweep every matching Schedule, then+ fail if visibility or any individual mutation failed. This prevents the+ database from publishing a stable Pause/Resume state while a producer is+ known to be in the opposite state.+ """query = f"EventWorkflowId='{deployment_workflow_id}' AND EventTriggerKind='cron'"changed = 0+ failures: list[Exception] = []try:iterator = await client.list_schedules(query=query)async for desc in iterator:@@ -249,6 +317,7 @@ async def set_cron_schedules_paused(await handle.unpause(note="OpenCompany workflow resumed")changed += 1except Exception as exc: # noqa: BLE001+ failures.append(exc)logger.warning(f"Failed to {'pause' if paused else 'resume'} cron Schedule {desc.id!r}: {exc}",deployment_workflow_id=deployment_workflow_id,@@ -258,6 +327,14 @@ async def set_cron_schedules_paused(f"Cron Schedule pause sweep failed: {exc}",deployment_workflow_id=deployment_workflow_id,)+ if strict:+ raise RuntimeError(+ "cron_schedule_pause_visibility_failed"+ ) from exc+ if strict and failures:+ raise RuntimeError(+ f"cron_schedule_pause_failed:{len(failures)}"+ ) from failures[0]return changeddiff --git a/server/services/temporal/trigger_listener_workflow.py b/server/services/temporal/trigger_listener_workflow.pyindex 277c41b9..00d2ee8b 100644--- a/server/services/temporal/trigger_listener_workflow.py+++ b/server/services/temporal/trigger_listener_workflow.py@@ -34,7 +34,12 @@ from datetime import timedeltafrom typing import Any, Dict, List, Optional, Setfrom temporalio import workflow-from temporalio.common import WorkflowIDReusePolicy+from temporalio.common import (+ SearchAttributeKey,+ SearchAttributePair,+ TypedSearchAttributes,+ WorkflowIDReusePolicy,+)from temporalio.workflow import ParentClosePolicy@@ -45,6 +50,26 @@ from temporalio.workflow import ParentClosePolicy# so this caps at ~16K triggers between continueAsNew checkpoints.# Per-event histogram is empirical; tune later._MAX_EVENTS_BEFORE_CONTINUE_AS_NEW = 16_000+CHILD_SEARCH_ATTRIBUTES_PATCH = "trigger-child-search-attributes-v1"+COOPERATIVE_PAUSE_SCHEDULING_PATCH = (+ "trigger-cooperative-pause-scheduling-v1"+)+++def event_workflow_search_attributes(+ workflow_id: Optional[str],+) -> Optional[TypedSearchAttributes]:+ """Tag detached execution roots for generation cleanup visibility."""+ if not workflow_id:+ return None+ return TypedSearchAttributes(+ [+ SearchAttributePair(+ SearchAttributeKey.for_keyword("EventWorkflowId"),+ str(workflow_id),+ )+ ]+ )@workflow.defn(name="TriggerListenerWorkflow", sandboxed=False)@@ -71,6 +96,10 @@ class TriggerListenerWorkflow:async def resume(self) -> None:self._control_paused = False+ async def _wait_until_resumed(self) -> None:+ if self._control_paused:+ await workflow.wait_condition(lambda: not self._control_paused)+@workflow.signalasync def on_event(self, event_payload: Dict[str, Any]) -> None:"""Receive an event from ``services.events.dispatch.emit``.@@ -116,6 +145,12 @@ class TriggerListenerWorkflow:f"node={listener_data.get('trigger_node_id')} "f"event_type={listener_data.get('event_type')}")+ use_child_search_attributes = workflow.patched(+ CHILD_SEARCH_ATTRIBUTES_PATCH+ )+ use_cooperative_pause_schedule_gate = workflow.patched(+ COOPERATIVE_PAUSE_SCHEDULING_PATCH+ )while True:await workflow.wait_condition(@@ -126,7 +161,22 @@ class TriggerListenerWorkflow:continuetry:- await self._spawn_child_run(event, listener_data)+ await self._spawn_child_run(+ event,+ listener_data,+ admission_check=(+ self._wait_until_resumed+ if use_cooperative_pause_schedule_gate+ else None+ ),+ search_attributes=(+ event_workflow_search_attributes(+ listener_data.get("workflow_id")+ )+ if use_child_search_attributes+ else None+ ),+ )except Exception as exc: # noqa: BLE001# Per-event spawn failures don't kill the listener.# Log and move on; the rejected event still counts as@@ -138,6 +188,10 @@ class TriggerListenerWorkflow:self._processed_count += 1if self._processed_count >= _MAX_EVENTS_BEFORE_CONTINUE_AS_NEW:workflow.logger.info(f"TriggerListener continue_as_new: processed={self._processed_count}")+ if use_cooperative_pause_schedule_gate:+ # Do not let continue-as-new reset a pause that landed+ # during the final spawn/status broadcast.+ await self._wait_until_resumed()workflow.continue_as_new(args=[listener_data])async def _spawn_child_run(@@ -145,6 +199,7 @@ class TriggerListenerWorkflow:event: Dict[str, Any],listener_data: Dict[str, Any],admission_check=None,+ search_attributes: Optional[TypedSearchAttributes] = None,) -> None:"""Build the filtered downstream graph + start a child MachinaWorkflow.@@ -230,6 +285,18 @@ class TriggerListenerWorkflow:if admission_check is not None:await admission_check()+ child_options: Dict[str, Any] = {+ "id": child_id,+ "parent_close_policy": ParentClosePolicy.ABANDON,+ "id_reuse_policy": (+ WorkflowIDReusePolicy.ALLOW_DUPLICATE_FAILED_ONLY+ ),+ "execution_timeout": timedelta(hours=1),+ "run_timeout": timedelta(hours=1),+ }+ if search_attributes is not None:+ child_options["search_attributes"] = search_attributes+await workflow.start_child_workflow("MachinaWorkflow",args=[@@ -245,11 +312,7 @@ class TriggerListenerWorkflow:"data_scope_id": listener_data.get("data_scope_id"),}],- id=child_id,- parent_close_policy=ParentClosePolicy.ABANDON,- id_reuse_policy=WorkflowIDReusePolicy.ALLOW_DUPLICATE_FAILED_ONLY,- execution_timeout=timedelta(hours=1),- run_timeout=timedelta(hours=1),+ **child_options,)workflow.logger.info(diff --git a/server/services/temporal/workflow.py b/server/services/temporal/workflow.pyindex b759c525..e695a301 100644--- a/server/services/temporal/workflow.py+++ b/server/services/temporal/workflow.py@@ -92,6 +92,9 @@ AGENT_WORKFLOW_TYPES = frozenset(# namespace's replay/reset window, then deprecate the marker in a later# release before removing that branch.AGENT_CHILD_ID_V2_PATCH = "machina-agent-child-id-v2"+COOPERATIVE_PAUSE_SCHEDULING_PATCH = (+ "machina-cooperative-pause-scheduling-v1"+)def _agent_child_workflow_id_v2(root_workflow_id: str, agent_node_id: str) -> str:@@ -141,6 +144,10 @@ class MachinaWorkflow:async def resume(self) -> None:self._control_paused = False+ async def _wait_until_resumed(self) -> None:+ if self._control_paused:+ await workflow.wait_condition(lambda: not self._control_paused)+@workflow.signalasync def on_event(self, event_payload: Dict[str, Any]) -> None:"""Wave 12 A7: receive an event from ``services.events.dispatch.emit``.@@ -249,6 +256,9 @@ class MachinaWorkflow:# new histories use the root Temporal id + exact canvas node id so# two same-label agents can start in the same graph execution.use_agent_child_id_v2 = workflow.patched(AGENT_CHILD_ID_V2_PATCH)+ use_cooperative_pause_schedule_gate = workflow.patched(+ COOPERATIVE_PAUSE_SCHEDULING_PATCH+ )workflow.logger.info(f"Starting workflow orchestration: {len(nodes)} nodes, {len(edges)} edges")@@ -305,6 +315,8 @@ class MachinaWorkflow:# ``_trigger_output`` is ``{not_triggered: True}``, no# downstream template should resolve against them.if not trigger_output.get("not_triggered"):+ if use_cooperative_pause_schedule_gate:+ await self._wait_until_resumed()await workflow.execute_activity("store_node_output_activity",{@@ -387,6 +399,11 @@ class MachinaWorkflow:# flag is on. Tool calls inside the agent loop become# per-type activities (F4.A path) automatically.dispatch = self._resolve_dispatch(node_type)+ if use_cooperative_pause_schedule_gate:+ # A preceding child start yields to the workflow event+ # loop, so a pause signal may have landed since ``ready``+ # was computed. Re-admit every command in the batch.+ await self._wait_until_resumed()if dispatch["kind"] == "child_workflow":# ``workflow.start_child_workflow`` is ``async def`` —# awaiting it returns the ``ChildWorkflowHandle`` oncediff --git a/server/services/temporal/workflow_control_workflow.py b/server/services/temporal/workflow_control_workflow.pyindex 77d6d424..34f218dc 100644--- a/server/services/temporal/workflow_control_workflow.py+++ b/server/services/temporal/workflow_control_workflow.py@@ -5,10 +5,16 @@ from datetime import timedeltafrom typing import Any, Dictfrom temporalio import workflow+from temporalio.exceptions import ApplicationErrorfrom services.temporal._retry_policies import DEFAULT_ACTIVITY_RETRY+COOPERATIVE_PAUSE_SCHEDULING_PATCH = (+ "workflow-control-cooperative-pause-scheduling-v1"+)++@workflow.defn(name="WorkflowControlWorkflow", sandboxed=False)class WorkflowControlWorkflow:"""Own control state and trigger scheduling without listener workflows.@@ -26,17 +32,47 @@ class WorkflowControlWorkflow:self._events: list[tuple[str, Dict[str, Any]]] = []self._seen_event_ids: set[str] = set()self._poll_tasks: dict[str, asyncio.Task] = {}+ self._use_cooperative_pause_schedule_gate = False@workflow.signalasync def pause(self) -> None:- if self._state == "running":- self._state = "paused"- self._revision += 1+ self._apply_control_state("paused")@workflow.signalasync def resume(self) -> None:- if self._state == "paused":- self._state = "running"+ self._apply_control_state("running")++ @workflow.update+ async def set_control_state(self, requested_state: str) -> Dict[str, Any]:+ """Idempotently apply and acknowledge a pause/resume transition.++ Updates give callers a durable acknowledgement that the controller+ processed the request. The legacy signals above remain registered for+ older callers and histories.+ """+ normalized = str(requested_state).strip().lower()+ aliases = {+ "pause": "paused",+ "paused": "paused",+ "resume": "running",+ "running": "running",+ }+ target_state = aliases.get(normalized)+ if target_state is None:+ raise ApplicationError(+ "Control state must be one of: pause, paused, resume, running",+ type="InvalidWorkflowControlState",+ non_retryable=True,+ )+ self._apply_control_state(target_state)+ return self.status()++ def _apply_control_state(self, target_state: str) -> None:+ """Apply one valid live-state transition without double revision."""+ if self._state not in {"running", "paused"}:+ return+ if self._state != target_state:+ self._state = target_stateself._revision += 1@workflow.signal@@ -82,6 +118,14 @@ class WorkflowControlWorkflow:@workflow.runasync def run(self, control_data: Dict[str, Any]) -> Dict[str, Any]:+ try:+ self._use_cooperative_pause_schedule_gate = workflow.patched(+ COOPERATIVE_PAUSE_SCHEDULING_PATCH+ )+ except Exception: # direct unit invocation outside Temporal runtime+ if workflow.in_workflow():+ raise+ self._use_cooperative_pause_schedule_gate = Falseself._state = control_data.get("state", "running")while not self._closed:await workflow.wait_condition(@@ -96,11 +140,27 @@ class WorkflowControlWorkflow:return {"state": self._state, "generation": control_data.get("generation")}async def _spawn_push_run(self, event: Dict[str, Any], spec: Dict[str, Any]) -> None:- from services.temporal.trigger_listener_workflow import TriggerListenerWorkflow+ from services.temporal.trigger_listener_workflow import (+ TriggerListenerWorkflow,+ event_workflow_search_attributes,+ )listener = TriggerListenerWorkflow()+ listener_args = spec["listener_args"]await listener._spawn_child_run(- event, spec["listener_args"], admission_check=self._wait_until_running,+ event,+ listener_args,+ # This admission check predates the patch marker and must remain+ # unconditional for replay compatibility with existing controller+ # histories. The marker gates only the new child metadata.+ admission_check=self._wait_until_running,+ search_attributes=(+ event_workflow_search_attributes(+ listener_args.get("workflow_id")+ )+ if self._use_cooperative_pause_schedule_gate+ else None+ ),)async def _wait_until_running(self) -> None:@@ -115,6 +175,9 @@ class WorkflowControlWorkflow:_ACTIVITY_TIMEOUT_MULT,_DEFAULT_POLL_INTERVAL_S,)+ from services.temporal.trigger_listener_workflow import (+ event_workflow_search_attributes,+ )listener_data = spec["listener_args"]node_type = listener_data["node_type"]@@ -163,7 +226,25 @@ class WorkflowControlWorkflow:await workflow.wait_condition(lambda: self._closed or self._state == "running")if self._closed:return- await runner._spawn_child_run(event, listener_data)+ await runner._spawn_child_run(+ event,+ listener_data,+ admission_check=(+ self._wait_until_running+ if self._use_cooperative_pause_schedule_gate+ else None+ ),+ search_attributes=(+ event_workflow_search_attributes(+ listener_data.get("workflow_id")+ )+ if self._use_cooperative_pause_schedule_gate+ else None+ ),+ )-__all__ = ["WorkflowControlWorkflow"]+__all__ = [+ "COOPERATIVE_PAUSE_SCHEDULING_PATCH",+ "WorkflowControlWorkflow",+]diff --git a/server/tests/services/test_workflow_control.py b/server/tests/services/test_workflow_control.pyindex 5881cb0b..45c39a9e 100644--- a/server/tests/services/test_workflow_control.py+++ b/server/tests/services/test_workflow_control.py@@ -51,6 +51,7 @@ async def test_begin_generation_is_idempotent():assert created is Trueassert control.generation == 1+ assert control.revision == 0assert control.status == "starting"assert control.data_scope_id == control.execution_idassert serialize_control(control)["data_scope_id"] == control.execution_id@@ -202,3 +203,4 @@ async def test_reset_generation_is_ready_for_explicit_start():)assert created is Trueassert control.generation == 4+ assert control.revision == 10diff --git a/server/tests/services/test_workflow_control_handlers.py b/server/tests/services/test_workflow_control_handlers.pynew file mode 100644index 00000000..a5830312--- /dev/null+++ b/server/tests/services/test_workflow_control_handlers.py@@ -0,0 +1,1142 @@+"""Control-handler contract tests for acknowledged Temporal lifecycle changes."""++from types import SimpleNamespace+from unittest.mock import AsyncMock, MagicMock++import pytest++from services.deployment import handlers+++def _control(+ status: str,+ revision: int,+ *,+ terminal_reason: str | None = None,+):+ return SimpleNamespace(+ id="workflow-control:wf:1",+ workflow_id="wf",+ generation=1,+ execution_id="execution-1",+ root_execution_id="execution-1",+ data_scope_id="execution-1",+ controller_workflow_id="workflow-control-wf-g1",+ controller_run_id="controller-run-1",+ status=status,+ revision=revision,+ created_at=None,+ updated_at=None,+ terminal_reason=terminal_reason,+ )+++async def test_start_waits_for_deployment_setup_before_publishing_running(monkeypatch):+ starting = _control("starting", 0)+ starting_with_run = _control("starting", 1)+ running = _control("running", 2)+ order: list[str] = []++ async def transition(_control_value, **kwargs):+ order.append(f"db:{kwargs['status']}")+ return (+ starting_with_run+ if kwargs["status"] == "starting"+ else running+ )++ service = SimpleNamespace(+ database=SimpleNamespace(+ get_workflow_control_by_idempotency_key=AsyncMock(return_value=None),+ get_latest_workflow_control=AsyncMock(return_value=None),+ update_workflow_run_data_scope=AsyncMock(return_value=True),+ ),+ begin_generation=AsyncMock(return_value=(starting, True)),+ transition=AsyncMock(side_effect=transition),+ )++ async def broadcast(control, **_kwargs):+ order.append(f"broadcast:{control.status}:{control.revision}")+ return {"state": control.status, "revision": control.revision}++ async def deployment_setup(_workflow_id):+ order.append("deployment:ready")+ return {"success": True}++ monkeypatch.setattr(handlers, "_control_service", lambda: service)+ monkeypatch.setattr(handlers, "_broadcast_control", broadcast)+ monkeypatch.setattr(+ handlers,+ "_start_controller",+ AsyncMock(return_value="controller-run-1"),+ )+ monkeypatch.setattr(+ handlers,+ "handle_deploy_workflow",+ AsyncMock(return_value={"success": True}),+ )+ monkeypatch.setattr(handlers, "_await_deployment_setup", deployment_setup)+ monkeypatch.setattr(+ handlers,+ "_query_controller_state",+ AsyncMock(return_value={"state": "running", "revision": 0}),+ )++ result = await handlers.handle_start_workflow(+ {+ "workflow_id": "wf",+ "nodes": [{"id": "start-1", "type": "start", "data": {}}],+ "edges": [],+ "expected_revision": 0,+ "idempotency_key": "start-request-1",+ },+ None,+ )++ assert result["success"] is True+ assert result["state"] == "running"+ assert order == [+ "broadcast:starting:0",+ "db:starting",+ "broadcast:starting:1",+ "deployment:ready",+ "db:running",+ "broadcast:running:2",+ ]+++async def test_start_post_commit_broadcast_failure_does_not_tear_down_running(+ monkeypatch,+):+ starting = _control("starting", 0)+ starting_with_run = _control("starting", 1)+ running = _control("running", 2)++ async def transition(_control_value, **kwargs):+ if kwargs["status"] == "starting":+ return starting_with_run+ return running++ async def broadcast(control, **_kwargs):+ if control.status == "running":+ raise RuntimeError("status broadcast unavailable")+ return {"state": control.status, "revision": control.revision}++ service = SimpleNamespace(+ database=SimpleNamespace(+ get_workflow_control_by_idempotency_key=AsyncMock(+ return_value=None,+ ),+ get_latest_workflow_control=AsyncMock(return_value=None),+ update_workflow_run_data_scope=AsyncMock(return_value=True),+ ),+ begin_generation=AsyncMock(return_value=(starting, True)),+ transition=AsyncMock(side_effect=transition),+ fail=AsyncMock(),+ )+ signal_controller = AsyncMock()+ terminate_generation = AsyncMock()++ monkeypatch.setattr(handlers, "_control_service", lambda: service)+ monkeypatch.setattr(handlers, "_broadcast_control", broadcast)+ monkeypatch.setattr(+ handlers,+ "_start_controller",+ AsyncMock(return_value="controller-run-1"),+ )+ monkeypatch.setattr(+ handlers,+ "handle_deploy_workflow",+ AsyncMock(return_value={"success": True}),+ )+ monkeypatch.setattr(+ handlers,+ "_await_deployment_setup",+ AsyncMock(return_value={"success": True}),+ )+ monkeypatch.setattr(+ handlers,+ "_query_controller_state",+ AsyncMock(return_value={"state": "running", "revision": 0}),+ )+ monkeypatch.setattr(handlers, "_signal_controller", signal_controller)+ monkeypatch.setattr(+ handlers,+ "_terminate_generation_workflows",+ terminate_generation,+ )++ result = await handlers.handle_start_workflow(+ {+ "workflow_id": "wf",+ "nodes": [{"id": "start-1", "type": "start", "data": {}}],+ "edges": [],+ "expected_revision": 0,+ "idempotency_key": "start-request-1",+ },+ None,+ )++ assert result == {+ "success": False,+ "error": "status broadcast unavailable",+ }+ assert service.transition.await_count == 2+ assert service.transition.await_args.kwargs["status"] == "running"+ service.fail.assert_not_awaited()+ signal_controller.assert_not_awaited()+ terminate_generation.assert_not_awaited()+++ ("control", "expected_error"),+ [+ (_control("starting", 1), "workflow_start_pending"),+ (+ _control(+ "failed",+ 2,+ terminal_reason="deployment_setup_failed",+ ),+ "deployment_setup_failed",+ ),+ ],+)+async def test_duplicate_start_reports_non_successful_durable_outcome(+ monkeypatch,+ control,+ expected_error,+):+ service = SimpleNamespace(+ database=SimpleNamespace(+ get_workflow_control_by_idempotency_key=AsyncMock(+ return_value=control,+ ),+ ),+ )+ monkeypatch.setattr(handlers, "_control_service", lambda: service)+ monkeypatch.setattr(+ handlers,+ "_reconcile_control",+ AsyncMock(return_value=(control, None)),+ )+ monkeypatch.setattr(+ handlers,+ "_control_payload",+ AsyncMock(+ return_value={+ "state": control.status,+ "revision": control.revision,+ }+ ),+ )++ result = await handlers.handle_start_workflow(+ {+ "workflow_id": "wf",+ "idempotency_key": "start-request-1",+ },+ None,+ )++ assert result == {+ "success": False,+ "error": expected_error,+ "idempotent": True,+ "state": control.status,+ "revision": control.revision,+ }+++async def test_concurrent_duplicate_start_reports_pending_winner(monkeypatch):+ starting = _control("starting", 1)+ service = SimpleNamespace(+ database=SimpleNamespace(+ get_workflow_control_by_idempotency_key=AsyncMock(+ return_value=None,+ ),+ get_latest_workflow_control=AsyncMock(return_value=None),+ ),+ begin_generation=AsyncMock(return_value=(starting, False)),+ )+ start_controller = AsyncMock()+ monkeypatch.setattr(handlers, "_control_service", lambda: service)+ monkeypatch.setattr(+ handlers,+ "_reconcile_control",+ AsyncMock(return_value=(starting, None)),+ )+ monkeypatch.setattr(+ handlers,+ "_control_payload",+ AsyncMock(return_value={"state": "starting", "revision": 1}),+ )+ monkeypatch.setattr(handlers, "_start_controller", start_controller)++ result = await handlers.handle_start_workflow(+ {+ "workflow_id": "wf",+ "expected_revision": 0,+ "idempotency_key": "start-request-1",+ },+ None,+ )++ assert result["success"] is False+ assert result["error"] == "workflow_start_pending"+ assert result["idempotent"] is True+ start_controller.assert_not_awaited()+++async def test_controller_state_change_waits_for_acknowledged_update(monkeypatch):+ handle = SimpleNamespace(+ execute_update=AsyncMock(return_value={+ "state": "paused",+ "revision": 3,+ "queued_events": 2,+ }),+ )+ monkeypatch.setattr(handlers, "_controller_handle", lambda _control: handle)++ result = await handlers._update_controller_state(+ _control("pausing", 2),+ "paused",+ update_id="request-1:paused",+ )++ assert result["state"] == "paused"+ handle.execute_update.assert_awaited_once_with(+ "set_control_state",+ "paused",+ id="request-1:paused",+ )+++async def test_reconcile_completes_interrupted_pause_from_temporal(monkeypatch):+ from core import container as container_module++ pausing = _control("pausing", 3)+ paused = _control("paused", 4)+ service = SimpleNamespace(+ transition=AsyncMock(return_value=paused),+ database=SimpleNamespace(get_latest_workflow_control=AsyncMock()),+ )+ update = AsyncMock(return_value={+ "state": "paused",+ "revision": 1,+ "queued_events": 0,+ })+ broadcast = AsyncMock(return_value={})+ workflow_service = SimpleNamespace(+ pause_deployment=MagicMock(),+ update_trigger_pause_status=AsyncMock(return_value=1),+ )+ monkeypatch.setattr(+ handlers,+ "_query_controller_state",+ AsyncMock(return_value={"state": "running", "revision": 0}),+ )+ monkeypatch.setattr(handlers, "_update_controller_state", update)+ monkeypatch.setattr(handlers, "_broadcast_control", broadcast)+ monkeypatch.setattr(handlers, "_set_cron_pause", AsyncMock(return_value=1))+ monkeypatch.setattr(+ handlers,+ "_signal_generation_workflows",+ AsyncMock(return_value=2),+ )+ monkeypatch.setattr(+ container_module.container,+ "workflow_service",+ lambda: workflow_service,+ )++ reconciled, temporal_status = await handlers._reconcile_control(+ service,+ pausing,+ )++ assert reconciled is paused+ assert temporal_status["state"] == "paused"+ update.assert_awaited_once_with(+ pausing,+ "paused",+ update_id="reconcile:workflow-control:wf:1:3:paused",+ )+ service.transition.assert_awaited_once_with(+ pausing,+ expected_revision=3,+ from_statuses={"pausing"},+ status="paused",+ )+ broadcast.assert_awaited_once_with(+ paused,+ controller_status=temporal_status,+ extra={+ "signalled_executions": 2,+ "paused_schedules": 1,+ "paused_triggers": 1,+ },+ )+++async def test_pause_response_is_final_only_after_temporal_ack(monkeypatch):+ from core import container as container_module++ running = _control("running", 2)+ pausing = _control("pausing", 3)+ paused = _control("paused", 4)+ order: list[str] = []+ update_calls: list[tuple[object, str, str]] = []++ async def transition(control, **kwargs):+ order.append(f"db:{kwargs['status']}")+ return pausing if kwargs["status"] == "pausing" else paused++ service = SimpleNamespace(+ database=SimpleNamespace(+ get_latest_workflow_control=AsyncMock(return_value=running),+ ),+ transition=AsyncMock(side_effect=transition),+ )+ workflow_service = SimpleNamespace(+ pause_deployment=MagicMock(side_effect=lambda _workflow_id: order.append("local:pause")),+ resume_deployment=AsyncMock(),+ update_trigger_pause_status=AsyncMock(return_value=1),+ get_deployment_status=MagicMock(return_value={+ "active_runs": 0,+ "queued_events": 0,+ }),+ )++ async def update_controller(control, requested_state, *, update_id):+ order.append("temporal:ack")+ update_calls.append((control, requested_state, update_id))+ return {"state": "paused", "revision": 1, "queued_events": 0}++ async def broadcast(control, **_kwargs):+ order.append(f"broadcast:{control.status}")+ return {+ "state": control.status,+ "revision": control.revision,+ "can_pause": control.status == "running",+ "can_resume": control.status == "paused",+ }++ monkeypatch.setattr(handlers, "_control_service", lambda: service)+ monkeypatch.setattr(+ handlers,+ "_reconcile_control",+ AsyncMock(return_value=(running, {"state": "running", "revision": 0})),+ )+ monkeypatch.setattr(handlers, "_update_controller_state", update_controller)+ monkeypatch.setattr(handlers, "_broadcast_control", broadcast)+ monkeypatch.setattr(handlers, "_set_cron_pause", AsyncMock(return_value=1))+ monkeypatch.setattr(+ handlers,+ "_signal_generation_workflows",+ AsyncMock(return_value=2),+ )+ monkeypatch.setattr(+ container_module.container,+ "workflow_service",+ lambda: workflow_service,+ )++ result = await handlers.handle_pause_workflow(+ {+ "workflow_id": "wf",+ "expected_revision": 2,+ "idempotency_key": "request-1",+ },+ None,+ )++ assert result["success"] is True+ assert result["state"] == "paused"+ assert order == [+ "db:pausing",+ "broadcast:pausing",+ "local:pause",+ "temporal:ack",+ "db:paused",+ "broadcast:paused",+ ]+ assert update_calls == [+ (+ pausing,+ "paused",+ "workflow-control:wf:1:3:request-1:paused",+ )+ ]+++ (+ "operation",+ "initial_status",+ "transitional_status",+ "requested_state",+ "signal_name",+ "cron_paused",+ ),+ [+ ("pause", "running", "pausing", "paused", "pause", True),+ ("resume", "paused", "resuming", "running", "resume", False),+ ],+)+async def test_stable_state_is_not_published_when_strict_fanout_fails(+ monkeypatch,+ operation,+ initial_status,+ transitional_status,+ requested_state,+ signal_name,+ cron_paused,+):+ from core import container as container_module++ initial = _control(initial_status, 2)+ transitional = _control(transitional_status, 3)+ service = SimpleNamespace(+ database=SimpleNamespace(+ get_latest_workflow_control=AsyncMock(return_value=initial),+ ),+ transition=AsyncMock(return_value=transitional),+ )+ workflow_service = SimpleNamespace(+ pause_deployment=MagicMock(),+ resume_deployment=AsyncMock(return_value=0),+ update_trigger_pause_status=AsyncMock(return_value=1),+ )+ set_cron_pause = AsyncMock(return_value=1)+ fanout = AsyncMock(+ side_effect=RuntimeError("workflow_signal_failed:1"),+ )++ monkeypatch.setattr(handlers, "_control_service", lambda: service)+ monkeypatch.setattr(+ handlers,+ "_reconcile_control",+ AsyncMock(+ return_value=(+ initial,+ {"state": initial_status, "revision": 0},+ )+ ),+ )+ monkeypatch.setattr(+ handlers,+ "_update_controller_state",+ AsyncMock(+ return_value={+ "state": requested_state,+ "revision": 1,+ "queued_events": 0,+ }+ ),+ )+ monkeypatch.setattr(+ handlers,+ "_broadcast_control",+ AsyncMock(return_value={}),+ )+ monkeypatch.setattr(handlers, "_set_cron_pause", set_cron_pause)+ monkeypatch.setattr(+ handlers,+ "_signal_generation_workflows",+ fanout,+ )+ monkeypatch.setattr(+ container_module.container,+ "workflow_service",+ lambda: workflow_service,+ )++ handler = (+ handlers.handle_pause_workflow+ if operation == "pause"+ else handlers.handle_resume_workflow+ )+ result = await handler(+ {+ "workflow_id": "wf",+ "expected_revision": 2,+ "idempotency_key": "request-1",+ },+ None,+ )++ assert result == {+ "success": False,+ "error": "workflow_signal_failed:1",+ }+ service.transition.assert_awaited_once_with(+ initial,+ expected_revision=2,+ from_statuses={initial_status},+ status=transitional_status,+ )+ set_cron_pause.assert_awaited_once_with(+ "wf",+ paused=cron_paused,+ strict=True,+ )+ fanout.assert_awaited_once_with(+ transitional,+ signal_name,+ strict=True,+ )+++async def test_pause_temporal_failure_restores_running_projection(monkeypatch):+ from core import container as container_module++ running = _control("running", 2)+ pausing = _control("pausing", 3)+ service = SimpleNamespace(+ database=SimpleNamespace(+ get_latest_workflow_control=AsyncMock(return_value=running),+ ),+ transition=AsyncMock(return_value=pausing),+ )+ workflow_service = SimpleNamespace(+ pause_deployment=MagicMock(),+ resume_deployment=AsyncMock(return_value=0),+ )+ restore = AsyncMock(return_value=running)++ monkeypatch.setattr(handlers, "_control_service", lambda: service)+ monkeypatch.setattr(+ handlers,+ "_reconcile_control",+ AsyncMock(return_value=(running, {"state": "running", "revision": 0})),+ )+ monkeypatch.setattr(handlers, "_broadcast_control", AsyncMock(return_value={}))+ monkeypatch.setattr(+ handlers,+ "_update_controller_state",+ AsyncMock(+ side_effect=handlers.TemporalControlUnavailable(+ "temporal_control_unavailable"+ )+ ),+ )+ monkeypatch.setattr(+ handlers,+ "_restore_control_after_failed_update",+ restore,+ )+ monkeypatch.setattr(+ container_module.container,+ "workflow_service",+ lambda: workflow_service,+ )++ result = await handlers.handle_pause_workflow(+ {"workflow_id": "wf", "expected_revision": 2},+ None,+ )++ assert result == {+ "success": False,+ "error": "temporal_control_unavailable",+ }+ workflow_service.resume_deployment.assert_awaited_once_with("wf")+ restore.assert_awaited_once_with(+ service,+ pausing,+ transitional_state="pausing",+ stable_state="running",+ )+++async def test_unknown_update_outcome_stays_transitional_for_reconciliation(+ monkeypatch,+):+ from core import container as container_module++ running = _control("running", 2)+ pausing = _control("pausing", 3)+ service = SimpleNamespace(+ database=SimpleNamespace(+ get_latest_workflow_control=AsyncMock(return_value=running),+ ),+ transition=AsyncMock(return_value=pausing),+ )+ workflow_service = SimpleNamespace(+ pause_deployment=MagicMock(),+ resume_deployment=AsyncMock(return_value=0),+ )+ restore = AsyncMock()++ monkeypatch.setattr(handlers, "_control_service", lambda: service)+ monkeypatch.setattr(+ handlers,+ "_reconcile_control",+ AsyncMock(return_value=(running, {"state": "running", "revision": 0})),+ )+ monkeypatch.setattr(handlers, "_broadcast_control", AsyncMock(return_value={}))+ monkeypatch.setattr(+ handlers,+ "_update_controller_state",+ AsyncMock(side_effect=TimeoutError("update outcome unknown")),+ )+ monkeypatch.setattr(+ handlers,+ "_restore_control_after_failed_update",+ restore,+ )+ monkeypatch.setattr(+ container_module.container,+ "workflow_service",+ lambda: workflow_service,+ )++ result = await handlers.handle_pause_workflow(+ {"workflow_id": "wf", "expected_revision": 2},+ None,+ )++ assert result == {"success": False, "error": "update outcome unknown"}+ workflow_service.resume_deployment.assert_not_awaited()+ restore.assert_not_awaited()+++async def test_runtime_counts_sum_controller_and_local_queues(monkeypatch):+ from core import container as container_module++ workflow_service = SimpleNamespace(+ get_deployment_status=MagicMock(+ return_value={+ "active_runs": 4,+ "queued_events": 3,+ }+ ),+ )+ monkeypatch.setattr(+ container_module.container,+ "workflow_service",+ lambda: workflow_service,+ )++ result = await handlers._with_runtime_counts(+ {"queued_count": 2},+ "wf",+ )++ assert result["active_count"] == 4+ assert result["in_flight_count"] == 4+ assert result["queued_count"] == 5+++async def test_reset_cleanup_failure_leaves_control_resetting(monkeypatch):+ running = _control("running", 2)+ resetting = _control("resetting", 3)+ cleanup_order: list[str] = []+ database = SimpleNamespace(+ get_latest_workflow_control=AsyncMock(return_value=running),+ update_workflow_run_data_scope=AsyncMock(return_value=True),+ )+ service = SimpleNamespace(+ database=database,+ transition=AsyncMock(return_value=resetting),+ )++ async def delete_schedules_impl(*_args, **_kwargs):+ cleanup_order.append("delete_schedules")+ return 0++ async def cancel_deployment_impl(*_args, **_kwargs):+ cleanup_order.append("cancel_local")+ return {"success": True}++ async def terminate_impl(*_args, **_kwargs):+ cleanup_order.append("terminate")+ raise RuntimeError("workflow_termination_failed:1")++ def close_local_admission(_workflow_id):+ cleanup_order.append("close_local_admission")++ async def signal_controller(*_args, **_kwargs):+ cleanup_order.append("close_controller")++ delete_schedules = AsyncMock(side_effect=delete_schedules_impl)+ cancel_deployment = AsyncMock(side_effect=cancel_deployment_impl)+ terminate = AsyncMock(side_effect=terminate_impl)++ monkeypatch.setattr(handlers, "_control_service", lambda: service)+ monkeypatch.setattr(+ handlers,+ "_broadcast_control",+ AsyncMock(return_value={}),+ )+ monkeypatch.setattr(+ handlers,+ "_close_local_admission",+ close_local_admission,+ )+ monkeypatch.setattr(+ handlers,+ "_signal_controller",+ AsyncMock(side_effect=signal_controller),+ )+ monkeypatch.setattr(+ handlers,+ "_terminate_generation_workflows",+ terminate,+ )+ monkeypatch.setattr(+ handlers,+ "_delete_cron_schedules",+ delete_schedules,+ )+ monkeypatch.setattr(+ handlers,+ "handle_cancel_deployment",+ cancel_deployment,+ )++ result = await handlers.handle_reset_workflow(+ {"workflow_id": "wf", "expected_revision": 2},+ None,+ )++ assert result == {+ "success": False,+ "error": "workflow_termination_failed:1",+ }+ assert service.transition.await_count == 1+ first_transition = service.transition.await_args+ assert first_transition.args == (running,)+ assert first_transition.kwargs["expected_revision"] == 2+ assert first_transition.kwargs["status"] == "resetting"+ assert cleanup_order == [+ "close_local_admission",+ "close_controller",+ "delete_schedules",+ "cancel_local",+ "terminate",+ ]+ delete_schedules.assert_awaited_once_with("wf", strict=True)+ cancel_deployment.assert_awaited_once_with(+ {"workflow_id": "wf"},+ None,+ )+ terminate.assert_awaited_once_with(resetting, strict=True)+ database.update_workflow_run_data_scope.assert_not_awaited()+++async def test_reset_local_cleanup_failure_leaves_control_resetting(+ monkeypatch,+):+ resetting = _control("resetting", 3)+ database = SimpleNamespace(+ get_latest_workflow_control=AsyncMock(return_value=resetting),+ update_workflow_run_data_scope=AsyncMock(return_value=True),+ )+ service = SimpleNamespace(+ database=database,+ transition=AsyncMock(),+ )++ monkeypatch.setattr(handlers, "_control_service", lambda: service)+ monkeypatch.setattr(+ handlers,+ "_broadcast_control",+ AsyncMock(return_value={}),+ )+ monkeypatch.setattr(handlers, "_signal_controller", AsyncMock())+ monkeypatch.setattr(+ handlers,+ "_terminate_generation_workflows",+ AsyncMock(return_value=2),+ )+ monkeypatch.setattr(+ handlers,+ "_delete_cron_schedules",+ AsyncMock(return_value=1),+ )+ monkeypatch.setattr(+ handlers,+ "handle_cancel_deployment",+ AsyncMock(+ return_value={+ "success": False,+ "error": "listener teardown failed",+ }+ ),+ )++ result = await handlers.handle_reset_workflow(+ {"workflow_id": "wf"},+ None,+ )++ assert result == {+ "success": False,+ "error": (+ "workflow_local_cleanup_failed:listener teardown failed"+ ),+ }+ service.transition.assert_not_awaited()+ database.update_workflow_run_data_scope.assert_not_awaited()+++async def test_already_reset_is_idempotent_without_repeating_cleanup(+ monkeypatch,+):+ reset = _control("reset", 4)+ database = SimpleNamespace(+ get_latest_workflow_control=AsyncMock(return_value=reset),+ update_workflow_run_data_scope=AsyncMock(),+ )+ service = SimpleNamespace(+ database=database,+ transition=AsyncMock(),+ )+ close_local_admission = MagicMock()+ signal_controller = AsyncMock()+ delete_schedules = AsyncMock()+ cancel_deployment = AsyncMock()+ terminate = AsyncMock()+ control_payload = AsyncMock(+ return_value={+ "state": "ready",+ "revision": 4,+ "generation": 1,+ }+ )++ monkeypatch.setattr(handlers, "_control_service", lambda: service)+ monkeypatch.setattr(+ handlers,+ "_close_local_admission",+ close_local_admission,+ )+ monkeypatch.setattr(handlers, "_signal_controller", signal_controller)+ monkeypatch.setattr(+ handlers,+ "_delete_cron_schedules",+ delete_schedules,+ )+ monkeypatch.setattr(+ handlers,+ "handle_cancel_deployment",+ cancel_deployment,+ )+ monkeypatch.setattr(+ handlers,+ "_terminate_generation_workflows",+ terminate,+ )+ monkeypatch.setattr(handlers, "_control_payload", control_payload)++ result = await handlers.handle_reset_workflow(+ {+ "workflow_id": "wf",+ "expected_revision": 4,+ "idempotency_key": "reset-request-1",+ },+ None,+ )++ assert result == {+ "success": True,+ "idempotent": True,+ "state": "ready",+ "revision": 4,+ "generation": 1,+ }+ control_payload.assert_awaited_once_with(reset)+ service.transition.assert_not_awaited()+ close_local_admission.assert_not_called()+ signal_controller.assert_not_awaited()+ delete_schedules.assert_not_awaited()+ cancel_deployment.assert_not_awaited()+ terminate.assert_not_awaited()+ database.update_workflow_run_data_scope.assert_not_awaited()+++async def test_reset_retry_resumes_cleanup_before_marking_ready(monkeypatch):+ from services.deployment import runtime_state+ from services import status_broadcaster++ resetting = _control("resetting", 3)+ reset = _control("reset", 4)+ database = SimpleNamespace(+ get_latest_workflow_control=AsyncMock(return_value=resetting),+ update_workflow_run_data_scope=AsyncMock(return_value=True),+ )+ service = SimpleNamespace(+ database=database,+ transition=AsyncMock(return_value=reset),+ )+ terminate = AsyncMock(return_value=2)+ delete_schedules = AsyncMock(return_value=3)+ cancel_deployment = AsyncMock(+ return_value={+ "success": False,+ "message": "Workflow wf is not deployed",+ }+ )+ archive_nodes = AsyncMock(+ return_value={+ "archived_nodes": 2,+ "reset_nodes": ["agent-1"],+ }+ )+ broadcaster = SimpleNamespace(broadcast=AsyncMock())++ async def broadcast_control(control, **_kwargs):+ return {+ "state": "ready" if control.status == "reset" else control.status,+ "revision": control.revision,+ }++ monkeypatch.setattr(handlers, "_control_service", lambda: service)+ monkeypatch.setattr(+ handlers,+ "_broadcast_control",+ AsyncMock(side_effect=broadcast_control),+ )+ monkeypatch.setattr(handlers, "_signal_controller", AsyncMock())+ monkeypatch.setattr(+ handlers,+ "_terminate_generation_workflows",+ terminate,+ )+ monkeypatch.setattr(+ handlers,+ "_delete_cron_schedules",+ delete_schedules,+ )+ monkeypatch.setattr(+ handlers,+ "handle_cancel_deployment",+ cancel_deployment,+ )+ monkeypatch.setattr(+ runtime_state,+ "archive_and_reset_node_state",+ archive_nodes,+ )+ monkeypatch.setattr(+ status_broadcaster,+ "get_status_broadcaster",+ lambda: broadcaster,+ )++ result = await handlers.handle_reset_workflow(+ {"workflow_id": "wf"},+ None,+ )++ assert result["success"] is True+ assert result["state"] == "ready"+ assert result["revision"] == 4+ assert result["terminated_executions"] == 2+ assert result["deleted_schedules"] == 3+ assert result["local_cleanup_completed"] is False+ terminate.assert_awaited_once_with(resetting, strict=True)+ delete_schedules.assert_awaited_once_with("wf", strict=True)+ cancel_deployment.assert_awaited_once_with(+ {"workflow_id": "wf"},+ None,+ )+ database.update_workflow_run_data_scope.assert_awaited_once()+ archive_nodes.assert_awaited_once_with(+ resetting,+ database,+ broadcaster,+ )+ service.transition.assert_awaited_once()+ final_transition = service.transition.await_args+ assert final_transition.args == (resetting,)+ assert final_transition.kwargs["expected_revision"] == 3+ assert final_transition.kwargs["from_statuses"] == {"resetting"}+ assert final_transition.kwargs["status"] == "reset"+++def test_generation_fanout_uses_temporal_root_tree():+ query = handlers._generation_visibility_query(_control("running", 2))++ assert query == (+ "(RootWorkflowId='workflow-control-wf-g1' OR "+ "EventWorkflowId='wf') "+ "AND ExecutionStatus='Running'"+ )+++async def test_generation_fanout_expands_tagged_standalone_root_trees():+ queries: list[str] = []+ controller = SimpleNamespace(+ id="workflow-control-wf-g1",+ run_id="controller-run",+ root_id="workflow-control-wf-g1",+ )+ cron_graph = SimpleNamespace(+ id="cron-graph",+ run_id="graph-run",+ root_id="cron-firing-root",+ )+ cron_agent = SimpleNamespace(+ id="cron-agent",+ run_id="agent-run",+ root_id="cron-firing-root",+ )++ def list_workflows(*, query):+ queries.append(query)++ async def iterate():+ values = (+ [controller, cron_graph]+ if query.startswith("(RootWorkflowId=")+ else [controller, cron_graph, cron_agent]+ )+ for value in values:+ yield value++ return iterate()++ client = SimpleNamespace(list_workflows=list_workflows)++ executions = await handlers._list_generation_workflows(+ client,+ _control("running", 2),+ )++ assert {+ execution.id+ for execution in executions+ } == {+ "workflow-control-wf-g1",+ "cron-graph",+ "cron-agent",+ }+ assert queries[0] == (+ "(RootWorkflowId='workflow-control-wf-g1' OR "+ "EventWorkflowId='wf') "+ "AND ExecutionStatus='Running'"+ )+ assert queries[1] == (+ "RootWorkflowId IN "+ "('cron-firing-root', 'workflow-control-wf-g1') "+ "AND ExecutionStatus='Running'"+ )diff --git a/server/tests/temporal/test_cooperative_pause.py b/server/tests/temporal/test_cooperative_pause.pyindex 6dd71735..b7e3c5eb 100644--- a/server/tests/temporal/test_cooperative_pause.py+++ b/server/tests/temporal/test_cooperative_pause.py@@ -1,7 +1,15 @@"""Cooperative pause flags shared by Temporal orchestration workflows."""+import asyncio+import inspect+from types import SimpleNamespace+from unittest.mock import AsyncMock, MagicMock+import pytest+from temporalio.exceptions import ApplicationError+from temporalio.workflow import ActivityCancellationType+from nodes.scheduler.cron_scheduler._workflow import CronTriggerWorkflowfrom services.temporal.agent_workflow import AgentWorkflow, DelegatedTaskWorkflowfrom services.temporal.polling_trigger_workflow import PollingTriggerWorkflowfrom services.temporal.trigger_listener_workflow import TriggerListenerWorkflow@@ -18,6 +26,7 @@ from services.temporal.workflow_control_workflow import WorkflowControlWorkflowDelegatedTaskWorkflow,TriggerListenerWorkflow,PollingTriggerWorkflow,+ CronTriggerWorkflow,],)async def test_pause_and_resume_mutate_durable_workflow_state(workflow_type):@@ -30,6 +39,1322 @@ async def test_pause_and_resume_mutate_durable_workflow_state(workflow_type):assert instance._control_paused is False+def test_controller_registers_acknowledged_control_update():+ definition = WorkflowControlWorkflow.__temporal_workflow_definition++ assert "set_control_state" in definition.updates+ assert {"pause", "resume"} <= set(definition.signals)+++async def test_controller_control_update_is_acknowledged_and_idempotent():+ controller = WorkflowControlWorkflow()++ paused = await controller.set_control_state("pause")+ assert paused["state"] == "paused"+ assert paused["revision"] == 1++ repeated = await controller.set_control_state("paused")+ assert repeated["state"] == "paused"+ assert repeated["revision"] == 1++ resumed = await controller.set_control_state("resume")+ assert resumed["state"] == "running"+ assert resumed["revision"] == 2++ repeated = await controller.set_control_state("running")+ assert repeated["state"] == "running"+ assert repeated["revision"] == 2+++async def test_controller_control_update_rejects_unknown_state_without_mutation():+ controller = WorkflowControlWorkflow()++ with pytest.raises(ApplicationError, match="Control state"):+ await controller.set_control_state("stopped")++ assert controller.status()["state"] == "running"+ assert controller.status()["revision"] == 0+++def test_pause_scheduling_changes_are_replay_patch_guarded():+ from nodes.scheduler.cron_scheduler import _workflow as cron_module+ from services.temporal import agent_workflow as agent_module+ from services.temporal import polling_trigger_workflow as polling_module+ from services.temporal import trigger_listener_workflow as trigger_module+ from services.temporal import workflow as machina_module+ from services.temporal import workflow_control_workflow as control_module++ expected = {+ machina_module.COOPERATIVE_PAUSE_SCHEDULING_PATCH:+ inspect.getsource(MachinaWorkflow.run),+ polling_module.COOPERATIVE_PAUSE_SCHEDULING_PATCH:+ inspect.getsource(PollingTriggerWorkflow.run),+ agent_module.COOPERATIVE_PAUSE_SCHEDULING_PATCH:+ inspect.getsource(AgentWorkflow.run),+ agent_module.DELEGATED_TASK_COOPERATIVE_PAUSE_SCHEDULING_PATCH:+ inspect.getsource(DelegatedTaskWorkflow.run),+ agent_module.DELEGATION_PERMIT_CLEANUP_PATCH:+ inspect.getsource(AgentWorkflow.run),+ agent_module.DELEGATION_ACQUIRE_CANCELLATION_PATCH:+ inspect.getsource(AgentWorkflow.run),+ agent_module.DELEGATED_TASK_CANCELLATION_TERMINAL_PATCH:+ inspect.getsource(DelegatedTaskWorkflow.run),+ control_module.COOPERATIVE_PAUSE_SCHEDULING_PATCH:+ inspect.getsource(WorkflowControlWorkflow.run),+ trigger_module.COOPERATIVE_PAUSE_SCHEDULING_PATCH:+ inspect.getsource(TriggerListenerWorkflow.run),+ cron_module.COOPERATIVE_PAUSE_SCHEDULING_PATCH:+ inspect.getsource(CronTriggerWorkflow.run),+ }++ assert set(expected) == {+ "machina-cooperative-pause-scheduling-v1",+ "polling-trigger-cooperative-pause-scheduling-v1",+ "agent-cooperative-pause-scheduling-v1",+ "delegated-task-cooperative-pause-scheduling-v1",+ "agent-delegation-permit-cleanup-v1",+ "agent-delegation-acquire-cancellation-v1",+ "delegated-task-cancellation-terminal-v1",+ "workflow-control-cooperative-pause-scheduling-v1",+ "trigger-cooperative-pause-scheduling-v1",+ "cron-cooperative-pause-scheduling-v1",+ }+ for patch_id, source in expected.items():+ assert "workflow.patched(" in source+ assert "COOPERATIVE_PAUSE_SCHEDULING_PATCH" in source+ assert patch_id.endswith("-v1")+++async def test_controller_preserves_legacy_admission_and_patch_gates_metadata(+ monkeypatch,+):+ from services.temporal.trigger_listener_workflow import (+ TriggerListenerWorkflow,+ )++ spawn = AsyncMock()+ monkeypatch.setattr(TriggerListenerWorkflow, "_spawn_child_run", spawn)+ controller = WorkflowControlWorkflow()+ spec = {+ "listener_args": {+ "trigger_node_id": "push-1",+ "workflow_id": "workflow-1",+ }+ }++ controller._use_cooperative_pause_schedule_gate = False+ await controller._spawn_push_run({"id": "legacy"}, spec)+ assert (+ spawn.await_args.kwargs["admission_check"]+ == controller._wait_until_running+ )+ assert spawn.await_args.kwargs["search_attributes"] is None++ controller._use_cooperative_pause_schedule_gate = True+ await controller._spawn_push_run({"id": "new"}, spec)+ assert (+ spawn.await_args.kwargs["admission_check"]+ == controller._wait_until_running+ )+ assert spawn.await_args.kwargs["search_attributes"] is not None+++async def _wait_for_predicate(predicate, blocked: asyncio.Event) -> None:+ if not predicate():+ blocked.set()+ while not predicate():+ await asyncio.sleep(0)+++async def test_machina_rechecks_pause_between_ready_child_starts(monkeypatch):+ from services.temporal import workflow as workflow_module++ temporal_workflow = workflow_module.workflow+ monkeypatch.setattr(temporal_workflow, "logger", MagicMock())+ monkeypatch.setattr(temporal_workflow, "patched", lambda _patch_id: True)+ monkeypatch.setattr(+ temporal_workflow,+ "info",+ lambda: SimpleNamespace(workflow_id="root-run"),+ )++ paused_gate = asyncio.Event()+ monkeypatch.setattr(+ temporal_workflow,+ "wait_condition",+ lambda predicate: _wait_for_predicate(predicate, paused_gate),+ )++ instance = MachinaWorkflow()+ scheduled: list[str] = []++ async def start_child(_name, *, args, **_kwargs):+ node_id = args[0]["node_id"]+ scheduled.append(node_id)+ handle = asyncio.get_running_loop().create_future()+ handle.set_result(+ {"success": True, "node_id": node_id, "result": {"node": node_id}}+ )+ if len(scheduled) == 1:+ instance._control_paused = True+ return handle++ monkeypatch.setattr(temporal_workflow, "start_child_workflow", start_child)+ monkeypatch.setattr(+ MachinaWorkflow,+ "_resolve_dispatch",+ lambda self, _node_type: {+ "kind": "child_workflow",+ "name": "AgentWorkflow",+ },+ )++ task = asyncio.create_task(+ instance.run(+ {+ "nodes": [+ {"id": "agent-a", "type": "aiAgent", "data": {}},+ {"id": "agent-b", "type": "aiAgent", "data": {}},+ ],+ "edges": [],+ "workflow_id": "graph",+ "workflow_slug": "graph",+ "execution_id": "root-run",+ }+ )+ )++ await asyncio.wait_for(paused_gate.wait(), timeout=1)+ assert scheduled == ["agent-a"]++ await instance.resume()+ result = await asyncio.wait_for(task, timeout=1)++ assert scheduled == ["agent-a", "agent-b"]+ assert result["success"] is True+++def _poll_listener_data() -> dict:+ return {+ "node_type": "testPoll",+ "version": 1,+ "trigger_node_id": "poll-1",+ "filter_params": {"poll_interval": 1},+ "seen_ids": ["existing"],+ "nodes": [],+ "edges": [],+ "workflow_id": None,+ "session_id": "test",+ }+++async def test_polling_rechecks_pause_after_sleep_before_activity(monkeypatch):+ from services.temporal import polling_trigger_workflow as polling_module++ temporal_workflow = polling_module.workflow+ monkeypatch.setattr(temporal_workflow, "logger", MagicMock())+ monkeypatch.setattr(temporal_workflow, "patched", lambda _patch_id: True)++ instance = PollingTriggerWorkflow()+ paused_gate = asyncio.Event()+ activity_started = asyncio.Event()++ async def sleep_then_pause(_duration):+ instance._control_paused = True++ async def execute_activity(*_args, **_kwargs):+ activity_started.set()+ raise asyncio.CancelledError++ monkeypatch.setattr(temporal_workflow, "sleep", sleep_then_pause)+ monkeypatch.setattr(temporal_workflow, "execute_activity", execute_activity)+ monkeypatch.setattr(+ temporal_workflow,+ "wait_condition",+ lambda predicate: _wait_for_predicate(predicate, paused_gate),+ )++ task = asyncio.create_task(instance.run(_poll_listener_data()))+ await asyncio.wait_for(paused_gate.wait(), timeout=1)+ assert not activity_started.is_set()++ await instance.resume()+ with pytest.raises(asyncio.CancelledError):+ await asyncio.wait_for(task, timeout=1)+ assert activity_started.is_set()+++async def test_polling_rechecks_pause_after_activity_before_spawn(monkeypatch):+ from services.temporal import polling_trigger_workflow as polling_module++ temporal_workflow = polling_module.workflow+ monkeypatch.setattr(temporal_workflow, "logger", MagicMock())+ monkeypatch.setattr(temporal_workflow, "patched", lambda _patch_id: True)++ instance = PollingTriggerWorkflow()+ paused_gate = asyncio.Event()+ child_started = asyncio.Event()++ async def no_sleep(_duration):+ return None++ async def poll_activity(*_args, **_kwargs):+ instance._control_paused = True+ return {+ "seen_ids": ["existing", "event-1"],+ "events": [{"id": "event-1"}],+ }++ async def spawn_child(+ _event,+ _listener_data,+ admission_check=None,+ search_attributes=None,+ ):+ assert admission_check is not None+ child_started.set()+ raise asyncio.CancelledError++ monkeypatch.setattr(temporal_workflow, "sleep", no_sleep)+ monkeypatch.setattr(temporal_workflow, "execute_activity", poll_activity)+ monkeypatch.setattr(+ temporal_workflow,+ "wait_condition",+ lambda predicate: _wait_for_predicate(predicate, paused_gate),+ )+ monkeypatch.setattr(instance, "_spawn_child_run", spawn_child)++ task = asyncio.create_task(instance.run(_poll_listener_data()))+ await asyncio.wait_for(paused_gate.wait(), timeout=1)+ assert not child_started.is_set()++ await instance.resume()+ with pytest.raises(asyncio.CancelledError):+ await asyncio.wait_for(task, timeout=1)+ assert child_started.is_set()+++async def test_polling_child_admission_rechecks_after_idle_broadcast(monkeypatch):+ from services.temporal import polling_trigger_workflow as polling_module+ from services.temporal import trigger_listener_workflow as listener_module++ temporal_workflow = polling_module.workflow+ monkeypatch.setattr(temporal_workflow, "logger", MagicMock())+ monkeypatch.setattr(temporal_workflow, "patched", lambda _patch_id: False)+ monkeypatch.setattr(+ listener_module,+ "_build_run_graph",+ lambda **_kwargs: ([], []),+ )++ instance = PollingTriggerWorkflow()+ paused_gate = asyncio.Event()+ child_started = asyncio.Event()++ async def broadcast_idle(**_kwargs):+ instance._control_paused = True++ async def broadcast_waiting(**_kwargs):+ return None++ async def start_child(*_args, **_kwargs):+ child_started.set()+ return MagicMock()++ monkeypatch.setattr(listener_module, "_broadcast_trigger_idle", broadcast_idle)+ monkeypatch.setattr(+ listener_module,+ "_broadcast_trigger_waiting",+ broadcast_waiting,+ )+ monkeypatch.setattr(temporal_workflow, "start_child_workflow", start_child)+ monkeypatch.setattr(+ temporal_workflow,+ "wait_condition",+ lambda predicate: _wait_for_predicate(predicate, paused_gate),+ )++ task = asyncio.create_task(+ instance._spawn_child_run(+ {"id": "event-1"},+ _poll_listener_data(),+ admission_check=instance._wait_until_resumed,+ )+ )+ await asyncio.wait_for(paused_gate.wait(), timeout=1)+ assert not child_started.is_set()++ await instance.resume()+ await asyncio.wait_for(task, timeout=1)+ assert child_started.is_set()+++def _agent_payload() -> dict:+ return {+ "node_id": "agent-1",+ "node_type": "aiAgent",+ "workflow_id": "graph-1",+ "session_id": "session-1",+ "provider": "openai",+ "model": "test-model",+ "max_tokens": 100,+ "temperature": 0,+ "system_message": "Be useful",+ "user_prompt": "do the work",+ "tools": [+ {+ "name": "do_work",+ "definition": {+ "name": "do_work",+ "description": "Do work",+ "parameters": {"type": "object", "properties": {}},+ },+ "node_type": "testTool",+ "version": 1,+ "task_queue": "test",+ "tool_node_id": "tool-1",+ "parameters": {},+ "tool_info": {},+ }+ ],+ "memory_node_id": "",+ "memory_content": "",+ "memory_window_size": 10,+ "max_iterations": 2,+ "thinking_config": None,+ "compaction_threshold": None,+ "llm_engine": "native",+ "message_wire_version": 2,+ }+++async def test_agent_rechecks_pause_before_tool_command_batch(monkeypatch):+ from services.temporal import agent_workflow as agent_module++ temporal_workflow = agent_module.workflow+ monkeypatch.setattr(temporal_workflow, "logger", MagicMock())+ monkeypatch.setattr(temporal_workflow, "patched", lambda _patch_id: True)+ monkeypatch.setattr(+ temporal_workflow,+ "info",+ lambda: SimpleNamespace(+ workflow_id="agent-run",+ run_id="run-id-12345678",+ ),+ )+ monkeypatch.setattr(+ agent_module,+ "get_node_class",+ lambda _node_type: SimpleNamespace(needs_canvas=False),+ )++ instance = AgentWorkflow()+ paused_gate = asyncio.Event()+ tool_started = asyncio.Event()+ llm_step = 0++ async def execute_activity(name, *, args, **_kwargs):+ nonlocal llm_step+ if name == "agent.prepare_payload.v1":+ return _agent_payload()+ if name == "agent.broadcast_progress.v1":+ return {"emitted": True}+ if name == "agent.execute_llm_step.v1":+ llm_step += 1+ if llm_step == 1:+ instance._control_paused = True+ return {+ "kind": "tool_calls",+ "calls": [+ {+ "id": "call-1",+ "name": "do_work",+ "args": {},+ }+ ],+ "usage": {},+ }+ return {+ "kind": "final",+ "content": "done",+ "usage": {},+ }+ if name == "node.testTool.v1":+ tool_started.set()+ return {"success": True}+ if name == "agent.store_output.v1":+ return {"stored": True}+ if name == "agent.skill.clear.v1":+ return {"cleared": True}+ raise AssertionError(f"Unexpected activity {name}")++ monkeypatch.setattr(temporal_workflow, "execute_activity", execute_activity)+ monkeypatch.setattr(+ temporal_workflow,+ "wait_condition",+ lambda predicate: _wait_for_predicate(predicate, paused_gate),+ )++ task = asyncio.create_task(+ instance.run({"node_id": "agent-1", "execution_id": "root-run"})+ )+ await asyncio.wait_for(paused_gate.wait(), timeout=1)+ assert not tool_started.is_set()++ await instance.resume()+ result = await asyncio.wait_for(task, timeout=1)++ assert tool_started.is_set()+ assert result["success"] is True+++async def test_agent_rechecks_pause_after_tool_phase_before_activity(monkeypatch):+ from services.temporal import agent_workflow as agent_module++ temporal_workflow = agent_module.workflow+ monkeypatch.setattr(temporal_workflow, "logger", MagicMock())+ monkeypatch.setattr(temporal_workflow, "patched", lambda _patch_id: True)+ monkeypatch.setattr(+ temporal_workflow,+ "info",+ lambda: SimpleNamespace(+ workflow_id="agent-run",+ run_id="run-id-12345678",+ ),+ )+ monkeypatch.setattr(+ agent_module,+ "get_node_class",+ lambda _node_type: SimpleNamespace(needs_canvas=False),+ )++ instance = AgentWorkflow()+ paused_gate = asyncio.Event()+ tool_started = asyncio.Event()+ llm_step = 0++ async def execute_activity(name, *, args, **_kwargs):+ nonlocal llm_step+ if name == "agent.prepare_payload.v1":+ return _agent_payload()+ if name == "agent.broadcast_progress.v1":+ if args[0]["phase"] == "executing_tool":+ instance._control_paused = True+ return {"emitted": True}+ if name == "agent.execute_llm_step.v1":+ llm_step += 1+ if llm_step == 1:+ return {+ "kind": "tool_calls",+ "calls": [{"id": "call-1", "name": "do_work", "args": {}}],+ "usage": {},+ }+ return {"kind": "final", "content": "done", "usage": {}}+ if name == "node.testTool.v1":+ tool_started.set()+ return {"success": True}+ if name == "agent.store_output.v1":+ return {"stored": True}+ if name == "agent.skill.clear.v1":+ return {"cleared": True}+ raise AssertionError(f"Unexpected activity {name}")++ monkeypatch.setattr(temporal_workflow, "execute_activity", execute_activity)+ monkeypatch.setattr(+ temporal_workflow,+ "wait_condition",+ lambda predicate: _wait_for_predicate(predicate, paused_gate),+ )++ task = asyncio.create_task(+ instance.run({"node_id": "agent-1", "execution_id": "root-run"})+ )+ await asyncio.wait_for(paused_gate.wait(), timeout=1)+ assert not tool_started.is_set()++ await instance.resume()+ result = await asyncio.wait_for(task, timeout=1)++ assert tool_started.is_set()+ assert result["success"] is True+++def _delegation_agent_payload() -> dict:+ payload = _agent_payload()+ payload.update(+ {+ "team_id": "team-1",+ "tools": [+ {+ "name": "delegate_to_child",+ "definition": {+ "name": "delegate_to_child",+ "description": "Delegate work",+ "parameters": {"type": "object", "properties": {}},+ },+ "node_type": "aiAgent",+ "version": 1,+ "task_queue": "test",+ "tool_node_id": "child-agent",+ "parameters": {},+ "tool_info": {"label": "Child"},+ }+ ],+ }+ )+ return payload+++class _CompletedChildHandle:+ id = "child-workflow"+ first_execution_run_id = "child-run"++ def __await__(self):+ async def result():+ return {+ "success": True,+ "result": {"response": "delegated result"},+ }++ return result().__await__()+++class _PendingChildHandle:+ first_execution_run_id = "child-run"++ def __init__(self, child_id: str) -> None:+ self.id = child_id+ self._result = asyncio.get_running_loop().create_future()++ def __await__(self):+ return self._result.__await__()+++def _two_delegation_agent_payload() -> dict:+ payload = _delegation_agent_payload()+ template = payload["tools"][0]+ payload["tools"] = [+ {+ **template,+ "name": "delegate_to_child_a",+ "definition": {+ **template["definition"],+ "name": "delegate_to_child_a",+ },+ "tool_node_id": "child-agent-a",+ "tool_info": {"label": "Child A"},+ },+ {+ **template,+ "name": "delegate_to_child_b",+ "definition": {+ **template["definition"],+ "name": "delegate_to_child_b",+ },+ "tool_node_id": "child-agent-b",+ "tool_info": {"label": "Child B"},+ },+ ]+ return payload+++async def test_acquire_activity_compensates_cancelled_attempt(monkeypatch):+ from services import agent_team+ from services.temporal.agent_activities import acquire_subagent_permit++ calls: list[tuple[str, str, str]] = []++ async def acquire(root_id, permit_id, _limit):+ calls.append(("acquire", root_id, permit_id))+ # Model cancellation winning after the durable acquire side effect but+ # before an activity result can be recorded in workflow history.+ raise asyncio.CancelledError++ async def release(root_id, permit_id):+ calls.append(("release", root_id, permit_id))+ return True++ service = SimpleNamespace(+ acquire_subagent_permit=acquire,+ release_subagent_permit=release,+ )+ monkeypatch.setattr(+ agent_team,+ "get_agent_team_service",+ lambda: service,+ )++ with pytest.raises(asyncio.CancelledError):+ await acquire_subagent_permit({+ "root_execution_id": "root-1",+ "permit_id": "permit-1",+ "limit": 3,+ })++ assert calls == [+ ("acquire", "root-1", "permit-1"),+ ("release", "root-1", "permit-1"),+ ]++[email protected]("patch_enabled", [False, True])+async def test_delegated_acquire_cancellation_type_is_replay_patch_guarded(+ monkeypatch,+ patch_enabled,+):+ from services.temporal import agent_workflow as agent_module++ temporal_workflow = agent_module.workflow+ monkeypatch.setattr(temporal_workflow, "logger", MagicMock())+ monkeypatch.setattr(+ temporal_workflow,+ "patched",+ lambda patch_id: (+ patch_enabled+ if patch_id+ == agent_module.DELEGATION_ACQUIRE_CANCELLATION_PATCH+ else False+ ),+ )+ monkeypatch.setattr(+ temporal_workflow,+ "info",+ lambda: SimpleNamespace(+ workflow_id="delegated-runner",+ run_id="runner-run-id",+ ),+ )++ acquire_kwargs: dict = {}++ async def execute_activity(name, *, args, **kwargs):+ if name == "agent.register_task_execution.v1":+ return {}+ if name == "agent.acquire_subagent_permit.v1":+ acquire_kwargs.update(kwargs)+ raise asyncio.CancelledError+ raise AssertionError(f"Unexpected activity {name}")++ monkeypatch.setattr(temporal_workflow, "execute_activity", execute_activity)++ with pytest.raises(asyncio.CancelledError):+ await DelegatedTaskWorkflow().run({+ "lifecycle": {+ "team_id": "team-1",+ "team_task_id": "task-1",+ "root_execution_id": "root-1",+ },+ "child_workflow_id": "child-1",+ "child_context": {},+ })++ if patch_enabled:+ assert acquire_kwargs["cancellation_type"] == (+ ActivityCancellationType.WAIT_CANCELLATION_COMPLETED+ )+ else:+ assert "cancellation_type" not in acquire_kwargs++[email protected]("patch_enabled", [False, True])+async def test_agent_acquire_cancellation_type_is_replay_patch_guarded(+ monkeypatch,+ patch_enabled,+):+ from services.temporal import agent_workflow as agent_module++ temporal_workflow = agent_module.workflow+ monkeypatch.setattr(temporal_workflow, "logger", MagicMock())+ monkeypatch.setattr(+ temporal_workflow,+ "patched",+ lambda patch_id: (+ patch_enabled+ if patch_id+ == agent_module.DELEGATION_ACQUIRE_CANCELLATION_PATCH+ else True+ ),+ )+ monkeypatch.setattr(+ temporal_workflow,+ "info",+ lambda: SimpleNamespace(+ workflow_id="agent-run",+ run_id="run-id-12345678",+ ),+ )++ acquire_kwargs: dict = {}++ async def execute_activity(name, *, args, **kwargs):+ if name == "agent.prepare_payload.v1":+ return _delegation_agent_payload()+ if name == "agent.broadcast_progress.v1":+ return {"emitted": True}+ if name == "agent.execute_llm_step.v1":+ return {+ "kind": "tool_calls",+ "calls": [{+ "id": "delegate-1",+ "name": "delegate_to_child",+ "args": {"task": "research this"},+ }],+ "usage": {},+ }+ if name == "agent.queue_delegation.v1":+ return {}+ if name == "agent.acquire_subagent_permit.v1":+ acquire_kwargs.update(kwargs)+ raise asyncio.CancelledError+ raise AssertionError(f"Unexpected activity {name}")++ monkeypatch.setattr(temporal_workflow, "execute_activity", execute_activity)++ with pytest.raises(asyncio.CancelledError):+ await AgentWorkflow().run({+ "node_id": "agent-1",+ "execution_id": "root-run",+ "nodes": [],+ "edges": [],+ })++ if patch_enabled:+ assert acquire_kwargs["cancellation_type"] == (+ ActivityCancellationType.WAIT_CANCELLATION_COMPLETED+ )+ else:+ assert "cancellation_type" not in acquire_kwargs+++ "failing_cleanup",+ [None, "agent.finish_delegation.v1", "agent.release_subagent_permit.v1"],+)+async def test_delegated_cancellation_persists_terminal_releases_and_reraises(+ monkeypatch,+ failing_cleanup,+):+ from services.temporal import agent_workflow as agent_module++ temporal_workflow = agent_module.workflow+ monkeypatch.setattr(temporal_workflow, "logger", MagicMock())+ monkeypatch.setattr(temporal_workflow, "patched", lambda _patch_id: True)+ monkeypatch.setattr(+ temporal_workflow,+ "info",+ lambda: SimpleNamespace(+ workflow_id="delegated-runner",+ run_id="runner-run-id",+ ),+ )++ child_registered = asyncio.Event()+ calls: list[tuple[str, dict]] = []+ register_count = 0++ async def execute_activity(name, *, args, **_kwargs):+ nonlocal register_count+ calls.append((name, args[0]))+ if name == "agent.register_task_execution.v1":+ register_count += 1+ if register_count == 2:+ child_registered.set()+ return {}+ if name in {+ "agent.acquire_subagent_permit.v1",+ "agent.begin_delegation.v1",+ }:+ return {}+ if name == failing_cleanup:+ raise RuntimeError(f"{name} failed")+ if name in {+ "agent.finish_delegation.v1",+ "agent.release_subagent_permit.v1",+ }:+ return {}+ raise AssertionError(f"Unexpected activity {name}")++ async def start_child(*_args, **kwargs):+ return _PendingChildHandle(kwargs["id"])++ monkeypatch.setattr(temporal_workflow, "execute_activity", execute_activity)+ monkeypatch.setattr(temporal_workflow, "start_child_workflow", start_child)++ task = asyncio.create_task(+ DelegatedTaskWorkflow().run({+ "lifecycle": {+ "team_id": "team-1",+ "team_task_id": "task-1",+ "root_execution_id": "root-1",+ "parent_agent_node_id": "agent-1",+ "child_agent_node_id": "agent-2",+ },+ "child_workflow_id": "child-1",+ "child_context": {},+ })+ )+ await asyncio.wait_for(child_registered.wait(), timeout=1)+ task.cancel()++ with pytest.raises(asyncio.CancelledError):+ await asyncio.wait_for(task, timeout=1)++ terminal = next(+ payload+ for name, payload in calls+ if name == "agent.finish_delegation.v1"+ )+ assert terminal["success"] is False+ assert terminal["error"] == "Delegated task workflow cancelled"+ assert any(+ name == "agent.release_subagent_permit.v1"+ for name, _payload in calls+ )+++ "failed_release_activity_id",+ [None, "release-permit-1-1"],+)+async def test_agent_cancellation_releases_all_started_delegation_permits(+ monkeypatch,+ failed_release_activity_id,+):+ from services.temporal import agent_workflow as agent_module++ temporal_workflow = agent_module.workflow+ monkeypatch.setattr(temporal_workflow, "logger", MagicMock())+ monkeypatch.setattr(temporal_workflow, "patched", lambda _patch_id: True)+ monkeypatch.setattr(+ temporal_workflow,+ "info",+ lambda: SimpleNamespace(+ workflow_id="agent-run",+ run_id="run-id-12345678",+ ),+ )++ children_started = asyncio.Event()+ child_count = 0+ releases: list[tuple[str, str]] = []++ async def execute_activity(name, *, args, **kwargs):+ if name == "agent.prepare_payload.v1":+ return _two_delegation_agent_payload()+ if name == "agent.broadcast_progress.v1":+ return {"emitted": True}+ if name == "agent.execute_llm_step.v1":+ return {+ "kind": "tool_calls",+ "calls": [+ {+ "id": "delegate-a",+ "name": "delegate_to_child_a",+ "args": {"task": "research A"},+ },+ {+ "id": "delegate-b",+ "name": "delegate_to_child_b",+ "args": {"task": "research B"},+ },+ ],+ "usage": {},+ }+ if name == "agent.release_subagent_permit.v1":+ releases.append(+ (kwargs["activity_id"], args[0]["permit_id"])+ )+ if kwargs["activity_id"] == failed_release_activity_id:+ raise RuntimeError("release failed")+ return {"released": True}+ if name in {+ "agent.queue_delegation.v1",+ "agent.acquire_subagent_permit.v1",+ "agent.begin_delegation.v1",+ "agent.register_task_execution.v1",+ }:+ return {}+ raise AssertionError(f"Unexpected activity {name}")++ async def start_child(*_args, **kwargs):+ nonlocal child_count+ child_count += 1+ if child_count == 2:+ children_started.set()+ return _PendingChildHandle(kwargs["id"])++ monkeypatch.setattr(temporal_workflow, "execute_activity", execute_activity)+ monkeypatch.setattr(temporal_workflow, "start_child_workflow", start_child)++ task = asyncio.create_task(+ AgentWorkflow().run(+ {+ "node_id": "agent-1",+ "execution_id": "root-run",+ "nodes": [],+ "edges": [],+ }+ )+ )+ await asyncio.wait_for(children_started.wait(), timeout=1)+ task.cancel()+ with pytest.raises(asyncio.CancelledError):+ await asyncio.wait_for(task, timeout=1)++ assert releases == [+ (+ "release-permit-1-1",+ "task-root-run-agent-1-1-1",+ ),+ (+ "release-permit-1-2",+ "task-root-run-agent-1-1-2",+ ),+ ]+++ ("pause_activity", "blocked_command"),+ [+ ("agent.acquire_subagent_permit.v1", "agent.begin_delegation.v1"),+ ("agent.begin_delegation.v1", "start_child_workflow"),+ ],+)+async def test_agent_rechecks_pause_between_delegation_commands(+ monkeypatch,+ pause_activity,+ blocked_command,+):+ from services.temporal import agent_workflow as agent_module++ temporal_workflow = agent_module.workflow+ monkeypatch.setattr(temporal_workflow, "logger", MagicMock())+ monkeypatch.setattr(temporal_workflow, "patched", lambda _patch_id: True)+ monkeypatch.setattr(+ temporal_workflow,+ "info",+ lambda: SimpleNamespace(+ workflow_id="agent-run",+ run_id="run-id-12345678",+ ),+ )++ instance = AgentWorkflow()+ paused_gate = asyncio.Event()+ child_started = asyncio.Event()+ scheduled_commands: list[str] = []+ llm_step = 0++ async def execute_activity(name, *, args, **_kwargs):+ nonlocal llm_step+ if name == "agent.prepare_payload.v1":+ return _delegation_agent_payload()+ if name == "agent.broadcast_progress.v1":+ return {"emitted": True}+ if name == "agent.execute_llm_step.v1":+ llm_step += 1+ if llm_step == 1:+ return {+ "kind": "tool_calls",+ "calls": [+ {+ "id": "delegate-1",+ "name": "delegate_to_child",+ "args": {"task": "research this"},+ }+ ],+ "usage": {},+ }+ return {"kind": "final", "content": "done", "usage": {}}+ if name in {+ "agent.queue_delegation.v1",+ "agent.acquire_subagent_permit.v1",+ "agent.begin_delegation.v1",+ "agent.register_task_execution.v1",+ "agent.release_subagent_permit.v1",+ "agent.finish_delegation.v1",+ }:+ scheduled_commands.append(name)+ if name == pause_activity:+ instance._control_paused = True+ return {}+ if name == "agent.store_output.v1":+ return {"stored": True}+ if name == "agent.skill.clear.v1":+ return {"cleared": True}+ raise AssertionError(f"Unexpected activity {name}")++ async def start_child(*_args, **_kwargs):+ scheduled_commands.append("start_child_workflow")+ child_started.set()+ return _CompletedChildHandle()++ monkeypatch.setattr(temporal_workflow, "execute_activity", execute_activity)+ monkeypatch.setattr(temporal_workflow, "start_child_workflow", start_child)+ monkeypatch.setattr(+ temporal_workflow,+ "wait_condition",+ lambda predicate: _wait_for_predicate(predicate, paused_gate),+ )++ task = asyncio.create_task(+ instance.run(+ {+ "node_id": "agent-1",+ "execution_id": "root-run",+ "nodes": [],+ "edges": [],+ }+ )+ )+ await asyncio.wait_for(paused_gate.wait(), timeout=1)+ assert blocked_command not in scheduled_commands+ assert not child_started.is_set()++ await instance.resume()+ result = await asyncio.wait_for(task, timeout=1)++ assert child_started.is_set()+ assert result["success"] is True+ assert scheduled_commands == [+ "agent.queue_delegation.v1",+ "agent.acquire_subagent_permit.v1",+ "agent.begin_delegation.v1",+ "start_child_workflow",+ "agent.register_task_execution.v1",+ "agent.release_subagent_permit.v1",+ "agent.finish_delegation.v1",+ ]+++def _mixed_preflight_payload() -> dict:+ payload = _delegation_agent_payload()+ payload["tools"].append(+ {+ "name": "manage_tasks",+ "definition": {+ "name": "manage_tasks",+ "description": "Manage team tasks",+ "parameters": {"type": "object", "properties": {}},+ },+ "node_type": "taskManager",+ "version": 1,+ "task_queue": "test",+ "tool_node_id": "task-manager",+ "parameters": {},+ "tool_info": {"label": "Task Manager"},+ }+ )+ return payload+++async def test_agent_rechecks_pause_before_task_manager_preflight_batch(monkeypatch):+ from services.temporal import agent_workflow as agent_module++ temporal_workflow = agent_module.workflow+ monkeypatch.setattr(temporal_workflow, "logger", MagicMock())+ monkeypatch.setattr(temporal_workflow, "patched", lambda _patch_id: True)+ monkeypatch.setattr(+ temporal_workflow,+ "info",+ lambda: SimpleNamespace(+ workflow_id="agent-run",+ run_id="run-id-12345678",+ ),+ )++ instance = AgentWorkflow()+ paused_gate = asyncio.Event()+ preflight_started = asyncio.Event()++ async def execute_activity(name, *, args, **kwargs):+ if name == "agent.prepare_payload.v1":+ return _mixed_preflight_payload()+ if name == "agent.broadcast_progress.v1":+ return {"emitted": True}+ if name == "agent.execute_llm_step.v1":+ return {+ "kind": "tool_calls",+ "calls": [+ {+ "id": "delegate-1",+ "name": "delegate_to_child",+ "args": {"task": "research this"},+ },+ {+ "id": "assign-1",+ "name": "manage_tasks",+ "args": {+ "operation": "assign_task",+ "task": "follow up",+ },+ },+ ],+ "usage": {},+ }+ if (+ name == "agent.release_subagent_permit.v1"+ and kwargs.get("activity_id") == "yield-own-permit-1"+ ):+ instance._control_paused = True+ return {}+ raise AssertionError(f"Unexpected activity {name}")++ def start_activity(*_args, **_kwargs):+ preflight_started.set()+ raise asyncio.CancelledError++ monkeypatch.setattr(temporal_workflow, "execute_activity", execute_activity)+ monkeypatch.setattr(temporal_workflow, "start_activity", start_activity)+ monkeypatch.setattr(+ temporal_workflow,+ "wait_condition",+ lambda predicate: _wait_for_predicate(predicate, paused_gate),+ )++ task = asyncio.create_task(+ instance.run(+ {+ "node_id": "agent-1",+ "execution_id": "root-run",+ "team_task_id": "own-permit",+ "nodes": [],+ "edges": [],+ }+ )+ )+ await asyncio.wait_for(paused_gate.wait(), timeout=1)+ assert not preflight_started.is_set()++ await instance.resume()+ with pytest.raises(asyncio.CancelledError):+ await asyncio.wait_for(task, timeout=1)+ assert preflight_started.is_set()+++async def test_agent_rechecks_pause_before_task_manager_child_start(monkeypatch):+ from services.temporal import agent_workflow as agent_module++ temporal_workflow = agent_module.workflow+ monkeypatch.setattr(temporal_workflow, "logger", MagicMock())+ monkeypatch.setattr(temporal_workflow, "patched", lambda _patch_id: True)+ monkeypatch.setattr(+ temporal_workflow,+ "info",+ lambda: SimpleNamespace(+ workflow_id="agent-run",+ run_id="run-id-12345678",+ ),+ )+ monkeypatch.setattr(+ agent_module,+ "get_node_class",+ lambda _node_type: SimpleNamespace(needs_canvas=False),+ )++ instance = AgentWorkflow()+ paused_gate = asyncio.Event()+ child_started = asyncio.Event()+ llm_step = 0++ async def execute_activity(name, *, args, **_kwargs):+ nonlocal llm_step+ if name == "agent.prepare_payload.v1":+ return _mixed_preflight_payload()+ if name == "agent.broadcast_progress.v1":+ return {"emitted": True}+ if name == "agent.execute_llm_step.v1":+ llm_step += 1+ if llm_step == 1:+ return {+ "kind": "tool_calls",+ "calls": [+ {+ "id": "assign-1",+ "name": "manage_tasks",+ "args": {+ "operation": "assign_task",+ "task": "follow up",+ },+ }+ ],+ "usage": {},+ }+ return {"kind": "final", "content": "done", "usage": {}}+ if name == "agent.queue_delegation.v1":+ instance._control_paused = True+ return {"queued": True}+ if name == "agent.store_output.v1":+ return {"stored": True}+ if name == "agent.skill.clear.v1":+ return {"cleared": True}+ raise AssertionError(f"Unexpected activity {name}")++ def start_activity(*_args, **_kwargs):+ handle = asyncio.get_running_loop().create_future()+ handle.set_result(+ {+ "delegation_request": {+ "delegate_name": "delegate_to_child",+ "assignee_node_id": "child-agent",+ "team_task_id": "task-1",+ "task": "follow up",+ "context": "",+ }+ }+ )+ return handle++ async def start_child(*_args, **_kwargs):+ child_started.set()+ return MagicMock()++ monkeypatch.setattr(temporal_workflow, "execute_activity", execute_activity)+ monkeypatch.setattr(temporal_workflow, "start_activity", start_activity)+ monkeypatch.setattr(temporal_workflow, "start_child_workflow", start_child)+ monkeypatch.setattr(+ temporal_workflow,+ "wait_condition",+ lambda predicate: _wait_for_predicate(predicate, paused_gate),+ )++ task = asyncio.create_task(+ instance.run(+ {+ "node_id": "agent-1",+ "execution_id": "root-run",+ "nodes": [],+ "edges": [],+ }+ )+ )+ await asyncio.wait_for(paused_gate.wait(), timeout=1)+ assert not child_started.is_set()++ await instance.resume()+ result = await asyncio.wait_for(task, timeout=1)++ assert child_started.is_set()+ assert result["success"] is True++@pytest.mark.asyncioasync def test_controller_routes_push_events_without_listener_workflow():controller = WorkflowControlWorkflow()diff --git a/server/tests/test_cron_canary.py b/server/tests/test_cron_canary.pyindex 78c8e6b0..3df23cea 100644--- a/server/tests/test_cron_canary.py+++ b/server/tests/test_cron_canary.py@@ -27,7 +27,7 @@ from __future__ import annotationsimport sysimport typesfrom typing import Any, Dict, List-from unittest.mock import AsyncMock, MagicMock+from unittest.mock import AsyncMock, MagicMock, sentinelimport pytest@@ -159,6 +159,15 @@ class TestCreateCronSchedule:assert passed_id == schedule_idassert passed_schedule.spec.cron_expressions == ["*/5 * * * *"]assert passed_schedule.spec.time_zone_name == "America/New_York"+ action_attributes = {+ pair.key.name: pair.value+ for pair in passed_schedule.action.typed_search_attributes+ }+ assert action_attributes == {+ "EventWorkflowId": "wf-1",+ "TriggerNodeId": "cron-1",+ "EventTriggerKind": "cron",+ }@pytest.mark.asyncioasync def test_create_schedule_idempotent_on_already_running(self):@@ -171,6 +180,8 @@ class TestCreateCronSchedule:client = MagicMock()client.create_schedule = AsyncMock(side_effect=ScheduleAlreadyRunningError())+ update = AsyncMock()+ client.get_schedule_handle.return_value = MagicMock(update=update)schedule_id = await create_cron_schedule(client,@@ -184,6 +195,79 @@ class TestCreateCronSchedule:)assert schedule_id == "wf-1-cron-1"+ client.get_schedule_handle.assert_called_once_with(schedule_id)+ update.assert_awaited_once()++ updater = update.await_args.args[0]+ existing_schedule = client.create_schedule.await_args.args[1]+ existing_attributes = (+ client.create_schedule.await_args.kwargs["search_attributes"]+ )+ update_result = updater(+ types.SimpleNamespace(+ description=types.SimpleNamespace(+ schedule=types.SimpleNamespace(+ state=sentinel.paused_state,+ action=existing_schedule.action,+ ),+ typed_search_attributes=existing_attributes,+ ),+ )+ )+ assert update_result.schedule.state is sentinel.paused_state+ action_attributes = {+ pair.key.name: pair.value+ for pair in update_result.schedule.action.typed_search_attributes+ }+ assert action_attributes["EventWorkflowId"] == "wf-1"+ assert update_result.search_attributes == (+ update_result.schedule.action.typed_search_attributes+ )++ with pytest.raises(+ RuntimeError,+ match="cron_schedule_ownership_conflict",+ ):+ updater(+ types.SimpleNamespace(+ description=types.SimpleNamespace(+ schedule=types.SimpleNamespace(+ state=sentinel.paused_state,+ action=existing_schedule.action,+ ),+ typed_search_attributes=[+ types.SimpleNamespace(+ key=types.SimpleNamespace(+ name="EventWorkflowId",+ ),+ value="different-workflow",+ ),+ ],+ ),+ )+ )++ @pytest.mark.asyncio+ async def test_cron_workflow_pause_resume_gate(self, monkeypatch):+ from nodes.scheduler.cron_scheduler import _workflow as cron_workflow++ wait_condition = AsyncMock()+ monkeypatch.setattr(+ cron_workflow.workflow,+ "wait_condition",+ wait_condition,+ )+ instance = cron_workflow.CronTriggerWorkflow()++ await instance.pause()+ assert instance._control_paused is True+ await instance._wait_until_resumed()+ wait_condition.assert_awaited_once()+ predicate = wait_condition.await_args.args[0]+ assert predicate() is False++ await instance.resume()+ assert predicate() is Trueclass _FakeScheduleIterator:@@ -291,6 +375,68 @@ class TestDeleteCronSchedulesForDeployment:# + skipped.assert count == 1+ @pytest.mark.asyncio+ async def test_strict_cleanup_raises_after_sweeping_all_schedules(self):+ from services.temporal.schedules import (+ delete_cron_schedules_for_deployment,+ )++ deleted_ids: List[str] = []++ async def fake_list(query, **kwargs):+ return _FakeScheduleIterator(+ ["cron-schedule-wf-1-bad", "cron-schedule-wf-1-good"],+ )++ def fake_get_handle(sid):+ handle = MagicMock()+ if "bad" in sid:+ handle.delete = AsyncMock(+ side_effect=RuntimeError("simulated delete failure"),+ )+ else:+ handle.delete = AsyncMock(+ side_effect=lambda: deleted_ids.append(sid),+ )+ return handle++ client = MagicMock()+ client.list_schedules = fake_list+ client.get_schedule_handle = fake_get_handle++ with pytest.raises(+ RuntimeError,+ match="cron_schedule_cleanup_failed:1",+ ):+ await delete_cron_schedules_for_deployment(+ client,+ "wf-1",+ strict=True,+ )++ assert deleted_ids == ["cron-schedule-wf-1-good"]++ @pytest.mark.asyncio+ async def test_strict_cleanup_raises_when_visibility_scan_fails(self):+ from services.temporal.schedules import (+ delete_cron_schedules_for_deployment,+ )++ client = MagicMock()+ client.list_schedules = AsyncMock(+ side_effect=RuntimeError("visibility unavailable"),+ )++ with pytest.raises(+ RuntimeError,+ match="cron_schedule_visibility_cleanup_failed",+ ):+ await delete_cron_schedules_for_deployment(+ client,+ "wf-1",+ strict=True,+ )+def test_source_awaits_list_schedules_before_async_for(self):"""Regression: ``Client.list_schedules`` is ``async def`` inthe temporalio SDK. Bare ``async for ... in client.list_schedules(...)``diff --git a/server/tests/test_deployment_canary_listener.py b/server/tests/test_deployment_canary_listener.pyindex d2573aac..a6a94a5e 100644--- a/server/tests/test_deployment_canary_listener.py+++ b/server/tests/test_deployment_canary_listener.py@@ -673,6 +673,8 @@ class TestCancelSweepsStuckNodeStatuses:edges=[],session_id="sess",)+ mgr._paused_workflows.add("wf-1")+ mgr._paused_events["wf-1"] = [("trigger-1", {"event": "queued"})]# Patch _cancel_canary_listeners / _cancel_canary_cron_schedules# to be no-ops so the test focuses on the sweep + terminal@@ -682,6 +684,8 @@ class TestCancelSweepsStuckNodeStatuses:result = await mgr.cancel("wf-1")assert result["success"] is True+ assert "wf-1" not in mgr._paused_workflows+ assert "wf-1" not in mgr._paused_events# Sweep ran once with include_waiting=True (every indicator# goes quiet on explicit user cancel).@@ -696,6 +700,74 @@ class TestCancelSweepsStuckNodeStatuses:f"Expected update_workflow_status(executing=False, " f"workflow_id='wf-1') on cancel; got {status_calls!r}")+ @pytest.mark.asyncio+ async def test_cancel_clears_pause_state_when_deployment_is_already_absent(self):+ """Reset may cancel after another path has already torn deployment down."""+ from services.deployment.manager import DeploymentManager++ mgr = DeploymentManager(+ database=MagicMock(),+ execute_workflow_fn=AsyncMock(return_value={"success": True}),+ store_output_fn=AsyncMock(),+ broadcaster=MagicMock(),+ )+ mgr._paused_workflows.update({"wf-gone", "wf-other"})+ mgr._paused_events.update(+ {+ "wf-gone": [("trigger-1", {"event": "stale"})],+ "wf-other": [("trigger-2", {"event": "keep"})],+ }+ )++ result = await mgr.cancel("wf-gone")++ assert result == {+ "success": False,+ "error": "Workflow wf-gone is not deployed",+ }+ assert "wf-gone" not in mgr._paused_workflows+ assert "wf-gone" not in mgr._paused_events+ assert "wf-other" in mgr._paused_workflows+ assert "wf-other" in mgr._paused_events++ @pytest.mark.asyncio+ async def test_cancel_failure_keeps_admission_closed_and_events_queued(self):+ """A partial teardown must not reopen a deployment that still exists."""+ from services.deployment.manager import DeploymentManager+ from services.deployment.state import DeploymentState++ broadcaster = MagicMock()+ broadcaster.update_node_status = AsyncMock()+ mgr = DeploymentManager(+ database=MagicMock(),+ execute_workflow_fn=AsyncMock(return_value={"success": True}),+ store_output_fn=AsyncMock(),+ broadcaster=broadcaster,+ )+ mgr._deployments["wf-1"] = DeploymentState(+ deployment_id="deploy_wf-1",+ workflow_id="wf-1",+ is_running=True,+ nodes=[],+ edges=[],+ session_id="sess",+ )+ mgr._paused_events["wf-1"] = [+ ("trigger-1", {"event": "queued"}),+ ]+ mgr._cancel_canary_listeners = AsyncMock(+ side_effect=RuntimeError("temporal cleanup failed"),+ )++ with pytest.raises(RuntimeError, match="temporal cleanup failed"):+ await mgr.cancel("wf-1")++ assert "wf-1" in mgr._paused_workflows+ assert mgr._paused_events["wf-1"] == [+ ("trigger-1", {"event": "queued"}),+ ]+ assert "wf-1" in mgr._deployments+class TestNoInstanceStateForCanaryListeners:"""Lock the architectural decision: DeploymentManager does NOT keep