Skip to content

Self-improving AI Employees for everyone

feat: add Temporal workflow control and team traces

trohitg0d9ed8a42777
diff --git a/.env.template b/.env.template
index 1d68bdd2..73c541fc 100644
--- a/.env.template
+++ b/.env.template
@@ -100,7 +100,9 @@ TEMPORAL_SQLITE_PATH=temporal.db
# Visibility yet — resumed workflows would otherwise keep executing
# but be invisible to the OpenCompany UI. Flip to ``false`` once the
# reconcile lands to re-enable durable resumption.
-TEMPORAL_TERMINATE_RUNNING_ON_STARTUP=true
+# Durable workflow-control generations reconcile active Temporal executions.
+# Keep false so running and paused deployments survive an application restart.
+TEMPORAL_TERMINATE_RUNNING_ON_STARTUP=false
# Graceful shutdown window for the Temporal child process.
TEMPORAL_GRACEFUL_SHUTDOWN_SECONDS=30
# Supervisor readiness-probe window. The CLI binary may need to
diff --git a/client/src/Dashboard.tsx b/client/src/Dashboard.tsx
index fa1776ac..51a833f9 100644
--- a/client/src/Dashboard.tsx
+++ b/client/src/Dashboard.tsx
@@ -221,7 +221,7 @@ const DashboardContent: React.FC = () => {
} = useWorkflowManagement();
const { collapsedSections, searchQuery, setSearchQuery, toggleSection } = useComponentPalette();
- const { saveNodeParameters, getAllNodeParameters, executeWorkflow, deployWorkflow, cancelDeployment, cancelExecution, getWorkflowStatus, nodeStatuses, deploymentStatus, workflowLock, isConnected, isReady, sendRequest } = useWebSocket();
+ const { saveNodeParameters, getAllNodeParameters, executeWorkflow, deployWorkflow, getWorkflowStatus, nodeStatuses, deploymentStatus, workflowControlStatuses, startWorkflow, pauseWorkflow, resumeWorkflow, resetWorkflow, getWorkflowControlStatus, workflowLock, isConnected, isReady, sendRequest } = useWebSocket();
const applyUIDefaults = useAppStore((state) => state.applyUIDefaults);
// Workflows list: server-owned data, cached by TanStack Query.
@@ -237,7 +237,13 @@ const DashboardContent: React.FC = () => {
// current workflow (ad-hoc node, whole-workflow run, deployed trigger
// run, or deployment registration). The toolbar Start/Stop button
// tracks this unified signal so it stays Stop while nodes glow.
- const isCurrentWorkflowActive = isExecuting || isCurrentWorkflowDeployed;
+ const workflowControl = currentWorkflow?.id
+ ? workflowControlStatuses[currentWorkflow.id] || {
+ workflow_id: currentWorkflow.id, state: 'never_started' as const, revision: 0,
+ active_count: 0, in_flight_count: 0, queued_count: 0,
+ can_start: true, can_pause: false, can_resume: false, can_reset: false,
+ }
+ : { state: 'never_started' as const, revision: 0, active_count: 0, in_flight_count: 0, queued_count: 0, can_start: false, can_pause: false, can_resume: false, can_reset: false };
const isCurrentWorkflowLocked = workflowLock.locked &&
workflowLock.workflow_id === currentWorkflow?.id;
const [globalModelDefaults, setGlobalModelDefaults] = React.useState<{ provider: string; model: string } | null>(null);
@@ -399,7 +405,10 @@ const DashboardContent: React.FC = () => {
let cancelled = false;
(async () => {
try {
- const { executing } = await getWorkflowStatus(wfId);
+ const [{ executing }] = await Promise.all([
+ getWorkflowStatus(wfId),
+ getWorkflowControlStatus(wfId),
+ ]);
if (cancelled) return;
useAppStore.getState().setWorkflowExecuting(wfId, executing);
} catch {
@@ -407,7 +416,7 @@ const DashboardContent: React.FC = () => {
}
})();
return () => { cancelled = true; };
- }, [isReady, currentWorkflow?.id, getWorkflowStatus]);
+ }, [isReady, currentWorkflow?.id, getWorkflowStatus, getWorkflowControlStatus]);
// Load UI defaults from database on initial WebSocket connection
const hasLoadedUIDefaults = React.useRef(false);
@@ -775,12 +784,7 @@ const DashboardContent: React.FC = () => {
})),
});
- const result = await deployWorkflow(currentWorkflow.id, nodes, edges, 'default');
-
- if (!result.success) {
- console.error('[Dashboard] Deployment failed:', result.error);
- alert(`Failed to start deployment: ${result.error}`);
- }
+ await startWorkflow(currentWorkflow.id, nodes, edges, 'default', workflowControl.revision);
} catch (error: any) {
console.error('[Dashboard] Deployment error:', error);
alert(`Deployment error: ${error.message}`);
@@ -807,31 +811,28 @@ const DashboardContent: React.FC = () => {
}
}, [openExampleAndChat, deployWorkflow, deploymentStatus.isRunning, deploymentStatus.workflow_id]);
- // Stop click handler — routes to the right cancel based on what's actually
- // running. If the workflow is deployed, cancel the deployment (existing
- // path). Otherwise cancel any in-flight ad-hoc execution(s). Falls
- // through silently if neither is true.
- const handleCancelDeployment = async () => {
+ const handlePauseWorkflow = async () => {
const workflowId = currentWorkflow?.id;
if (!workflowId) return;
try {
- if (isCurrentWorkflowDeployed) {
- console.log('[Dashboard] Cancelling deployment for workflow:', workflowId);
- const result = await cancelDeployment(workflowId);
- if (!result?.success) {
- console.error('[Dashboard] Failed to cancel deployment:', result?.message);
- }
- return;
- }
- if (isExecuting) {
- console.log('[Dashboard] Cancelling ad-hoc execution for workflow:', workflowId);
- await cancelExecution(workflowId);
- }
+ await pauseWorkflow(workflowId, workflowControl.revision);
} catch (error: any) {
- console.error('[Dashboard] Cancel error:', error);
+ toast.error(error?.message || 'Unable to pause workflow');
}
};
+ const handleResumeWorkflow = async () => {
+ if (!currentWorkflow?.id) return;
+ try { await resumeWorkflow(currentWorkflow.id, workflowControl.revision); }
+ catch (error: any) { toast.error(error?.message || 'Unable to resume workflow'); }
+ };
+
+ const handleResetWorkflow = async () => {
+ if (!currentWorkflow?.id) return;
+ try { await resetWorkflow(currentWorkflow.id, workflowControl.revision); }
+ catch (error: any) { toast.error(error?.message || 'Unable to reset workflow'); }
+ };
+
// Helper: fetch all node parameters from DB for export
const fetchNodeParametersForExport = async (): Promise<Record<string, Record<string, any>>> => {
if (!currentWorkflow?.nodes.length) return {};
@@ -1259,9 +1260,11 @@ const DashboardContent: React.FC = () => {
onOpen={handleOpen}
onRun={withSound('run', handleRun)}
isRunning={isExecuting}
- onDeploy={withSound('run', handleDeploy)}
- onCancelDeployment={handleCancelDeployment}
- isDeploying={isCurrentWorkflowActive}
+ workflowControl={workflowControl}
+ onStartWorkflow={withSound('run', handleDeploy)}
+ onPauseWorkflow={handlePauseWorkflow}
+ onResumeWorkflow={withSound('run', handleResumeWorkflow)}
+ onResetWorkflow={handleResetWorkflow}
hasUnsavedChanges={hasUnsavedChanges}
sidebarVisible={sidebarVisible}
onToggleSidebar={toggleSidebar}
@@ -1421,9 +1424,11 @@ const DashboardContent: React.FC = () => {
save: withSound('save', handleSave),
newWorkflow: handleNew,
open: handleOpen,
- run: withSound('run', handleDeploy),
- stop: handleCancelDeployment,
- isDeploying: isCurrentWorkflowActive,
+ start: withSound('run', handleDeploy),
+ pause: handlePauseWorkflow,
+ resume: withSound('run', handleResumeWorkflow),
+ reset: handleResetWorkflow,
+ workflowControl,
exportFile: handleExportFile,
importJSON: handleImportJSON,
openSettings: () => setSettingsOpen(true),
diff --git a/client/src/components/TriggerNode.tsx b/client/src/components/TriggerNode.tsx
index 85e90131..1b1c624e 100644
--- a/client/src/components/TriggerNode.tsx
+++ b/client/src/components/TriggerNode.tsx
@@ -29,6 +29,7 @@ const TriggerNode: React.FC<NodeProps<NodeData>> = ({ id, type, data, isConnecta
// not re-render this trigger.
const nodeStatus = useNodeStatus(id);
const executionStatus = nodeStatus?.status || 'idle';
+ const isPaused = nodeStatus?.data?.paused === true;
// Check if this is a WhatsApp trigger
const isWhatsAppTrigger = type === 'whatsappReceive';
@@ -87,6 +88,7 @@ const TriggerNode: React.FC<NodeProps<NodeData>> = ({ id, type, data, isConnecta
})();
const getStatusTitle = () => {
+ if (isPaused) return nodeStatus?.data?.message || 'Trigger paused';
switch (executionStatus) {
case 'executing':
return 'Executing...';
@@ -233,7 +235,7 @@ const TriggerNode: React.FC<NodeProps<NodeData>> = ({ id, type, data, isConnecta
width: theme.nodeSize.outputBadge,
height: theme.nodeSize.outputBadge,
borderRadius: theme.borderRadius.sm,
- backgroundColor: 'var(--warning)',
+ backgroundColor: isPaused ? 'var(--fg-faint)' : 'var(--warning)',
border: `1px solid ${theme.isDarkMode ? theme.colors.background : '#ffffff'}`,
display: 'flex',
alignItems: 'center',
@@ -244,9 +246,9 @@ const TriggerNode: React.FC<NodeProps<NodeData>> = ({ id, type, data, isConnecta
? `0 1px 3px ${theme.colors.shadow}`
: '0 1px 3px rgba(0,0,0,0.15)',
}}
- title="Trigger Node - Starts workflow execution"
+ title={isPaused ? 'Trigger paused — events will run after resume' : 'Trigger Node - Starts workflow execution'}
>
- <span style={{ lineHeight: 1, color: theme.isDarkMode ? theme.colors.background : '#1a1d21' }}>⚡</span>
+ <span style={{ lineHeight: 1, color: theme.isDarkMode ? theme.colors.background : '#1a1d21' }}>{isPaused ? 'Ⅱ' : '⚡'}</span>
</div>
{/* Output Handle (right side) — CSS owns bg/border via .sq-node-handle.out */}
diff --git a/client/src/components/parameterPanel/TaskManagerPanel.tsx b/client/src/components/parameterPanel/TaskManagerPanel.tsx
index 2a4aaae8..ffaf1fc9 100644
--- a/client/src/components/parameterPanel/TaskManagerPanel.tsx
+++ b/client/src/components/parameterPanel/TaskManagerPanel.tsx
@@ -1,9 +1,9 @@
import React, { useCallback, useEffect, useMemo, useState } from 'react';
-import { Eye, Filter, RefreshCw, Search } from 'lucide-react';
+import { Activity, AlertTriangle, Eye, Filter, RefreshCw, Search } from 'lucide-react';
import { toast } from 'sonner';
import type { Edge, Node } from 'reactflow';
-import { useWebSocket } from '@/contexts/WebSocketContext';
+import { useWebSocket, type TeamTaskTraceResponse } from '@/contexts/WebSocketContext';
import { Badge } from '@/components/ui/badge';
import { Button } from '@/components/ui/button';
import { Input } from '@/components/ui/input';
@@ -27,6 +27,9 @@ export interface TeamTaskAttempt {
error?: string;
started_at?: string;
completed_at?: string;
+ workflow_id?: string;
+ run_id?: string;
+ trace_id?: string;
}
export interface TeamTaskView {
@@ -138,7 +141,7 @@ const taskUsage = (task: TeamTaskView) => {
interface Props { nodeId: string; workflowId?: string; nodes: Node[]; edges: Edge[] }
const TaskManagerPanel: React.FC<Props> = ({ nodeId, workflowId, nodes, edges }) => {
- const { sendRequest, addEventListener } = useWebSocket();
+ const { sendRequest, addEventListener, getTeamTaskTrace } = useWebSocket();
const [status, setStatus] = useState<TeamStatusView | null>(null);
const [loading, setLoading] = useState(false);
const [search, setSearch] = useState('');
@@ -150,7 +153,12 @@ const TaskManagerPanel: React.FC<Props> = ({ nodeId, workflowId, nodes, edges })
const [assignee, setAssignee] = useState('');
const [busy, setBusy] = useState(false);
const [page, setPage] = useState(0);
- const [nowMs, setNowMs] = useState(() => Date.now());
+ const [nowMs, setNowMs] = useState(0);
+ const [detailTab, setDetailTab] = useState<'details' | 'trace'>('details');
+ const [trace, setTrace] = useState<TeamTaskTraceResponse | null>(null);
+ const [traceLoading, setTraceLoading] = useState(false);
+ const [traceAttempt, setTraceAttempt] = useState('current');
+ const [traceSearch, setTraceSearch] = useState('');
const pageSize = 25;
const connectedLead = useMemo(() => {
@@ -227,10 +235,44 @@ const TaskManagerPanel: React.FC<Props> = ({ nodeId, workflowId, nodes, edges })
useEffect(() => setPage(0), [filter, search, executionId]);
useEffect(() => {
+ setNowMs(Date.now());
const timer = window.setInterval(() => setNowMs(Date.now()), 1_000);
return () => window.clearInterval(timer);
}, []);
+ const fetchTrace = useCallback(async (cursor?: string) => {
+ if (!selected || !workflowId || !connectedLead || !status?.execution_id) return;
+ setTraceLoading(true);
+ try {
+ const response = await getTeamTaskTrace({
+ workflow_id: workflowId,
+ team_lead_node_id: connectedLead.id,
+ execution_id: status.execution_id,
+ task_id: selected.id,
+ ...(traceAttempt !== 'current' ? { attempt: Number(traceAttempt) } : {}),
+ ...(cursor ? { cursor } : {}),
+ limit: 50,
+ detail: 'timeline',
+ });
+ setTrace((previous) => cursor && previous
+ ? { ...response, events: [...previous.events, ...response.events] }
+ : response);
+ } catch (error) {
+ setTrace({ state: 'temporal_unavailable', events: [], error: error instanceof Error ? error.message : 'Unable to load trace' });
+ } finally { setTraceLoading(false); }
+ }, [connectedLead, getTeamTaskTrace, selected, status?.execution_id, traceAttempt, workflowId]);
+
+ useEffect(() => {
+ if (detailTab !== 'trace' || !selected) return;
+ setTrace(null);
+ void fetchTrace();
+ }, [detailTab, fetchTrace, selected, traceAttempt]);
+
+ const traceEvents = useMemo(() => (trace?.events || []).filter((event) => {
+ const value = `${event.event_type} ${event.name || ''} ${event.summary || ''} ${event.failure || ''}`.toLowerCase();
+ return value.includes(traceSearch.toLowerCase());
+ }), [trace, traceSearch]);
+
const runAction = async () => {
if (!pending || !workflowId || !connectedLead) return;
setBusy(true);
@@ -296,7 +338,9 @@ const TaskManagerPanel: React.FC<Props> = ({ nodeId, workflowId, nodes, edges })
{pages > 1 && <div className="mt-3 flex items-center justify-end gap-2 text-sm text-fg-muted"><Button variant="outline" size="sm" disabled={page === 0} onClick={() => setPage(page - 1)}>Previous</Button><span>{page + 1} / {pages}</span><Button variant="outline" size="sm" disabled={page + 1 >= pages} onClick={() => setPage(page + 1)}>Next</Button></div>}
</div>
- <Dialog open={!!selected} onOpenChange={(open) => !open && setSelected(null)}><DialogContent className="max-h-[85vh] max-w-3xl overflow-y-auto"><DialogHeader><DialogTitle>{selected?.title}</DialogTitle></DialogHeader>{selected && <div className="space-y-5 text-sm">
+ <Dialog open={!!selected} onOpenChange={(open) => { if (!open) { setSelected(null); setDetailTab('details'); setTrace(null); } }}><DialogContent className="max-h-[85vh] max-w-4xl overflow-y-auto"><DialogHeader><DialogTitle>{selected?.title}</DialogTitle></DialogHeader>{selected && <div className="space-y-5 text-sm">
+ <div className="flex gap-1 border-b border-border-default"><Button variant={detailTab === 'details' ? 'secondary' : 'ghost'} size="sm" onClick={() => setDetailTab('details')}>Details</Button><Button variant={detailTab === 'trace' ? 'secondary' : 'ghost'} size="sm" onClick={() => setDetailTab('trace')}><Activity className="h-4 w-4" /> Temporal trace</Button></div>
+ {detailTab === 'details' && <>
<div className="flex flex-wrap items-center gap-2"><Badge variant="outline" className={cn('capitalize', STATUS_STYLES[selected.status])}>{selected.status}</Badge><span className="text-fg-muted">Revision {selected.revision ?? 0} · Attempt {selected.current_attempt ?? 0} · {elapsed(selected, nowMs)}</span></div>
<section className="grid gap-2 rounded border border-border-default p-3 text-xs text-fg-muted sm:grid-cols-3"><span>Created<br/><strong className="font-medium text-fg-default">{formatTimestamp(selected.created_at)}</strong></span><span>Started<br/><strong className="font-medium text-fg-default">{formatTimestamp(selected.started_at)}</strong></span><span>Completed<br/><strong className="font-medium text-fg-default">{formatTimestamp(selected.completed_at)}</strong></span></section>
<section><h3 className="mb-2 font-medium">Token usage</h3><div className="grid grid-cols-3 gap-2 rounded border border-border-default p-3 text-center"><div><div className="text-xs text-fg-muted">Input</div><div className="font-semibold tabular-nums">{taskUsage(selected).input.toLocaleString()}</div></div><div><div className="text-xs text-fg-muted">Output</div><div className="font-semibold tabular-nums">{taskUsage(selected).output.toLocaleString()}</div></div><div><div className="text-xs text-fg-muted">Total</div><div className="font-semibold tabular-nums">{taskUsage(selected).total.toLocaleString()}</div></div></div></section>
@@ -307,6 +351,13 @@ const TaskManagerPanel: React.FC<Props> = ({ nodeId, workflowId, nodes, edges })
<section><h3 className="mb-2 font-medium">Attempt history</h3><div className="space-y-2">{(selected.attempts || []).map((attempt) => <div key={attempt.id} className="rounded border border-border-default p-3"><div className="flex justify-between"><span>Attempt {attempt.attempt_number} · {attempt.assignee_node_id || 'Unassigned'}</span><Badge variant="outline">{attempt.status}</Badge></div>{attempt.error && <p className="mt-2 text-destructive">{attempt.error}</p>}</div>)}{!selected.attempts?.length && <p className="text-fg-muted">No previous attempts.</p>}</div></section>
<section className="grid grid-cols-2 gap-2 rounded border border-border-default p-3 text-xs text-fg-muted"><span>Trace: {selected.trace_id || '—'}</span><span>Child workflow: {selected.child_workflow_id || '—'}</span><span>Child run: {selected.child_run_id || '—'}</span><span>Dependencies: {selected.depends_on?.join(', ') || 'None'}</span></section>
<div className="flex flex-wrap justify-end gap-2">{actionsFor(selected).map((action) => <Button key={action} variant={action === 'cancel' ? 'destructive' : action === 'accept' ? 'default' : 'outline'} onClick={() => { setSelected(null); openAction(action.replace('_task', ''), action.replace('_task','').replace('_',' '), selected); }}>{action.replace('_task','').replace('_',' ')}</Button>)}</div>
+ </>}
+ {detailTab === 'trace' && <section className="space-y-3">
+ <div className="flex flex-wrap gap-2"><Select value={traceAttempt} onValueChange={setTraceAttempt}><SelectTrigger className="w-44" aria-label="Trace attempt"><SelectValue /></SelectTrigger><SelectContent><SelectItem value="current">Current attempt</SelectItem>{(selected.attempts || []).map((attempt) => <SelectItem key={attempt.id} value={String(attempt.attempt_number)}>Attempt {attempt.attempt_number}</SelectItem>)}</SelectContent></Select><div className="relative min-w-52 flex-1"><Search className="absolute left-3 top-1/2 h-4 w-4 -translate-y-1/2 text-fg-muted"/><Input className="pl-9" value={traceSearch} onChange={(event) => setTraceSearch(event.target.value)} placeholder="Search events, activities, failures"/></div><Button variant="outline" onClick={() => void fetchTrace()} disabled={traceLoading}><RefreshCw className={cn('h-4 w-4', traceLoading && 'animate-spin')}/> Refresh</Button></div>
+ {trace?.state === 'available' && <><div className="grid gap-2 rounded border border-border-default p-3 text-xs text-fg-muted sm:grid-cols-3"><span>Workflow<br/><strong className="break-all font-medium text-fg-default">{trace.workflow_id || selected.child_workflow_id || '—'}</strong></span><span>Run<br/><strong className="break-all font-medium text-fg-default">{trace.run_id || selected.child_run_id || '—'}</strong></span><span>Trace<br/><strong className="break-all font-medium text-fg-default">{trace.trace_id || selected.trace_id || '—'}</strong></span></div><ol className="divide-y divide-border-default overflow-hidden rounded border border-border-default">{traceEvents.map((event) => <li key={event.event_id} className="grid grid-cols-[9rem_minmax(0,1fr)] gap-3 p-3"><time className="text-xs text-fg-muted">{formatTimestamp(event.timestamp)}</time><div className="min-w-0"><div className="flex items-center gap-2"><Badge variant="outline">{event.category || event.event_type}</Badge><span className="truncate font-medium">{event.name || event.event_type}</span></div>{event.summary && <p className="mt-1 whitespace-pre-wrap text-xs text-fg-muted">{event.summary}</p>}{event.failure && <p className="mt-2 whitespace-pre-wrap rounded bg-destructive/10 p-2 text-xs text-destructive"><AlertTriangle className="mr-1 inline h-3 w-3"/>{event.failure}</p>}</div></li>)}{!traceEvents.length && <li className="p-8 text-center text-fg-muted">No trace events match this view.</li>}</ol>{trace.next_cursor && <Button variant="outline" className="w-full" disabled={traceLoading} onClick={() => void fetchTrace(trace.next_cursor || undefined)}>Load older events</Button>}</>}
+ {trace && trace.state !== 'available' && <div className="rounded border border-warning/30 bg-warning/10 p-6 text-center"><AlertTriangle className="mx-auto mb-2 h-5 w-5 text-warning"/><p className="font-medium capitalize">{trace.state.replace(/_/g, ' ')}</p><p className="mt-1 text-sm text-fg-muted">{trace.error || (trace.state === 'retention_expired' ? 'Temporal history has expired; the durable task result remains available in Details.' : 'No Temporal execution history is currently available for this attempt.')}</p></div>}
+ {!trace && traceLoading && <div className="p-10 text-center text-fg-muted">Loading Temporal history…</div>}
+ </section>}
</div>}</DialogContent></Dialog>
<AlertDialog open={!!pending} onOpenChange={(open) => !open && setPending(null)}><AlertDialogContent><AlertDialogHeader><AlertDialogTitle className="capitalize">{pending?.label}</AlertDialogTitle><AlertDialogDescription>{pending?.task ? `Apply this action to “${pending.task.title}”? The latest task revision will be checked.` : 'Finish this team only if every required task is resolved.'}</AlertDialogDescription></AlertDialogHeader>
diff --git a/client/src/components/parameterPanel/TeamMonitorPanel.tsx b/client/src/components/parameterPanel/TeamMonitorPanel.tsx
index 6ab1684b..f6e5c132 100644
--- a/client/src/components/parameterPanel/TeamMonitorPanel.tsx
+++ b/client/src/components/parameterPanel/TeamMonitorPanel.tsx
@@ -17,7 +17,7 @@ const STATUS_STYLE: Record<string, string> = {
interface Props { nodeId: string; workflowId?: string; nodes: Node[]; edges: Edge[] }
interface Member { agent_node_id: string; agent_type?: string; label?: string; status?: string }
-interface Task { id: string; title: string; status: string; assigned_to?: string; assignee_label?: string; queue_position?: number }
+interface Task { id: string; title: string; status: string; assigned_to?: string; assignee_label?: string; queue_position?: number; child_workflow_id?: string; child_run_id?: string; temporal_status?: string; trace_id?: string }
const TeamMonitorPanel: React.FC<Props> = ({ nodeId, workflowId, nodes, edges }) => {
const { sendRequest, addEventListener } = useWebSocket();
@@ -76,7 +76,8 @@ const TeamMonitorPanel: React.FC<Props> = ({ nodeId, workflowId, nodes, edges })
useEffect(() => {
const update = () => void refresh(true);
const removers = ['team_event', 'team.task.queued', 'team.task.running', 'team.task.submitted',
- 'team.task.accepted', 'team.task.failed', 'team.task.cancelled'].map((event) => addEventListener(event, update));
+ 'team.task.accepted', 'team.task.failed', 'team.task.cancelled', 'workflow_control_status',
+ 'workflow.control.status'].map((event) => addEventListener(event, update));
return () => removers.forEach((remove) => remove());
}, [addEventListener, refresh]);
@@ -97,6 +98,7 @@ const TeamMonitorPanel: React.FC<Props> = ({ nodeId, workflowId, nodes, edges })
<div className="mt-4 grid grid-cols-4 gap-2 lg:grid-cols-7">
{counts.map(({ name, value }) => <div key={name} className="rounded-md border border-border-default bg-bg-elevated p-2"><div className="text-xs capitalize text-fg-muted">{name}</div><div className={cn('text-lg font-semibold tabular-nums', STATUS_STYLE[name])}>{value}</div></div>)}
</div>
+ <div className="mt-3 flex flex-wrap gap-x-5 gap-y-1 text-xs text-fg-muted"><span>Generation <strong className="text-fg-default">{status?.control_generation ?? status?.generation ?? '—'}</strong></span><span>Root execution <strong className="text-fg-default">{status?.root_execution_id ? String(status.root_execution_id).slice(0, 16) : '—'}</strong></span><span>Controller <strong className="text-fg-default">{status?.controller_workflow_id ? String(status.controller_workflow_id).slice(0, 20) : '—'}</strong></span><span>Temporal state <Badge variant="outline" className="ml-1 capitalize">{status?.control_state || status?.temporal_status || 'unavailable'}</Badge></span></div>
</div>
<div className="grid min-h-0 flex-1 grid-cols-1 gap-4 overflow-auto p-4 lg:grid-cols-[minmax(220px,0.75fr)_minmax(0,2fr)]">
<section className="rounded-md border border-border-default bg-bg-elevated">
@@ -105,7 +107,7 @@ const TeamMonitorPanel: React.FC<Props> = ({ nodeId, workflowId, nodes, edges })
</section>
<section className="min-w-0 rounded-md border border-border-default bg-bg-elevated">
<div className="border-b border-border-default px-4 py-3 font-medium">Current execution tasks</div>
- <div className="divide-y divide-border-default/70">{tasks.map((task) => <div key={task.id} className="grid grid-cols-[minmax(0,1fr)_auto_auto] items-center gap-4 px-4 py-3"><div className="truncate font-medium text-fg-default">{task.title}</div><span className="text-xs text-fg-muted">{task.assignee_label || task.assigned_to || 'Unassigned'}</span><Badge variant="outline" className={cn('capitalize', STATUS_STYLE[task.status])}>{task.status}{task.queue_position ? ` #${task.queue_position}` : ''}</Badge></div>)}{tasks.length === 0 && <div className="p-8 text-center text-sm text-fg-muted">No tasks in this execution.</div>}</div>
+ <div className="divide-y divide-border-default/70">{tasks.map((task) => <div key={task.id} className="grid grid-cols-[minmax(0,1fr)_auto_auto] items-center gap-4 px-4 py-3"><div className="min-w-0"><div className="truncate font-medium text-fg-default">{task.title}</div><div className="truncate text-[11px] text-fg-muted">{task.child_workflow_id ? `Temporal ${task.child_workflow_id}${task.child_run_id ? ` · ${task.child_run_id.slice(0, 8)}` : ''}` : 'Temporal execution not registered'}</div></div><span className="text-xs text-fg-muted">{task.assignee_label || task.assigned_to || 'Unassigned'}</span><div className="text-right"><Badge variant="outline" className={cn('capitalize', STATUS_STYLE[task.status])}>{task.status}{task.queue_position ? ` #${task.queue_position}` : ''}</Badge>{task.temporal_status && <div className="mt-1 text-[11px] capitalize text-fg-muted">{task.temporal_status}</div>}</div></div>)}{tasks.length === 0 && <div className="p-8 text-center text-sm text-fg-muted">No tasks in this execution.</div>}</div>
</section>
</div>
</div>;
diff --git a/client/src/components/ui/CommandPaletteHost.tsx b/client/src/components/ui/CommandPaletteHost.tsx
index a786bc46..a73649d1 100644
--- a/client/src/components/ui/CommandPaletteHost.tsx
+++ b/client/src/components/ui/CommandPaletteHost.tsx
@@ -18,7 +18,8 @@ import {
KeyRound,
Save,
Play,
- Square,
+ Pause,
+ RotateCcw,
FilePlus,
FolderOpen,
PanelLeftClose,
@@ -30,14 +31,17 @@ import {
} from 'lucide-react';
import { CommandPalette, type CommandItem } from './CommandPalette';
import { AVAILABLE_THEMES, useTheme, type ThemeName } from '../../contexts/ThemeContext';
+import type { WorkflowControlStatus } from '../../contexts/WebSocketContext';
interface Handlers {
save: () => void;
newWorkflow: () => void;
open: () => void;
- run: () => void;
- stop: () => void;
- isDeploying: boolean;
+ start: () => void;
+ pause: () => void;
+ resume: () => void;
+ reset: () => void;
+ workflowControl: WorkflowControlStatus;
exportFile: () => void;
importJSON: () => void;
openSettings: () => void;
@@ -112,21 +116,27 @@ export const CommandPaletteHost: React.FC<Props> = ({ open, onOpenChange, handle
},
// ── Run ────────────────────────────────────────────────────────
- handlers.isDeploying
+ handlers.workflowControl.state === 'running'
? {
- id: 'run.stop',
- label: 'Stop Workflow',
+ id: 'run.pause',
+ label: 'Pause Workflow',
group: 'Run',
- icon: Square,
- onRun: handlers.stop,
+ icon: Pause,
+ onRun: handlers.pause,
}
+ : handlers.workflowControl.state === 'paused'
+ ? { id: 'run.resume', label: 'Resume Workflow', group: 'Run', icon: Play, onRun: handlers.resume }
: {
id: 'run.start',
label: 'Start Workflow',
group: 'Run',
icon: Play,
- onRun: handlers.run,
+ onRun: handlers.start,
},
+ ...(handlers.workflowControl.can_reset ? [{
+ id: 'run.reset', label: 'Reset Workflow Execution', group: 'Run', icon: RotateCcw,
+ hint: 'terminates active work', onRun: handlers.reset,
+ } satisfies CommandItem] : []),
// ── Open panels ────────────────────────────────────────────────
{
diff --git a/client/src/components/ui/TopToolbar.tsx b/client/src/components/ui/TopToolbar.tsx
index 95827ef7..381ea9f0 100644
--- a/client/src/components/ui/TopToolbar.tsx
+++ b/client/src/components/ui/TopToolbar.tsx
@@ -17,7 +17,9 @@ import {
LogOut,
Save,
Play,
- Square,
+ Pause,
+ RotateCcw,
+ LoaderCircle,
Repeat,
Clock,
Zap,
@@ -41,6 +43,11 @@ import {
} from '@/components/ui/dropdown-menu';
import { Button } from '@/components/ui/button';
import { ActionButton } from '@/components/ui/action-button';
+import {
+ AlertDialog, AlertDialogAction, AlertDialogCancel, AlertDialogContent,
+ AlertDialogDescription, AlertDialogFooter, AlertDialogHeader, AlertDialogTitle,
+} from '@/components/ui/alert-dialog';
+import type { WorkflowControlStatus } from '../../contexts/WebSocketContext';
import { ThemeSwitcher } from '@/components/ui/ThemeSwitcher';
import { cn } from '@/lib/utils';
import { useAuth } from '../../contexts/AuthContext';
@@ -61,9 +68,11 @@ interface TopToolbarProps {
onOpen: () => void;
onRun: () => void;
isRunning?: boolean;
- onDeploy: () => void;
- onCancelDeployment: () => void;
- isDeploying?: boolean;
+ workflowControl: WorkflowControlStatus;
+ onStartWorkflow: () => void;
+ onPauseWorkflow: () => void;
+ onResumeWorkflow: () => void;
+ onResetWorkflow: () => void;
hasUnsavedChanges: boolean;
sidebarVisible: boolean;
onToggleSidebar: () => void;
@@ -88,9 +97,11 @@ const TopToolbar: React.FC<TopToolbarProps> = ({
onOpen,
onRun: _onRun,
isRunning = false,
- onDeploy,
- onCancelDeployment,
- isDeploying = false,
+ workflowControl,
+ onStartWorkflow,
+ onPauseWorkflow,
+ onResumeWorkflow,
+ onResetWorkflow,
hasUnsavedChanges,
sidebarVisible,
onToggleSidebar,
@@ -107,6 +118,7 @@ const TopToolbar: React.FC<TopToolbarProps> = ({
onOverrideAllAgents,
}) => {
const [isEditing, setIsEditing] = useState(false);
+ const [resetOpen, setResetOpen] = useState(false);
const [tempName, setTempName] = useState(workflowName);
const { user, logout } = useAuth();
@@ -391,28 +403,39 @@ const TopToolbar: React.FC<TopToolbarProps> = ({
<Divider />
- {/* Start / Stop */}
- {!isDeploying ? (
+ {/* Durable workflow lifecycle */}
+ {workflowControl.can_start ? (
<ActionButton
intent="run"
- onClick={onDeploy}
+ onClick={onStartWorkflow}
disabled={isRunning}
title="Start workflow"
>
<Play className="h-3 w-3 fill-current" />
Start
</ActionButton>
- ) : (
+ ) : workflowControl.state === 'paused' ? (
+ <ActionButton intent="run" onClick={onResumeWorkflow} title="Resume this workflow execution">
+ <Play className="h-3 w-3 fill-current" /> Resume
+ </ActionButton>
+ ) : workflowControl.state === 'running' ? (
<ActionButton
intent="stop"
- onClick={onCancelDeployment}
- title="Stop workflow"
+ onClick={onPauseWorkflow}
+ title="Pause new workflow scheduling after in-flight work finishes"
>
- <Square className="h-3 w-3 fill-current" />
- Stop
+ <Pause className="h-3 w-3 fill-current" /> Pause
+ </ActionButton>
+ ) : (
+ <ActionButton intent="run" disabled title={`Workflow is ${workflowControl.state}`}>
+ <LoaderCircle className="h-3 w-3 animate-spin" /> {workflowControl.state.replace('_', ' ')}
</ActionButton>
)}
+ {workflowControl.can_reset && <ActionButton intent="stop" onClick={() => setResetOpen(true)} title="Terminate and archive this execution"><RotateCcw className="h-3 w-3" /> Reset</ActionButton>}
+
+ <AlertDialog open={resetOpen} onOpenChange={setResetOpen}><AlertDialogContent><AlertDialogHeader><AlertDialogTitle>Reset workflow execution?</AlertDialogTitle><AlertDialogDescription>This immediately terminates active work and archives the current generation. Task history remains available. The workflow will remain stopped until you press Start.</AlertDialogDescription></AlertDialogHeader><AlertDialogFooter><AlertDialogCancel>Keep current execution</AlertDialogCancel><AlertDialogAction onClick={onResetWorkflow}>Reset workflow</AlertDialogAction></AlertDialogFooter></AlertDialogContent></AlertDialog>
+
<ActionButton
intent="save"
onClick={() => typeof onSave === 'function' && onSave()}
diff --git a/client/src/contexts/WebSocketContext.tsx b/client/src/contexts/WebSocketContext.tsx
index d281dff4..26b9e96a 100644
--- a/client/src/contexts/WebSocketContext.tsx
+++ b/client/src/contexts/WebSocketContext.tsx
@@ -109,6 +109,55 @@ export interface DeploymentStatus {
error?: string;
}
+export type WorkflowControlState =
+ | 'never_started' | 'ready' | 'starting' | 'running' | 'pausing' | 'paused'
+ | 'resuming' | 'resetting' | 'failed';
+
+export interface WorkflowControlStatus {
+ workflow_id?: string | null;
+ generation?: number;
+ execution_id?: string | null;
+ root_execution_id?: string | null;
+ controller_workflow_id?: string | null;
+ controller_run_id?: string | null;
+ state: WorkflowControlState;
+ revision: number;
+ active_count: number;
+ in_flight_count: number;
+ queued_count: number;
+ can_start: boolean;
+ can_pause: boolean;
+ can_resume: boolean;
+ can_reset: boolean;
+ started_at?: string | null;
+ updated_at?: string | null;
+ error?: string | null;
+}
+
+export interface TeamTaskTraceEvent {
+ event_id: string | number;
+ event_type: string;
+ timestamp?: string;
+ category?: string;
+ status?: string;
+ name?: string;
+ summary?: string;
+ failure?: string;
+ retry_state?: string;
+}
+
+export interface TeamTaskTraceResponse {
+ state: 'available' | 'retention_expired' | 'temporal_unavailable' | 'execution_not_registered';
+ task_id?: string;
+ attempt?: number;
+ workflow_id?: string;
+ run_id?: string;
+ trace_id?: string;
+ events: TeamTaskTraceEvent[];
+ next_cursor?: string | null;
+ error?: string;
+}
+
export interface WorkflowLock {
locked: boolean;
workflow_id: string | null;
@@ -302,6 +351,7 @@ interface WebSocketContextValue {
variables: Record<string, any>;
workflowStatus: WorkflowStatus;
deploymentStatus: DeploymentStatus;
+ workflowControlStatuses: Record<string, WorkflowControlStatus>;
workflowLock: WorkflowLock;
compactionStats: Record<string, CompactionStats>; // session_id -> stats (current workflow)
@@ -349,6 +399,12 @@ interface WebSocketContextValue {
getDeploymentStatus: (workflowId?: string) => Promise<{ isRunning: boolean; activeRuns: number; settings?: any; workflow_id?: string }>;
cancelExecution: (workflowId: string, nodeId?: string) => Promise<any>;
getWorkflowStatus: (workflowId: string) => Promise<{ executing: boolean }>;
+ startWorkflow: (workflowId: string, nodes: any[], edges: any[], sessionId?: string, expectedRevision?: number) => Promise<WorkflowControlStatus>;
+ pauseWorkflow: (workflowId: string, expectedRevision: number) => Promise<WorkflowControlStatus>;
+ resumeWorkflow: (workflowId: string, expectedRevision: number) => Promise<WorkflowControlStatus>;
+ resetWorkflow: (workflowId: string, expectedRevision: number) => Promise<WorkflowControlStatus>;
+ getWorkflowControlStatus: (workflowId: string) => Promise<WorkflowControlStatus>;
+ getTeamTaskTrace: (scope: Record<string, any>) => Promise<TeamTaskTraceResponse>;
// AI Operations
executeAiNode: (nodeId: string, nodeType: string, parameters: Record<string, any>, model: string, workflowId: string, nodes: any[], edges: any[]) => Promise<any>;
@@ -411,6 +467,38 @@ const defaultDeploymentStatus: DeploymentStatus = {
status: 'idle'
};
+const emptyWorkflowControlStatus = (workflowId?: string): WorkflowControlStatus => ({
+ workflow_id: workflowId || null,
+ state: 'never_started',
+ revision: 0,
+ active_count: 0,
+ in_flight_count: 0,
+ queued_count: 0,
+ can_start: true,
+ can_pause: false,
+ can_resume: false,
+ can_reset: false,
+});
+
+const normalizeWorkflowControlStatus = (value: any, workflowId?: string): WorkflowControlStatus => {
+ const source = value?.status && typeof value.status === 'object' ? value.status : value || {};
+ const state = source.state || source.control_state || (source.is_running ? 'running' : 'never_started');
+ return {
+ ...emptyWorkflowControlStatus(workflowId),
+ ...source,
+ workflow_id: source.workflow_id ?? workflowId ?? null,
+ state,
+ revision: Number(source.revision || 0),
+ active_count: Number(source.active_count ?? source.active_runs ?? 0),
+ in_flight_count: Number(source.in_flight_count ?? source.active_count ?? source.active_runs ?? 0),
+ queued_count: Number(source.queued_count || 0),
+ can_start: source.can_start ?? state === 'never_started',
+ can_pause: source.can_pause ?? state === 'running',
+ can_resume: source.can_resume ?? state === 'paused',
+ can_reset: source.can_reset ?? state !== 'never_started',
+ };
+};
+
const defaultWorkflowLock: WorkflowLock = {
locked: false,
workflow_id: null,
@@ -505,6 +593,7 @@ export const WebSocketProvider: React.FC<{ children: React.ReactNode }> = ({ chi
const [allVariables, setAllVariables] = useState<Record<string, Record<string, any>>>({});
const [workflowStatus, setWorkflowStatus] = useState<WorkflowStatus>(defaultWorkflowStatus);
const [deploymentStatus, setDeploymentStatus] = useState<DeploymentStatus>(defaultDeploymentStatus);
+ const [workflowControlStatuses, setWorkflowControlStatuses] = useState<Record<string, WorkflowControlStatus>>({});
const [workflowLock, setWorkflowLock] = useState<WorkflowLock>(defaultWorkflowLock);
// Per-workflow compaction stats: workflow_id -> session_id -> CompactionStats (n8n pattern)
const [allCompactionStats, setAllCompactionStats] = useState<Record<string, Record<string, CompactionStats>>>({});
@@ -1119,6 +1208,19 @@ export const WebSocketProvider: React.FC<{ children: React.ReactNode }> = ({ chi
break;
}
+ case 'workflow_control_status':
+ case 'workflow.control.status': {
+ const payload = message.data || message;
+ const workflowId = payload.workflow_id;
+ if (workflowId) {
+ setWorkflowControlStatuses((previous) => ({
+ ...previous,
+ [workflowId]: normalizeWorkflowControlStatus(payload, workflowId),
+ }));
+ }
+ break;
+ }
+
case 'deployment_status':
// Handle deployment status updates (event-driven, no iterations)
// Per-workflow scoping (n8n pattern): Only apply updates for current workflow
@@ -2255,6 +2357,76 @@ export const WebSocketProvider: React.FC<{ children: React.ReactNode }> = ({ chi
}
}, [sendRequest]);
+ const controlMutation = useCallback(async (
+ type: 'start_workflow' | 'pause_workflow' | 'resume_workflow' | 'reset_workflow',
+ workflowId: string,
+ data: Record<string, any>,
+ ): Promise<WorkflowControlStatus> => {
+ const response = await sendRequest<any>(type, {
+ workflow_id: workflowId,
+ idempotency_key: crypto.randomUUID(),
+ ...data,
+ });
+ const normalized = normalizeWorkflowControlStatus(response, workflowId);
+ setWorkflowControlStatuses((previous) => ({ ...previous, [workflowId]: normalized }));
+ return normalized;
+ }, [sendRequest]);
+
+ const graphEnvelope = (nodes: any[], edges: any[]) => ({
+ nodes: nodes.map((node) => ({ id: node.id, type: node.type || '', data: node.data || {} })),
+ edges: edges.map((edge) => ({
+ id: edge.id, source: edge.source, target: edge.target,
+ sourceHandle: edge.sourceHandle || undefined,
+ targetHandle: edge.targetHandle || undefined,
+ })),
+ });
+
+ const startWorkflowAsync = useCallback((workflowId: string, nodes: any[], edges: any[], sessionId?: string, expectedRevision = 0) =>
+ controlMutation('start_workflow', workflowId, {
+ ...graphEnvelope(nodes, edges), session_id: sessionId || 'default', expected_revision: expectedRevision,
+ }), [controlMutation]);
+
+ const pauseWorkflowAsync = useCallback((workflowId: string, expectedRevision: number) =>
+ controlMutation('pause_workflow', workflowId, { expected_revision: expectedRevision }), [controlMutation]);
+
+ const resumeWorkflowAsync = useCallback((workflowId: string, expectedRevision: number) =>
+ controlMutation('resume_workflow', workflowId, { expected_revision: expectedRevision }), [controlMutation]);
+
+ const resetWorkflowAsync = useCallback((workflowId: string, expectedRevision: number) =>
+ controlMutation('reset_workflow', workflowId, { expected_revision: expectedRevision }), [controlMutation]);
+
+ const getWorkflowControlStatusAsync = useCallback(async (workflowId: string) => {
+ try {
+ const response = await sendRequest<any>('get_workflow_control_status', { workflow_id: workflowId });
+ const normalized = normalizeWorkflowControlStatus(response, workflowId);
+ setWorkflowControlStatuses((previous) => ({ ...previous, [workflowId]: normalized }));
+ return normalized;
+ } catch (error) {
+ console.error('[WebSocket] Failed to get workflow control status:', error);
+ return emptyWorkflowControlStatus(workflowId);
+ }
+ }, [sendRequest]);
+
+ const getTeamTaskTraceAsync = useCallback(async (scope: Record<string, any>): Promise<TeamTaskTraceResponse> => {
+ const response = await sendRequest<any>('get_team_task_trace', scope);
+ const payload = response?.trace || response || {};
+ const execution = payload.execution || {};
+ return {
+ ...payload,
+ state: payload.status || payload.state || payload.trace_state || 'execution_not_registered',
+ workflow_id: execution.child_workflow_id || execution.runner_workflow_id || execution.workflow_id,
+ run_id: execution.child_run_id || execution.runner_run_id || execution.run_id,
+ events: Array.isArray(payload.events) ? payload.events.map((event: any) => ({
+ ...event,
+ event_id: event.event_id ?? event.id,
+ event_type: event.event_type || event.type || event.activity_type || event.workflow_type || 'event',
+ category: event.category || event.activity_type || event.workflow_type,
+ name: event.name || event.activity_type || event.workflow_type,
+ failure: typeof event.failure === 'string' ? event.failure : event.failure?.message,
+ })) : [],
+ };
+ }, [sendRequest]);
+
// Cancel any active ad-hoc execution(s) for a workflow. The backend
// resets every node currently glowing for this workflow_id and clears
// its active-run counter. Distinct from cancelDeployment, which only
@@ -2860,6 +3032,7 @@ export const WebSocketProvider: React.FC<{ children: React.ReactNode }> = ({ chi
variables,
workflowStatus,
deploymentStatus,
+ workflowControlStatuses,
workflowLock,
// Compaction stats (real-time via broadcasts)
@@ -2918,6 +3091,12 @@ export const WebSocketProvider: React.FC<{ children: React.ReactNode }> = ({ chi
getDeploymentStatus: getDeploymentStatusAsync,
cancelExecution: cancelExecutionAsync,
getWorkflowStatus: getWorkflowStatusAsync,
+ startWorkflow: startWorkflowAsync,
+ pauseWorkflow: pauseWorkflowAsync,
+ resumeWorkflow: resumeWorkflowAsync,
+ resetWorkflow: resetWorkflowAsync,
+ getWorkflowControlStatus: getWorkflowControlStatusAsync,
+ getTeamTaskTrace: getTeamTaskTraceAsync,
// AI Operations
executeAiNode: executeAiNodeAsync,
@@ -2963,7 +3142,7 @@ export const WebSocketProvider: React.FC<{ children: React.ReactNode }> = ({ chi
apiKeyStatuses,
consoleLogs, terminalLogs, chatMessages,
nodeStatuses, nodeParameters,
- variables, workflowStatus, deploymentStatus, workflowLock,
+ variables, workflowStatus, deploymentStatus, workflowControlStatuses, workflowLock,
compactionStats, updateCompactionStats,
getNodeStatus, getApiKeyStatus, getVariable,
requestStatus, clearNodeStatus,
@@ -2975,6 +3154,8 @@ export const WebSocketProvider: React.FC<{ children: React.ReactNode }> = ({ chi
cancelEventWaitAsync,
deployWorkflowAsync, cancelDeploymentAsync, getDeploymentStatusAsync,
cancelExecutionAsync, getWorkflowStatusAsync,
+ startWorkflowAsync, pauseWorkflowAsync, resumeWorkflowAsync, resetWorkflowAsync,
+ getWorkflowControlStatusAsync, getTeamTaskTraceAsync,
executeAiNodeAsync, getAiModelsAsync,
validateApiKeyAsync, getStoredApiKeyAsync, saveApiKeyAsync, deleteApiKeyAsync,
getAndroidDevicesAsync, executeAndroidActionAsync,
diff --git a/docs-internal/DESIGN.md b/docs-internal/DESIGN.md
index 878393d0..1fb6ecd0 100644
--- a/docs-internal/DESIGN.md
+++ b/docs-internal/DESIGN.md
@@ -5,6 +5,7 @@
The OpenCompany execution engine implements a robust workflow orchestration system combining industry-standard patterns from **Netflix Conductor**, **Prefect 3.0**, **Temporal**, and **Redis Streams**. This document details the architectural decisions, design patterns, and standards used.
Related docs:
+- [temporal-execution-engine-rfc.md](temporal-execution-engine-rfc.md) - canonical current Temporal control, trigger, graph, agent-team, and trace architecture
- [TEMPORAL_ARCHITECTURE.md](TEMPORAL_ARCHITECTURE.md) - distributed execution via Temporal activities
- [event_waiter_system.md](event_waiter_system.md) - push-based trigger waiters
- [native_llm_sdk.md](native_llm_sdk.md) - LLM provider layer
diff --git a/docs-internal/TEMPORAL_ARCHITECTURE.md b/docs-internal/TEMPORAL_ARCHITECTURE.md
index 2248d099..30f23b31 100644
--- a/docs-internal/TEMPORAL_ARCHITECTURE.md
+++ b/docs-internal/TEMPORAL_ARCHITECTURE.md
@@ -1,5 +1,11 @@
# Temporal Distributed Node Execution Architecture
+> **Canonical system design:** See
+> [Temporal Execution Engine RFC](temporal-execution-engine-rfc.md) for the
+> current control-generation, trigger-hub, agent-team, pause/reset, trace, and
+> security architecture. This document remains the detailed node dispatch and
+> activity/worker inventory.
+
## Overview
Each workflow node executes as a **Temporal activity** with its own isolated context, enabling horizontal scaling across distributed workers. The orchestrator dispatches in one of three ways depending on settings flags:
diff --git a/docs-internal/TEMPORAL_CLEANUP_AND_RESILIENCE_PLAN.md b/docs-internal/TEMPORAL_CLEANUP_AND_RESILIENCE_PLAN.md
index ac3c17c2..5101295c 100644
--- a/docs-internal/TEMPORAL_CLEANUP_AND_RESILIENCE_PLAN.md
+++ b/docs-internal/TEMPORAL_CLEANUP_AND_RESILIENCE_PLAN.md
@@ -1,5 +1,10 @@
# RFC: Temporal Cleanup, Queue Routing, Resilience & Performance (Waves 15-18)
+> **Historical implementation plan.** This document records the Waves 15-18
+> migration and retains names that must remain replay-compatible. The current
+> execution and control architecture is specified in
+> [Temporal Execution Engine RFC](temporal-execution-engine-rfc.md).
+
- **Status**: Approved, in implementation
- **Drafted**: 2026-07-03 (all code facts verified against the working tree at that date via a 5-agent parallel sweep)
- **Baseline**: `5b384b7` (v0.0.92)
diff --git a/docs-internal/email_service.md b/docs-internal/email_service.md
index 5924c3e1..041fda08 100644
--- a/docs-internal/email_service.md
+++ b/docs-internal/email_service.md
@@ -237,17 +237,27 @@ Group: `['email', 'trigger']`. No inputs. Single output `main` with the new emai
**Baseline detection:** On first execution the handler calls `poll_ids(creds, folder)` to capture the set of currently-existing envelope IDs. The poll loop then diffs against this baseline -- only newly appearing IDs trigger the workflow. This avoids firing on existing historical mail.
-**Event dispatch:** When a new message arrives, `EmailReceiveNode.execute_op`:
+**Standalone event dispatch:** When a new message arrives during an interactive
+Run, `EmailReceiveNode.execute_op`:
1. Fetches full detail via `fetch_detail`
2. Optionally flags it as read
-3. Calls `dispatch_email_received(email_data)` (in `_events.py`), which builds a `WorkflowEvent` (`type="com.opencompany.email.message.received"`) and routes it via a single `dispatch.emit` so Temporal-durable `TriggerListenerWorkflow` consumers fire
-4. Returns the first new email's result (single-shot in standalone mode; the `PollingTriggerNode` deployment loop keeps polling)
+3. Calls `dispatch_email_received(email_data)` (in `_events.py`), which builds a `WorkflowEvent` (`type="com.opencompany.email.message.received"`) and routes it through `dispatch.emit`
+4. Returns the first new email's result
## Deployment Mode (Continuous Polling)
-When a workflow containing `emailReceive` is **deployed** (not just run once), the trigger runs as a continuous polling loop owned by its `PollingTriggerNode` base class — not by a hand-written factory in the deployment manager.
+When a workflow containing `emailReceive` is deployed through workflow control,
+its generation's `WorkflowControlWorkflow` owns continuous polling. It invokes
+the activity generated by `PollingTriggerNode.as_poll_activity()`, maintains the
+deduplication baseline in durable workflow state, and starts `MachinaWorkflow`
+only for new messages. Pause gates new polls and graph starts; Resume continues
+the same controller execution.
-`EmailReceiveNode(PollingTriggerNode)` declares the polling hooks (`poll_ids`, `fetch_detail`); the base class's poll-coroutine factory self-registers via `services.deployment.poll_registry.register_poll_coroutine_factory`. The deployment manager's per-type factories (`_create_poll_coroutine` / `_create_gmail_poll_coroutine` / `_create_email_poll_coroutine`) were **removed in Wave 11.I, milestone L** — the dispatch now flows through `DeploymentManager._setup_event_trigger`, which looks up the registered factory.
+`EmailReceiveNode(PollingTriggerNode)` declares the polling hooks (`poll_ids`,
+`fetch_detail`); the plugin system registers the corresponding Temporal poll
+activity. The older deployment poll-coroutine registry and
+`PollingTriggerWorkflow` remain compatibility paths for uncontrolled deployments
+and replay of existing histories, not the authority for new controlled runs.
The poll loop:
1. Resolves credentials once via `EmailService.resolve_credentials`
@@ -257,7 +267,10 @@ The poll loop:
- For each new ID: fetches detail, optionally marks read, dispatches the event
- Handles `asyncio.CancelledError` cleanly on teardown
-All credential resolution and IMAP access still delegate to `EmailService` — nothing is duplicated in the deployment manager. (Post-2026-05-15, with the event framework enabled, deployed `emailReceive` triggers run as a Temporal-durable `PollingTriggerWorkflow` via the canary registration in `nodes/email/__init__.py`.)
+All credential resolution and IMAP access still delegate to `EmailService` —
+nothing is duplicated in the controller. See
+[Temporal Execution Engine RFC](temporal-execution-engine-rfc.md) for the
+control and trigger lifecycle.
## Credentials Storage
diff --git a/docs-internal/event_framework.md b/docs-internal/event_framework.md
index ffdd363b..d3f5afea 100644
--- a/docs-internal/event_framework.md
+++ b/docs-internal/event_framework.md
@@ -1,5 +1,11 @@
# Event Framework (Wave 12)
+> **Current architecture:** This document retains the Wave-12 rollout history.
+> Controlled deployments now consolidate push registration, event Signals, and
+> polling activities inside `WorkflowControlWorkflow`; see
+> [Temporal Execution Engine RFC](temporal-execution-engine-rfc.md). Listener
+> workflows described in the phase log are legacy compatibility contracts.
+
Temporal-native event-routing layer for OpenCompany. Implements RFC sections
6.3 (Temporal worker contract) + 6.4 (CloudEvents broadcast contract) from
[plugin_authoring_rfc.md](./ARCHIVE/plugin_authoring_rfc.md).
@@ -222,8 +228,14 @@ Locked by `TestCloudEventTypeMatchesSearchAttribute` in [`test_canary_registry.p
`event_waiter.dispatch` / `broadcaster.send_custom_event` calls inside canary-registered plugin `_events.py` files (chat / webhook / task / telegram / email) had zero consumers in canary-on mode — the deployment manager skips `setup_event_trigger` when `is_canary_trigger_type(...)` is True, so no legacy waiter ever registered. Removed.
-- `dispatch.emit(envelope, wire_routing_key=...)` is now the single delivery path. It Signals running `TriggerListenerWorkflow` consumers via Temporal Visibility AND broadcasts to FE on the same wire key.
-- `google/_events.py:dispatch_gmail_received` deleted (gmail polling uses `PollingTriggerWorkflow`, not `event_waiter`).
+- `dispatch.emit(envelope, wire_routing_key=...)` is the single delivery path. It signals legacy `EventType` consumers and running workflow controllers through one Temporal Visibility query, while broadcasting to the frontend on the same wire key. Each controller filters against its durable trigger registry.
+- For workflow-control generations, trigger definitions, push-event signals,
+ and polling activities live directly in the generation's
+ `WorkflowControlWorkflow`. There are no separate listener workflow runs;
+ only an actual graph execution is started as a child. Standalone
+ `TriggerListenerWorkflow` / `PollingTriggerWorkflow` starts remain only as a
+ compatibility path for legacy deployments without a controller.
+- `google/_events.py:dispatch_gmail_received` deleted. Controlled Gmail polling runs as an activity inside the controller; `PollingTriggerWorkflow` remains the legacy compatibility implementation.
- `whatsapp/_events.py` kept the legacy raw frame on `whatsapp_message_received` for received messages because the FE message-list handler reads `data.sender` (legacy shape) directly — drop blocked on FE migration (D4 follow-up). The duplicate typed-envelope sibling on the same wire key was dropped.
- `twitter/_events.py` keeps both paths — twitter is the only deferred canary plugin (needs PollingTriggerNode subclass refactor first).
@@ -231,11 +243,11 @@ Locked by `TestCloudEventTypeMatchesSearchAttribute` in [`test_canary_registry.p
Pre-fix the canary listener broadcast `waiting` once at deploy and stayed there forever — when an event fired, FE saw downstream nodes light up but no visual signal on the trigger node itself. The legacy `services/deployment/triggers.py` collector/processor did `waiting → idle (Graph executing...) → waiting` per event; canary skipped this.
-Fix: new `broadcast_trigger_status_activity` ([activities.py](../server/services/temporal/activities.py)). `TriggerListenerWorkflow._spawn_child_run` calls it before + after each child spawn:
+Fix: `broadcast_trigger_status_activity` ([activities.py](../server/services/temporal/activities.py)). The shared trigger-run helper used by the controller and legacy `TriggerListenerWorkflow` calls it before + after each child spawn:
- Before: `status="idle"` with `message="Graph executing..."` data
- After: `status="waiting"` with the next-event message
-`PollingTriggerWorkflow._spawn_child_run` does the same. Matches legacy UX.
+The controller's polling path and legacy `PollingTriggerWorkflow` use the same status lifecycle. Pause overrides the armed `waiting` projection with an explicit paused state and Resume rearms it.
### 4. Polling `seen` set OOM leak
diff --git a/docs-internal/event_waiter_system.md b/docs-internal/event_waiter_system.md
index 68d34ed3..06f79170 100644
--- a/docs-internal/event_waiter_system.md
+++ b/docs-internal/event_waiter_system.md
@@ -3,7 +3,7 @@
> **⚠️ Pre-Wave-11 — historical reference only.**
> Node authoring now happens on the backend: each node is a Python plugin under `server/nodes/<category>/<node>.py` that emits a `NodeSpec`. The frontend reads specs via [client/src/lib/nodeSpec.ts](../client/src/lib/nodeSpec.ts) + [adapters/nodeSpecToDescription.ts](../client/src/adapters/nodeSpecToDescription.ts). See [plugin_system.md](./plugin_system.md) and [server/nodes/README.md](../server/nodes/README.md) for the current model. The snippets below that reference `client/src/nodeDefinitions/*` are kept for historical context.
-Trigger nodes in OpenCompany suspend workflow execution until an external event arrives (WhatsApp message, webhook request, Telegram message, chat input, delegated task completion, etc.). The Event Waiter system is the in-memory (`asyncio.Future`) primitive that backs push-based triggers on the canvas-Run path. Deployed triggers ride the Temporal-durable canary path (`services/events/dispatch.py` → `TriggerListenerWorkflow` / `PollingTriggerWorkflow`) instead; the Redis-Streams backend that previously offered cross-restart waiter persistence was retired in Wave 15.3 because Temporal owns durability now.
+Trigger nodes in OpenCompany suspend workflow execution until an external event arrives (WhatsApp message, webhook request, Telegram message, chat input, delegated task completion, etc.). The Event Waiter system is the in-memory (`asyncio.Future`) primitive that backs push-based triggers on the canvas-Run path. In a controlled deployment, trigger definitions, push Signals, polling activities, pause state, and queued events live in `WorkflowControlWorkflow`; only a real graph invocation starts `MachinaWorkflow`. `TriggerListenerWorkflow` / `PollingTriggerWorkflow` are legacy replay/compatibility paths. The Redis-Streams backend that previously offered cross-restart waiter persistence was retired because Temporal owns deployed-trigger durability.
Source file: `server/services/event_waiter.py`
@@ -113,7 +113,7 @@ Single in-memory backend using `asyncio.Future` with a module-level `_waiters` d
`capture_main_loop()` (called during app startup in `main.py`) stores the main event loop so future thread-context callers can hop onto it via `asyncio.run_coroutine_threadsafe`.
-Durability note: waiter state does NOT survive a process restart — that is by design. Deployed triggers get restart durability from the Temporal canary path (`TriggerListenerWorkflow` / `PollingTriggerWorkflow` / Temporal Schedules); the event waiter only backs interactive canvas-Run waits, where a dead process means the user's canvas session is gone anyway. (The Redis-Streams backend that previously covered this was retired in Wave 15.3.)
+Durability note: waiter state does NOT survive a process restart — that is by design. Controlled deployed triggers get restart durability from `WorkflowControlWorkflow`; cron uses Temporal Schedules. The event waiter only backs interactive canvas-Run waits and legacy uncontrolled deployments, where a dead process means the canvas session is gone. (The Redis-Streams backend that previously covered this was retired in Wave 15.3.)
## Polling Triggers vs Event Triggers
@@ -127,10 +127,10 @@ Some triggers do not fit the push model because the upstream service has no webh
| `taskTrigger` | Event (push via delegation) | `event_waiter.py` |
| `telegramReceive` | Event (push via long-polling) | `event_waiter.py` + `TelegramService` |
| `twitterReceive` | **Polling** | `deployment/triggers.py` + `asyncio.Queue` |
-| `googleGmailReceive` | **Polling** | `PollingTriggerWorkflow` (Temporal canary) |
+| `googleGmailReceive` | **Polling** | `WorkflowControlWorkflow` polling activity (controlled); `PollingTriggerWorkflow` legacy compatibility |
| `cronScheduler` | Temporal Schedule | `services/temporal/schedules.py` |
-Polling triggers are routed in `DeploymentManager._create_poll_coroutine()` and use `TriggerManager.setup_polling_trigger()` with a custom poll function and an `asyncio.Queue`. See [workflow-schema.md](workflow-schema.md) for the full trigger list.
+Controlled polling triggers are registered with `WorkflowControlWorkflow`, which invokes the plugin-generated `poll.{node_type}.v{version}` activity and starts a graph only for deduplicated new events. Legacy uncontrolled polling still uses the deployment compatibility layer. See [temporal-execution-engine-rfc.md](temporal-execution-engine-rfc.md) for the current deployment architecture.
## Cancellation
diff --git a/docs-internal/frontend_architecture.md b/docs-internal/frontend_architecture.md
index d5fe75fa..26fedf6b 100644
--- a/docs-internal/frontend_architecture.md
+++ b/docs-internal/frontend_architecture.md
@@ -388,7 +388,14 @@ See [ui_migration_plan.md](./ui_migration_plan.md) Phase 6.
- All modifying operations go through WebSocket — REST is reserved for auth + webhooks.
- No polling. If a component wants fresh data, call `sendRequest` once (or use TanStack Query with `staleTime: Infinity` + manual `invalidate`).
- **`sendRequest` queues during disconnect with backpressure.** When the socket is not open, the request enqueues with an `AbortController`-backed per-request timeout (default 30s) and replays on reconnect inside `ws.onopen` before `setIsReady(true)`. Queue caps at 200 with FIFO eviction (rejects oldest with `backpressure: too many queued requests`). Intentional close (`event.code === 1000`) drops the queue; transient closes preserve it. Eliminates indefinite spinners during the 3-second reconnect window. Implementation: `pendingSendQueueRef` + `drainPendingSends` in `WebSocketContext.tsx`.
-- **Deployment state reconciles on every WS connect.** When a backend restarts (or the socket transiently drops) the in-memory `DeploymentManager._deployments` dict is wiped and NEVER re-fires a `deployment_status: 'stopped'` broadcast because it has nothing to broadcast about. Before the fix (de8df87), the FE `deploymentStatus.isRunning=true` carried forward and the toolbar Start button stayed on "Stop" forever. Now `broadcaster.connect()` calls `_send_deployment_snapshot(websocket)` right after `initial_status` — a single-target push (NOT fan-out) carrying a CloudEvents envelope (`type = workflow.deployment.snapshot`) with the current `running_workflow_ids`. FE `case 'deployment_snapshot'` in `WebSocketContext.tsx` iterates `useAppStore.workflowUIStates` and clears `isExecuting=true` on any workflow NOT in that list (empty list is meaningful — this is the load-bearing reset). Distinct from `workflow_lifecycle("deployment.started")` (state-transition edge event) — the snapshot is an idempotent state dump tied to client connect.
+- **Workflow-control state reconciles on every WS connect.** The toolbar derives
+ Start/Pause/Resume/Reset availability from `get_workflow_control_status`, whose
+ persisted generation record is reconciled with the Temporal controller rather
+ than inferred from process-local tasks. `workflow_control_status` broadcasts
+ update transitions in real time; reconnect performs an authoritative read.
+ The older `deployment_snapshot` and binary deployment status remain migration
+ adapters for legacy deployments and must not override a resolved controller
+ generation. See [Temporal Execution Engine RFC](temporal-execution-engine-rfc.md).
- **`currentWorkflowId` lives in `useAppStore` only.** Non-React listeners (WS handlers) read it via `useAppStore.getState().currentWorkflow?.id` -- the documented Zustand escape hatch (https://github.com/pmndrs/zustand#read-state-without-subscription). The previous `currentWorkflowIdRef` mirror inside WebSocketContext was a one-render-late copy that misrouted broadcasts during workflow switches. The push to `nodeStatusStore.setCurrentWorkflowId` is driven from a single `useEffect` in `Dashboard.tsx`.
## Ownership boundary: TanStack Query vs Zustand vs WebSocketContext
diff --git a/docs-internal/node-logic-flows/ai_tools/taskManager.md b/docs-internal/node-logic-flows/ai_tools/taskManager.md
index 3a79edb3..be661e5a 100644
--- a/docs-internal/node-logic-flows/ai_tools/taskManager.md
+++ b/docs-internal/node-logic-flows/ai_tools/taskManager.md
@@ -19,6 +19,7 @@ the persisted execution membership snapshot.
| `assign_task` | `title`, `mission`, connected `assignee_node_id` or exact delegate name | new |
| `list_tasks` | optional `status_filter`, `include_history` | any |
| `get_task` | `task_id` | any |
+| `inspect_task_trace` | `task_id`; optional attempt, detail, cursor, limit; grep-style query/filter/context fields | any registered attempt |
| `modify_task` | `task_id`, `expected_revision`, changed fields | blocked/queued |
| `cancel_task` | `task_id`, `expected_revision`, optional `reason` | blocked/queued/running |
| `retry_task` | `task_id`, `expected_revision` | failed/submitted/cancelled |
@@ -42,6 +43,24 @@ compact result/usage persistence, `taskTrigger`, and permit release. Legacy
execution passes the same precreated task ID into its background child bridge.
Retries therefore cannot duplicate tasks.
+Each claimed attempt persists the actual Temporal parent, detached runner, and
+child workflow/run identities. `inspect_task_trace` authorizes through the
+task's saved workflow/lead/execution scope and then reads that persisted
+identity; callers cannot supply an arbitrary Temporal workflow ID. It returns
+sanitized `summary`, `failures`, or `timeline` events in cursor pages (50 by
+default, 100 maximum), with explicit `execution_not_registered`,
+`temporal_unavailable`, and `retention_expired` states. Trace inspection is
+audited. Historical task results remain readable when Temporal history has
+expired.
+
+For large histories, `detail="search"` performs a bounded server-side scan of
+sanitized event metadata. `query` is required; `search_mode` is `literal`,
+`all_terms`, or `any_terms`; optional category filters cover activity, child,
+failure, signal, timer, and workflow events. Results mark exact matches and
+include zero to five neighboring events. Each call scans at most 500 events and
+returns an opaque `next_cursor`, allowing the lead to progressively investigate
+a trace without placing the complete history in model context.
+
The human panel includes durable history by default, so workflow restarts do
not clear the visible list. Specific execution selections remain scoped for
safe mutations. Created/started/completed timestamps are normalized as UTC and
diff --git a/docs-internal/node-logic-flows/email/emailReceive.md b/docs-internal/node-logic-flows/email/emailReceive.md
index 63873bb0..5b163fad 100644
--- a/docs-internal/node-logic-flows/email/emailReceive.md
+++ b/docs-internal/node-logic-flows/email/emailReceive.md
@@ -3,7 +3,7 @@
| Field | Value |
|------|-------|
| **Category** | email / trigger |
-| **Backend handler** | [`server/nodes/email/email_receive/__init__.py`](../../../server/nodes/email/email_receive/__init__.py) — Run-button path is the `execute()` override; deployment-mode polling uses the `PollingTriggerNode` hooks (`setup_service` / `fetch_ids` / `fetch_detail` / `post_emit`) draining via the canary `TriggerListenerWorkflow` path |
+| **Backend handler** | [`server/nodes/email/email_receive/__init__.py`](../../../server/nodes/email/email_receive/__init__.py) — Run-button path is `execute()`; controlled deployment invokes the activity generated from its `PollingTriggerNode` hooks inside `WorkflowControlWorkflow` |
| **Tests** | [`server/tests/nodes/test_email.py`](../../../server/tests/nodes/test_email.py) |
| **Skill (if any)** | none shipped |
| **Dual-purpose tool** | no (trigger only) |
@@ -16,6 +16,11 @@ replayed), then subsequent polls diff message IDs to detect arrivals.
Mirrors the `googleGmailReceive` pattern but works against any IMAP provider via
Himalaya.
+In a controlled deployment the generation controller owns the poll baseline,
+pause gate, and child graph starts. It does not create a separate trigger
+listener workflow. Legacy listener/polling workflows remain registered for
+compatibility with older deployments and histories.
+
## Inputs (handles)
None. Triggers have no inputs.
@@ -103,15 +108,15 @@ flowchart TD
- **Database writes**: none directly.
- **Broadcasts**: one `update_node_status(..., "waiting", ...)` when polling
starts, with `workflow_id` from context.
-- **Event bus (canary path)**: `dispatch_email_received(email_data)` ->
+- **Event bus (standalone/compatibility path)**: `dispatch_email_received(email_data)` ->
`services.events.dispatch.emit` with a CloudEvents `WorkflowEvent`
(`type = "com.opencompany.email.message.received"`, `subject = message_id`,
outer wire-routing key `email_received`). `emailReceive` is canary-registered
via `register_canary_trigger_type("emailReceive", "com.opencompany.email.message.received")`
in [`nodes/email/__init__.py`](../../../server/nodes/email/__init__.py), so the
- deployment manager skips the legacy `setup_event_trigger` and a Temporal-durable
- `TriggerListenerWorkflow` consumes the event; the same call also broadcasts the
- envelope to the FE on the `email_received` wire key. The legacy
+ same call also broadcasts the envelope to the FE on the `email_received` wire
+ key. Controlled deployment polling is performed directly by
+ `WorkflowControlWorkflow`, so it does not require a listener workflow. The legacy
`event_waiter.dispatch` path has zero consumers in canary mode and is no longer
called (`_events.py`).
- **External API calls**: none direct - IMAP traffic flows through Himalaya.
diff --git a/docs-internal/node-logic-flows/google_workspace/googleGmailReceive.md b/docs-internal/node-logic-flows/google_workspace/googleGmailReceive.md
index 7b40e72e..3439830f 100644
--- a/docs-internal/node-logic-flows/google_workspace/googleGmailReceive.md
+++ b/docs-internal/node-logic-flows/google_workspace/googleGmailReceive.md
@@ -3,7 +3,7 @@
| Field | Value |
|------|-------|
| **Category** | google_workspace / trigger (polling) |
-| **Backend handler** | [`server/nodes/google/gmail_receive/__init__.py`](../../../server/nodes/google/gmail_receive/__init__.py) (`GmailReceiveNode`, a `PollingTriggerNode`; inline-Run path uses the `execute()` override, deployment path uses the `setup_service` / `fetch_ids` / `fetch_detail` / `post_emit` hooks driven by `PollingTriggerWorkflow`) |
+| **Backend handler** | [`server/nodes/google/gmail_receive/__init__.py`](../../../server/nodes/google/gmail_receive/__init__.py) (`GmailReceiveNode`, a `PollingTriggerNode`; inline-Run uses `execute()`, controlled deployment invokes its generated poll activity from `WorkflowControlWorkflow`) |
| **Tests** | [`server/tests/nodes/test_google_workspace.py`](../../../server/tests/nodes/test_google_workspace.py) |
| **Skill (if any)** | none |
| **Dual-purpose tool** | no |
@@ -17,13 +17,13 @@ has no official push without extra Pub/Sub setup, so this node polls
baseline, fetches full details for new messages, and optionally marks them
as read.
-`event_type = "gmail_email_received"`. In deployment (canary) mode the
-`PollingTriggerWorkflow` owns event fan-out directly: it calls the per-cycle
-activity built from `PollingTriggerNode.as_poll_activity()` and spawns the
-child `MachinaWorkflow` per new email — there is no `event_waiter.dispatch`
-step. The `_events.py` `gmail_message_received` CloudEvents factory is kept for
-parity / future ad-hoc emits; the legacy `dispatch_gmail_received` shim was
-removed in Wave 13.
+`event_type = "gmail_email_received"`. In a controlled deployment,
+`WorkflowControlWorkflow` invokes the per-cycle activity built by
+`PollingTriggerNode.as_poll_activity()`, persists the seen-ID baseline in
+workflow state, and starts a child `MachinaWorkflow` per deduplicated new
+email. Pause prevents new polls/runs and Resume continues the same controller.
+There is no `event_waiter.dispatch` step. `PollingTriggerWorkflow` stays
+registered only for legacy histories and uncontrolled deployments.
## Inputs (handles)
diff --git a/docs-internal/node-logic-flows/telegram_social/telegramReceive.md b/docs-internal/node-logic-flows/telegram_social/telegramReceive.md
index 291d6a5b..abcf17f6 100644
--- a/docs-internal/node-logic-flows/telegram_social/telegramReceive.md
+++ b/docs-internal/node-logic-flows/telegram_social/telegramReceive.md
@@ -16,9 +16,10 @@ first refuses to register if the bot is not connected, then defers to
`TriggerNode.execute`. The Telegram-specific `build_filter` pre-compiles a
`matches()` closure (via the legacy `build_telegram_filter`) from the node
parameters. Events are produced by the `TelegramService` long-polling loop
-inside `_on_message_received`. In deployment mode this trigger runs on the
-canary path (`TriggerListenerWorkflow` per deployment/trigger, fed via the
-event framework); the registered filter narrows the stream.
+inside `_on_message_received`. In controlled deployment mode, the trigger
+definition and queued CloudEvents live in `WorkflowControlWorkflow`; the
+controller filters/deduplicates the stream and starts only actual graph runs.
+`TriggerListenerWorkflow` remains a legacy compatibility path.
## Inputs (handles)
diff --git a/docs-internal/node-logic-flows/twitter/twitterReceive.md b/docs-internal/node-logic-flows/twitter/twitterReceive.md
index 8edd30ec..d024c338 100644
--- a/docs-internal/node-logic-flows/twitter/twitterReceive.md
+++ b/docs-internal/node-logic-flows/twitter/twitterReceive.md
@@ -20,10 +20,12 @@ matching event.
**Canary status**: `twitterReceive` is the lone **deferred** trigger plugin —
it is NOT in the canary registry (`register_canary_trigger_type` is not
-called). The Temporal-durable `TriggerListenerWorkflow` / `PollingTriggerWorkflow`
-path requires the node to subclass `PollingTriggerNode` and declare the four
-hooks (same shape as `GmailReceiveNode`); until that refactor lands it runs on
-the legacy `event_waiter` collector/processor path. `_events.py` carries the
+called). Adoption of the controlled Temporal path requires the node to subclass
+`PollingTriggerNode` and declare the four hooks (the `GmailReceiveNode` shape),
+so `WorkflowControlWorkflow` can own its polling activity. Until that refactor
+lands it runs on the legacy `event_waiter` collector/processor path.
+`TriggerListenerWorkflow` / `PollingTriggerWorkflow` are compatibility
+implementations, not the target architecture. `_events.py` carries the
typed `WorkflowEvent` factory + dual-path dispatcher (`dispatch_twitter_event_received`)
ready for canary opt-in.
diff --git a/docs-internal/node-logic-flows/whatsapp/whatsappReceive.md b/docs-internal/node-logic-flows/whatsapp/whatsappReceive.md
index ff22565a..6664b9e7 100644
--- a/docs-internal/node-logic-flows/whatsapp/whatsappReceive.md
+++ b/docs-internal/node-logic-flows/whatsapp/whatsappReceive.md
@@ -3,7 +3,7 @@
| Field | Value |
|------|-------|
| **Category** | whatsapp / trigger |
-| **Backend handler** | [`server/nodes/whatsapp/whatsapp_receive.py`](../../../server/nodes/whatsapp/whatsapp_receive.py) (`WhatsAppReceiveNode`, a `TriggerNode`); the filter closure is built by `build_filter()` -> [`server/nodes/whatsapp/_filters.py::build_filter`](../../../server/nodes/whatsapp/_filters.py) (self-registered into `event_waiter.FILTER_BUILDERS` as `whatsappReceive`). Canary-registered with CloudEvents type `com.opencompany.whatsapp.message.received`; deployment fires via `TriggerListenerWorkflow`. |
+| **Backend handler** | [`server/nodes/whatsapp/whatsapp_receive.py`](../../../server/nodes/whatsapp/whatsapp_receive.py) (`WhatsAppReceiveNode`, a `TriggerNode`); filtering is built by [`server/nodes/whatsapp/_filters.py`](../../../server/nodes/whatsapp/_filters.py). CloudEvents type: `com.opencompany.whatsapp.message.received`; controlled deployment routes through `WorkflowControlWorkflow`. |
| **Tests** | [`server/tests/nodes/test_whatsapp.py`](../../../server/tests/nodes/test_whatsapp.py) |
| **Skill (if any)** | n/a |
| **Dual-purpose tool** | no - pure trigger node |
@@ -11,11 +11,12 @@
## Purpose
Event-driven trigger that waits for incoming WhatsApp messages and starts a
-workflow run when one matches. `whatsappReceive` is canary-registered
-(CloudEvents type `com.opencompany.whatsapp.message.received`): on deployment the
-manager spawns a `TriggerListenerWorkflow` per (deployment, trigger) pair which
-receives the CloudEvents envelope via Temporal Signal and spawns a child
-`MachinaWorkflow` per matching event. Single-node "Run" (non-deployed test)
+workflow run when one matches. `whatsappReceive` is registered with CloudEvents
+type `com.opencompany.whatsapp.message.received`. In a controlled deployment,
+the manager signals its trigger definition into `WorkflowControlWorkflow`;
+`dispatch.emit` signals the controller, which filters/deduplicates the envelope
+and starts a child `MachinaWorkflow` only for a real match. There is no separate
+listener workflow run. Single-node "Run" (non-deployed test)
still uses the `TriggerNode.execute` event-waiter path: register a waiter with
the filter closure, broadcast `waiting`, then await a
`whatsapp_message_received` event. In both paths `build_filter` decides which
diff --git a/docs-internal/temporal-execution-engine-rfc.md b/docs-internal/temporal-execution-engine-rfc.md
new file mode 100644
index 00000000..740088ab
--- /dev/null
+++ b/docs-internal/temporal-execution-engine-rfc.md
@@ -0,0 +1,810 @@
+# RFC: Temporal Execution Engine Architecture
+
+| Field | Value |
+|---|---|
+| Status | Implemented; canonical current-state design |
+| Date | 2026-07-20 |
+| Owners | Workflow Platform, Agent Platform, Editor |
+| Scope | Durable workflow control, trigger admission, graph execution, AI agents, team delegation, observability |
+| Supersedes | The current-state portions of `TEMPORAL_ARCHITECTURE.md`; legacy execution remains documented there |
+
+## 1. Executive summary
+
+OpenCompany uses Temporal as its durable production execution engine. A saved
+canvas workflow is a stable application identity. Each deployment has a
+revision-guarded **control generation**, represented by a long-lived
+`WorkflowControlWorkflow`. Each actual graph invocation is a
+`MachinaWorkflow`. AI-agent nodes normally execute as `AgentWorkflow` child
+workflows, and asynchronous teammate assignments execute through
+`DelegatedTaskWorkflow` runners.
+
+The application database and Temporal have deliberately different authority:
+
+- The application database owns workflow authorization, graph snapshots,
+ control generations, teams, tasks, immutable attempts, and UI indexes.
+- Temporal owns durable command/event history, timers, retries, workflow and
+ activity state, and actual execution identity.
+- WebSocket broadcasts are a realtime projection. They are not authoritative.
+- Process-local registries are caches and legacy compatibility surfaces. They
+ must never be the only durable record of controlled execution.
+
+Controlled push and polling triggers live inside the control workflow rather
+than separate listener workflows. Only a real graph invocation appears as a
+child workflow. Temporal cron schedules remain server-side Schedule resources;
+their firings execute the plugin-owned `CronTriggerWorkflow` compatibility
+path.
+
+## 2. Goals and non-goals
+
+### 2.1 Goals
+
+1. Survive API-server and worker restarts without losing durable execution.
+2. Execute independent graph branches and agent tool calls concurrently.
+3. Isolate node, agent, teammate, memory, and tool capabilities.
+4. Make Start, Pause, Resume, and Reset explicit, revision-safe operations.
+5. Persist delegation before child startup and cap all descendants per root.
+6. Preserve immutable task-attempt history and link it to actual Temporal runs.
+7. Expose sanitized, bounded, searchable traces without exposing payloads or
+ trusting caller-supplied Temporal IDs.
+8. Keep plugin-owned activity schemas, retry behavior, and task queues
+ extensible.
+
+### 2.2 Non-goals
+
+- Native Temporal Workflow Pause. The engine uses cooperative Signals and
+ deterministic wait conditions.
+- Temporal's history-rewind Reset. Product Reset terminates and archives the
+ current application generation.
+- Shared parent/child LLM transcripts. Delegated agents start with isolated
+ conversations and return compact results.
+- Raw Temporal payload or prompt access through Task Manager.
+- Exactly-once side effects outside Temporal. Activities must be idempotent or
+ protected by database idempotency records.
+
+## 3. System context
+
+```mermaid
+flowchart LR
+ User[Workflow author / operator]
+ Editor[OpenCompany Editor<br/>Canvas, toolbar, Task Manager, Team Monitor]
+ API[OpenCompany API<br/>Validation, authorization, control and projections]
+ DB[(Application DB<br/>Graphs, generations, teams, tasks, attempts)]
+ Temporal[Temporal Cluster<br/>Histories, timers, retries, schedules]
+ Workers[Temporal Workers<br/>Framework and plugin workers]
+ LLM[LLM providers]
+ Services[Connected services<br/>Browser, Android, HTTP, SaaS, code]
+ OTel[OpenTelemetry backend]
+
+ User --> Editor
+ Editor <-->|Authenticated WebSocket| API
+ API <-->|Authoritative application state| DB
+ API <-->|Start, Signal, Query, history| Temporal
+ Temporal <-->|Workflow/activity tasks| Workers
+ Workers <-->|Idempotent lifecycle writes| DB
+ Workers --> LLM
+ Workers --> Services
+ API --> OTel
+ Workers --> OTel
+```
+
+## 4. Logical architecture
+
+```mermaid
+flowchart TB
+ subgraph UI[Editor]
+ Toolbar[Start / Pause / Resume / Reset]
+ Canvas[Canvas and node status]
+ TM[Task Manager and trace search]
+ Monitor[Team Monitor]
+ end
+
+ subgraph API[Application control plane]
+ WS[WebSocket handlers]
+ WCS[WorkflowControlService]
+ Deploy[DeploymentManager compatibility facade]
+ Team[AgentTeamService]
+ Trace[TeamTaskTraceService]
+ Events[CloudEvent dispatch]
+ end
+
+ subgraph DB[Application database]
+ Controls[(workflow_control_executions)]
+ Teams[(agent_teams)]
+ Tasks[(team_tasks)]
+ Attempts[(team_task_attempts)]
+ Messages[(agent_messages)]
+ Permits[(subagent_concurrency_permits)]
+ end
+
+ subgraph Temporal[Temporal namespace]
+ Controller[WorkflowControlWorkflow]
+ Root[MachinaWorkflow]
+ Agent[AgentWorkflow]
+ Runner[DelegatedTaskWorkflow]
+ Cron[Temporal Schedule / CronTriggerWorkflow]
+ Activities[Framework and plugin activities]
+ end
+
+ Toolbar --> WS --> WCS --> Controls
+ WCS --> Controller
+ Deploy --> Controller
+ Events --> Controller
+ Controller --> Root
+ Cron --> Root
+ Root --> Agent
+ Agent --> Runner
+ Runner --> Agent
+ Root --> Activities
+ Agent --> Activities
+ Runner --> Activities
+ Team --> Teams
+ Team --> Tasks
+ Team --> Attempts
+ Team --> Messages
+ Team --> Permits
+ Trace --> Tasks
+ Trace --> Attempts
+ Trace --> Temporal
+ TM --> WS
+ Monitor --> WS
+ WS --> Canvas
+```
+
+## 5. Identity model
+
+| Identity | Lifetime | Authority | Purpose |
+|---|---|---|---|
+| `workflow_id` | Saved workflow lifetime | Application DB | Stable product identity and authorization scope |
+| `generation` | One Start-to-Reset interval | Application DB | Separates clean deployment epochs |
+| `execution_id` / `root_execution_id` | Application execution tree | Application DB | Teams, tasks, permits, and descendant correlation |
+| Temporal Workflow ID | Temporal execution chain | Temporal | Stable address for Signal/Query/command routing |
+| Temporal Run ID | One run in a workflow chain | Temporal | Exact history version and attempt link |
+| `team_id` | Lead node in one execution | Application DB | Team membership and task authorization |
+| `task_id` | Durable assignment | Application DB | Stable task identity across retries |
+| `attempt_number` | One claim/run | Application DB | Immutable assignee, result, and Temporal links |
+| `trace_id` | Cross-component correlation | Application DB / OTel | Opaque correlation only; never authorization |
+
+Identity rules:
+
+- Browser and model callers never choose a Temporal workflow/run ID for trace
+ access. The server resolves it from an authorized task attempt.
+- Workflow IDs and Run IDs are distinct. Resume signals the existing workflow;
+ it does not manufacture a replacement Run ID.
+- A Reset archives the current generation and returns control state `ready`.
+ The next Start creates generation `N + 1`.
+
+## 6. Deployment control plane
+
+### 6.1 Persisted control record
+
+`WorkflowControlExecution` stores the workflow, generation, execution/root
+identities, controller workflow/run IDs, graph snapshot and hash, session,
+status, revision, idempotency key, timestamps, and terminal reason. Unique
+constraints protect `(workflow_id, generation)` and
+`(workflow_id, idempotency_key)`.
+
+### 6.2 State machine
+
+```mermaid
+stateDiagram-v2
+ [*] --> never_started
+ never_started --> starting: Start
+ ready --> starting: Start next generation
+ starting --> running: controller + deployment established
+ starting --> failed: validation/start/deployment failure
+ running --> pausing: Pause(expected revision)
+ pausing --> paused: controller, triggers, schedules gated
+ paused --> resuming: Resume(expected revision)
+ resuming --> running: gates released
+ running --> resetting: Reset
+ paused --> resetting: Reset
+ failed --> resetting: Reset
+ resetting --> ready: tree terminated and generation archived
+```
+
+Public state `ready` is persisted internally as terminal status `reset`. This
+distinguishes a workflow that has never run from one with archived generations,
+while presenting Start in both cases.
+
+### 6.3 Control command sequence
+
+```mermaid
+sequenceDiagram
+ autonumber
+ actor User
+ participant UI as Editor
+ participant API as Control handler
+ participant DB as Application DB
+ participant T as Temporal
+ participant C as WorkflowControlWorkflow
+ participant DM as DeploymentManager
+
+ User->>UI: Start
+ UI->>API: start_workflow(graph, revision, idempotency_key)
+ API->>DB: create generation + snapshot
+ API->>T: start controller
+ T->>C: run(generation)
+ API->>DM: deploy snapshot
+ DM->>C: register_trigger(spec) signals
+ API->>DB: starting -> running (CAS)
+ API-->>UI: running + new revision
+
+ User->>UI: Pause
+ UI->>API: pause_workflow(expected_revision)
+ API->>DB: running -> pausing (CAS)
+ API->>C: pause signal
+ API->>T: pause cron schedules
+ API->>DM: gate legacy admission + disarm trigger UI
+ API->>DB: pausing -> paused
+ API-->>UI: paused
+
+ User->>UI: Resume
+ UI->>API: resume_workflow(expected_revision)
+ API->>DB: paused -> resuming (CAS)
+ API->>C: resume signal
+ API->>T: unpause cron schedules
+ API->>DM: drain buffered legacy events FIFO + rearm UI
+ API->>DB: resuming -> running
+ API-->>UI: running
+```
+
+### 6.4 Cooperative pause semantics
+
+Pause is an admission barrier, not suspension of operating-system threads:
+
+- Push events are accepted as durable controller Signals and remain queued.
+- The controller does not create new graph child workflows while paused.
+- Poll cycles do not start new graph runs; a poll activity already in flight may
+ finish and its events wait for Resume.
+- Temporal cron schedules are explicitly paused.
+- `MachinaWorkflow`, `AgentWorkflow`, and `DelegatedTaskWorkflow` gate new
+ scheduling at deterministic wait conditions.
+- Activities already started are allowed to finish. Their results remain in
+ history and are consumed after the gate opens.
+- Trigger nodes broadcast an idle/paused projection, stop their listening
+ animation, and rearm on Resume.
+
+### 6.5 Reset semantics
+
+Reset transitions the generation to `resetting`, rejects new admission,
+signals the controller to close, terminates visible descendants carrying the
+application workflow search attribute, cancels local compatibility resources,
+and archives the record as `reset`. It does **not** start the next generation.
+
+Task history and historical Temporal histories are preserved. Existing
+historical rows continue to appear in Temporal Web until namespace retention or
+an explicit administrative deletion policy removes them.
+
+## 7. Trigger architecture
+
+### 7.1 Controlled push and polling triggers
+
+```mermaid
+sequenceDiagram
+ autonumber
+ participant Producer as Plugin producer
+ participant Dispatch as events.dispatch
+ participant Visibility as Temporal Visibility
+ participant Control as WorkflowControlWorkflow
+ participant Activity as Poll/plugin activity
+ participant Run as MachinaWorkflow
+
+ Producer->>Dispatch: emit(CloudEvent)
+ par realtime projection
+ Dispatch-->>Dispatch: broadcast WebSocket event
+ and durable routing
+ Dispatch->>Visibility: list legacy EventType consumers OR controllers
+ Dispatch->>Control: on_event(envelope)
+ end
+ Control->>Control: filter by registered types + deduplicate
+ alt running
+ Control->>Run: start actual graph child
+ else paused
+ Control->>Control: retain FIFO event
+ end
+
+ loop polling trigger while running
+ Control->>Activity: poll.{node_type}.v{version}
+ Activity-->>Control: seen_ids + events
+ Control->>Control: deduplicate
+ Control->>Run: start actual graph child
+ end
+```
+
+There are no `TriggerListenerWorkflow` or `PollingTriggerWorkflow` executions
+for a controlled generation. Those workflow types stay registered only to
+replay and serve legacy uncontrolled deployments.
+
+### 7.2 Trigger graph materialization
+
+Before a trigger starts `MachinaWorkflow`, the engine builds a downstream graph:
+
+1. Mark the firing trigger `_pre_executed` with the event payload.
+2. Mark sibling triggers as non-firing so they do not block dependency
+ resolution.
+3. Retain downstream executable nodes and their configuration attachments.
+4. Optionally load the latest persisted graph through an activity; the saved
+ deployment snapshot is the deterministic fallback.
+5. Start a child ID derived from workflow slug, trigger label, and event ID.
+
+### 7.3 Cron exception
+
+Cron remains a Temporal Schedule because server-side calendar evaluation,
+timezone handling, catch-up windows, and overlap policy belong to Temporal's
+Schedule service. Pause/unpause operates on the Schedule resource. A firing
+uses the plugin-owned `CronTriggerWorkflow`, which then starts the graph run.
+This is a scheduled execution, not a continuously running listener.
+
+## 8. Graph execution (`MachinaWorkflow`)
+
+```mermaid
+flowchart TD
+ Input[Graph payload] --> Normalize[Normalize nodes, edges, identities]
+ Normalize --> Filter[Remove configuration-only nodes from executable DAG]
+ Filter --> Deps[Build dependencies and reverse edges]
+ Deps --> Seed[Seed pre-executed trigger outputs]
+ Seed --> Ready[Find all ready nodes]
+ Ready --> Gate{Paused?}
+ Gate -- yes --> Wait[workflow.wait_condition]
+ Wait --> Ready
+ Gate -- no --> Route{Node class}
+ Route -- agent --> AgentWF[Start AgentWorkflow child]
+ Route -- plugin per-type --> PluginAct[node.type.version activity on declared queue]
+ Route -- legacy --> LegacyAct[execute_node_activity]
+ AgentWF --> Complete[Record result]
+ PluginAct --> Complete
+ LegacyAct --> Complete
+ Complete --> Fanout[Make downstream nodes eligible]
+ Fanout --> Done{All terminal?}
+ Done -- no --> Ready
+ Done -- yes --> Result[outputs + execution_trace + error]
+```
+
+The scheduler follows a continuous ready-set/FIRST_COMPLETED pattern. It does
+not wait for an entire topological layer when one branch finishes early.
+Configuration handles (`input-tools`, `input-memory`, `input-model`,
+`input-skill`, `input-task`, and `input-teammates`) bind capabilities but do not
+normally create executable dependencies. `taskTrigger -> input-task` is the
+explicit runtime-data exception.
+
+### 8.1 Dispatch matrix
+
+| Node | Primary dispatch | Compatibility dispatch |
+|---|---|---|
+| Ordinary plugin node | `node.{type}.v{version}` activity | `execute_node_activity` |
+| Supported AI/team agent | `AgentWorkflow` child | Per-type/legacy agent activity when feature-disabled |
+| RLM / CLI session agents | Per-type activity | Legacy dispatcher |
+| Controlled push/poll trigger | Controller Signal/activity | Listener workflows for legacy deployments |
+| Cron trigger | Temporal Schedule + plugin workflow | No in-memory APScheduler production path |
+
+### 8.2 Retries and timeouts
+
+- Framework activities use shared retry policies with user-correctable error
+ classes marked non-retryable.
+- Plugin activities may declare their own task queue, timeout, heartbeat, and
+ retry settings.
+- LLM steps use a longer LLM-specific retry/timeout policy.
+- Tool failures are returned to the agent as structured tool errors after
+ activity retries; sibling calls are not cancelled.
+- Workflow code contains orchestration only. Network, database, LLM, filesystem,
+ broadcast, and clock-dependent side effects belong in activities.
+
+## 9. Agent execution
+
+```mermaid
+sequenceDiagram
+ autonumber
+ participant Root as MachinaWorkflow
+ participant Agent as AgentWorkflow
+ participant LLM as agent.execute_llm_step.v1
+ participant Tools as Plugin activities
+ participant Memory as persist/compact activities
+
+ Root->>Agent: child(payload, isolated conversation)
+ loop bounded agent iterations
+ Agent->>Agent: pause gate
+ Agent->>LLM: messages + visible tool schemas
+ LLM-->>Agent: final answer or ordered tool calls
+ alt tool calls
+ par independent calls
+ Agent->>Tools: activity/child call 1
+ and
+ Agent->>Tools: activity/child call N
+ end
+ Tools-->>Agent: success/error results
+ Agent->>Agent: append results in original call order
+ Agent->>Memory: persist turn
+ opt token threshold exceeded
+ Agent->>Memory: compact memory
+ end
+ else final
+ Agent-->>Root: compact result + usage
+ end
+ end
+```
+
+Every agent receives only the tools, skills, memory, model configuration, and
+teammates connected to that node in the server-side graph. The canvas is used
+for capability resolution but unrelated canvas content is not serialized into
+the model prompt.
+
+Dynamic tool refresh uses `agent.refresh_tools.v1`. Provider-visible names must
+be unique. Specialized delegate names are type-derived; repeatable `aiAgent`
+delegates use label-derived, collision-safe names.
+
+## 10. Durable team delegation
+
+### 10.1 Runtime topology
+
+```mermaid
+flowchart LR
+ Lead[Lead AgentWorkflow] -->|task_manager.assign_task| Persist[Persist TeamTask]
+ Persist --> Runner[DelegatedTaskWorkflow]
+ Runner --> Permit[(Root-wide permit)]
+ Permit --> Child[Child AgentWorkflow]
+ Child --> Result[Submit compact result]
+ Result --> Attempt[(Immutable TeamTaskAttempt)]
+ Result --> Trigger[taskTrigger CloudEvent]
+ Trigger --> Control[WorkflowControlWorkflow]
+ Control --> Review[New lead review invocation]
+ Review --> Decision{Lead decision}
+ Decision -->|accept| Accepted[accepted]
+ Decision -->|retry/reassign| Runner
+ Decision -->|cancel| Cancelled[cancelled]
+```
+
+### 10.2 Delegation invariants
+
+- Only canonical `input-teammates` connections authorize assignment.
+- `aiAgent` is repeatable; other specialized teammate types are unique.
+- Maximum delegation depth is two child layers.
+- The default root-wide active-descendant cap is three. The root is excluded;
+ children and grandchildren are included.
+- Persistence succeeds before a child starts.
+- `DelegatedTaskWorkflow` waits for a durable permit and heartbeats while
+ queued.
+- Graceful success, failure, and cancellation paths release their permit
+ idempotently. Immediate Temporal termination requires the reset reconciler
+ described in Known Limitations.
+- A worker result moves a task to `submitted`, not `accepted`.
+- The lead or authorized human must review submitted work.
+- Retry and reassignment create new attempts; they never overwrite history.
+- The assigning lead returns after durable queueing and does not synchronously
+ wait for the teammate.
+
+### 10.3 Task state machine
+
+```mermaid
+stateDiagram-v2
+ [*] --> blocked: unresolved dependencies
+ [*] --> queued: eligible assignment
+ blocked --> queued: dependencies accepted
+ queued --> running: task claimed + permit acquired
+ running --> submitted: child completed
+ running --> queued: retryable failure
+ running --> failed: terminal failure
+ running --> cancelled: cancellation/reset
+ submitted --> accepted: lead/human review passes
+ submitted --> queued: retry or reassignment
+ submitted --> cancelled: waived/cancelled
+ failed --> queued: retry or reassignment
+ cancelled --> queued: revised retry/reassignment
+```
+
+## 11. Persistence model
+
+```mermaid
+erDiagram
+ WORKFLOW_CONTROL_EXECUTION {
+ string id PK
+ string workflow_id
+ int generation
+ string execution_id
+ string root_execution_id
+ string controller_workflow_id
+ string controller_run_id
+ string status
+ int revision
+ json graph_snapshot
+ string graph_hash
+ string idempotency_key
+ }
+ AGENT_TEAM {
+ string id PK
+ string workflow_id
+ string execution_id
+ string root_execution_id
+ string team_lead_node_id
+ string status
+ }
+ TEAM_TASK {
+ string id PK
+ string team_id FK
+ string execution_id
+ string root_execution_id
+ string assigned_to
+ string status
+ int revision
+ int current_attempt
+ json result
+ json usage
+ }
+ TEAM_TASK_ATTEMPT {
+ string id PK
+ string task_id FK
+ int attempt_number
+ string runner_workflow_id
+ string runner_run_id
+ string child_workflow_id
+ string child_run_id
+ string parent_workflow_id
+ string parent_run_id
+ string status
+ json result
+ json usage
+ }
+ AGENT_MESSAGE {
+ int id PK
+ string team_id FK
+ string event_id
+ string message_type
+ json extra_data
+ }
+ SUBAGENT_CONCURRENCY_PERMIT {
+ string permit_id PK
+ string root_execution_id
+ string status
+ }
+
+ WORKFLOW_CONTROL_EXECUTION ||--o{ AGENT_TEAM : scopes
+ AGENT_TEAM ||--o{ TEAM_TASK : owns
+ TEAM_TASK ||--o{ TEAM_TASK_ATTEMPT : preserves
+ AGENT_TEAM ||--o{ AGENT_MESSAGE : records
+ WORKFLOW_CONTROL_EXECUTION ||--o{ SUBAGENT_CONCURRENCY_PERMIT : limits
+```
+
+SQLite startup migrations add new nullable columns and indexes without making
+historical rows unreadable. Historical attempts without execution links report
+`execution_not_registered`.
+
+## 12. Trace and observability architecture
+
+```mermaid
+sequenceDiagram
+ autonumber
+ actor Lead as Team lead / human
+ participant TM as Task Manager
+ participant API as TeamTaskTraceService
+ participant DB as Application DB
+ participant Temporal as Temporal History
+ participant Audit as Agent messages/audit
+
+ Lead->>TM: inspect_task_trace(task_id, attempt, detail)
+ TM->>API: trusted workflow + lead + execution scope
+ API->>DB: authorize task and resolve persisted attempt link
+ DB-->>API: actual workflow/run IDs
+ API->>Audit: trace_inspection event (no raw query)
+ API->>Temporal: fetch_history_events(cursor, bounded page)
+ Temporal-->>API: event page
+ API->>API: allowlist normalize + redact
+ alt detail = search
+ API->>API: literal/all/any term match + category filter + context window
+ end
+ API-->>TM: sanitized events + next_cursor + availability state
+```
+
+Trace detail modes:
+
+| Mode | Purpose |
+|---|---|
+| `summary` | Failures, activities, and child transitions |
+| `failures` | Failure and timeout events only |
+| `timeline` | Sanitized normalized event page |
+| `search` | Bounded grep-style scan with context and continuation cursor |
+
+Search scans at most 500 normalized events per request, supports literal,
+all-term, and any-term matching, filters by event category, and returns at most
+five surrounding events. Raw payload mode does not exist. Prompts, tool
+arguments, credentials, tokens, raw results, and arbitrary failure payloads are
+excluded by allowlist normalization and secret redaction.
+
+Temporal clients and workers use the SDK OpenTelemetry tracing interceptor.
+Only opaque correlation identities are added. Application task records remain
+available when Temporal retention has removed detailed history.
+
+## 13. Worker and task-queue topology
+
+```mermaid
+flowchart TB
+ Temporal[Temporal Server]
+ MainQ[machina-tasks]
+ AIQ[ai-heavy]
+ CodeQ[code-exec]
+ BrowserQ[browser]
+ RestQ[rest-api / service queues]
+
+ Temporal --> MainQ
+ Temporal --> AIQ
+ Temporal --> CodeQ
+ Temporal --> BrowserQ
+ Temporal --> RestQ
+
+ MainQ --> ManagerWorker[Manager worker: workflows, framework activities, legacy dispatch]
+ AIQ --> AIWorker[Pool worker: model/agent plugins]
+ CodeQ --> CodeWorker[Pool worker: code and process plugins]
+ BrowserQ --> BrowserWorker[Pool worker: browser automation]
+ RestQ --> ServiceWorker[Pool worker: API/service plugins]
+
+ ManagerWorker --> Shared[Shared activity services]
+ AIWorker --> Shared
+ CodeWorker --> Shared
+ BrowserWorker --> Shared
+ ServiceWorker --> Shared
+```
+
+Worker construction registers framework workflows, agent workflows, legacy
+listener compatibility workflows, plugin activities, polling activities, agent
+activities, and plugin-contributed Temporal workflows. Autoscaling pollers and
+sticky workflow caching reduce idle polling and replay overhead. Worker
+identity includes the task queue for operational diagnosis.
+
+## 14. Failure model and recovery
+
+| Failure | Expected behavior |
+|---|---|
+| API process restart | Temporal histories continue; DB control/task state survives |
+| Worker restart | Outstanding workflow/activity tasks are redelivered |
+| Activity transient failure | Retry according to shared or plugin policy |
+| User/configuration error | Non-retryable structured failure |
+| One parallel tool/child fails | Successful siblings continue; error returns in call order |
+| Persistence failure before delegation | Child is not started |
+| Permit wait | Task stays queued; waiting activity heartbeats |
+| Root cancellation | Graceful cancellation runs workflow cleanup and permit release |
+| Immediate Reset termination | Visible descendants terminate; interrupted task/permit reconciliation is a current follow-up |
+| Temporal unavailable during trace read | `temporal_unavailable`; persisted task result remains readable |
+| History retention expired | `retention_expired`; attempt metadata/result remains readable |
+| API/UI reconnect | Query DB/control status, then resume event subscriptions |
+
+The startup termination sweep defaults off and skips active product-managed
+control generations. Controlled trigger registration is durable inside the
+controller history. A remaining compatibility limitation is that the
+process-local `DeploymentManager` projection is not fully reconstructed from
+all saved generations on boot; Temporal and database authority survive, but
+some legacy UI/deployment counters may require reconciliation.
+
+## 15. Security and isolation
+
+1. Server-derived workflow, lead, execution, team, and attempt scope is used for
+ every mutation and trace read.
+2. Cross-team, cross-execution, removed-teammate, ordinary tool-edge, ambiguous,
+ and forged assignments are rejected.
+3. Each child agent binds only its own graph-connected capabilities.
+4. Conversation history is not inherited by delegated children.
+5. Trace retrieval resolves persisted execution links; caller workflow IDs are
+ never treated as authority.
+6. Trace output uses allowlist serialization and secret redaction.
+7. OTel attributes exclude prompts, credentials, arguments, and results.
+8. Optimistic revisions prevent UI/model callers from overwriting newer task or
+ control state.
+
+## 16. Determinism and compatibility rules
+
+- Workflow code may mutate only replayable workflow state and issue Temporal
+ commands.
+- Non-deterministic work belongs in activities.
+- Command IDs derive from stable node, iteration, call-index, task, and
+ generation identities.
+- Temporal patch markers preserve old command histories when command identity
+ or scheduling structure changes.
+- Workflow type and activity names are durable protocol identifiers; product
+ rebranding does not rename existing Temporal contracts casually.
+- Legacy listener workflows and the single node dispatcher remain registered
+ until all replayable histories and imported deployments no longer require
+ them.
+
+## 17. Operational views
+
+### Toolbar
+
+- `never_started` or `ready`: Start
+- `running`: Pause
+- `paused`: Resume
+- transition states: disabled spinner
+- any active/failed generation: Reset when allowed
+
+### Task Manager
+
+Operational, mutable, execution-scoped view of all durable tasks, attempts,
+usage, timestamps, results, errors, and sanitized Temporal traces.
+
+### Team Monitor
+
+Compact read-only view of connected teammates, current generation, queued and
+active work, and child execution state. It does not replace Task Manager for
+review or mutation.
+
+### Temporal Web
+
+Use for workflow history, activity attempts, timers, Signals, child relations,
+and worker diagnosis. Historical executions remain visible by design. A
+controlled push/poll trigger does not create a listener workflow row; an actual
+triggered graph invocation does.
+
+## 18. Capacity and scaling
+
+- Root graph branches scale through independent activity task queues.
+- Agent tool calls and child work run concurrently when independent.
+- Global descendant admission defaults to three per root and is user-configurable
+ within server limits.
+- Delegation depth is capped at two.
+- Poller autoscaling and worker pools separate heavy AI, code, browser, and
+ service workloads.
+- Histories must use continue-as-new or bounded execution for genuinely
+ unbounded workloads. The consolidated controller currently accumulates
+ trigger registration and event history for its generation; production
+ retention/load testing should define a safe continue-as-new threshold that
+ carries trigger specs, dedup state, polling baselines, pause state, and
+ queued events.
+
+## 19. Known limitations and follow-ups
+
+1. Rebuild all process-local deployment projections from control records and
+ Temporal Visibility after API restart.
+2. Add controller continue-as-new with lossless trigger/poll state transfer.
+3. Consolidate cron firing into the controller only if Temporal Schedule
+ calendar semantics can be preserved without reimplementing them.
+4. Replace global controller event fan-out with an indexed multi-event routing
+ attribute or dedicated durable routing index as deployment count grows.
+5. Add end-to-end tests against a real Temporal test server for pause races,
+ reset termination, schedule pause, replay, and worker restart.
+6. Define namespace retention and archival policies for high-volume traces.
+7. Add an idempotent Reset reconciliation activity that marks interrupted
+ attempts `cancelled` with `workflow_reset` and releases permits even though
+ Temporal termination intentionally does not execute workflow `finally`
+ blocks.
+
+## 20. Source map
+
+| Concern | Primary implementation |
+|---|---|
+| Control persistence/service | `server/models/database.py`, `server/core/database.py`, `server/services/deployment/control.py` |
+| Control commands | `server/services/deployment/handlers.py` |
+| Controller and trigger hub | `server/services/temporal/workflow_control_workflow.py` |
+| Event routing | `server/services/events/dispatch.py` |
+| Cron schedules | `server/services/temporal/schedules.py` |
+| Root graph orchestration | `server/services/temporal/workflow.py` |
+| Temporal start identity | `server/services/temporal/executor.py` |
+| Agent and delegation workflows | `server/services/temporal/agent_workflow.py` |
+| Agent lifecycle activities | `server/services/temporal/agent_activities.py` |
+| Team/task authorization | `server/services/agent_team.py` |
+| Trace service | `server/services/team_task_trace.py` |
+| Worker topology | `server/services/temporal/worker.py`, `server/services/temporal/plugin_activities.py` |
+| Toolbar/UI control | `client/src/Dashboard.tsx`, `client/src/components/ui/TopToolbar.tsx`, `client/src/contexts/WebSocketContext.tsx` |
+| Task/monitor UI | `client/src/components/parameterPanel/TaskManagerPanel.tsx`, `TeamMonitorPanel.tsx` |
+
+## 21. References
+
+- [Temporal Python message passing](https://docs.temporal.io/develop/python/message-passing)
+- [Temporal Workflow identity](https://docs.temporal.io/workflow-execution/workflowid-runid)
+- [Temporal Schedules](https://docs.temporal.io/develop/python/schedules)
+- [Temporal observability](https://docs.temporal.io/develop/python/observability)
+- [Temporal child workflows](https://docs.temporal.io/child-workflows)
+- [OpenTelemetry context propagation](https://opentelemetry.io/docs/concepts/context-propagation/)
+- [Internal workflow-control notes](temporal-workflow-control.md)
+- [Detailed activity/worker inventory](TEMPORAL_ARCHITECTURE.md)
+- [Event framework](event_framework.md)
+- [Task Manager contract](node-logic-flows/ai_tools/taskManager.md)
+
+## 22. Acceptance criteria for architectural changes
+
+Any change to this engine must demonstrate:
+
+1. Replay determinism for affected workflows.
+2. Idempotency under activity, command, Signal, and client retries.
+3. Preservation of workflow/control/task authorization boundaries.
+4. No loss of queued trigger events or delegated tasks across pause/restart.
+5. Permit release under success, failure, cancellation, and reset.
+6. Stable or migration-protected workflow/activity identifiers.
+7. Sanitized trace output with bounded pagination/search.
+8. UI reconciliation from backend authority after reconnect.
+9. Focused unit tests plus an integration or replay test proportional to risk.
diff --git a/docs-internal/temporal-workflow-control.md b/docs-internal/temporal-workflow-control.md
new file mode 100644
index 00000000..d217954a
--- /dev/null
+++ b/docs-internal/temporal-workflow-control.md
@@ -0,0 +1,60 @@
+# Temporal Workflow Control and Team Traces
+
+Workflow deployments use a persisted control generation and a long-lived
+`WorkflowControlWorkflow`. The application database is authoritative for the
+current generation, revision, graph snapshot, and UI authorization; Temporal
+is authoritative for execution history.
+
+## Control lifecycle
+
+- **Start** creates the first generation and deploys its snapshotted graph.
+- **Pause** cooperatively gates new trigger admissions, workflow nodes, agent
+ turns, tool calls, polling iterations, and delegated work. In-flight work may
+ finish and remains durable. Push events stay queued in the controller,
+ provider polling cannot launch graph runs, Temporal cron schedules are
+ paused, and armed trigger nodes switch to an explicit paused visual state.
+- **Resume** signals the same running Temporal executions and drains buffered
+ trigger events in FIFO order, unpauses cron schedules, and rearms trigger
+ nodes.
+- **Reset** revision-guards the old generation, terminates visible executions
+ carrying the workflow search attribute, cancels local deployment resources,
+ and archives the old generation. It leaves the control state `ready`; the
+ user must press **Start** to create the next generation.
+
+All mutations require an expected revision and support an idempotency key.
+The toolbar and command palette derive available actions from the server's
+`can_start`, `can_pause`, `can_resume`, and `can_reset` fields.
+
+On server restart, active controlled deployments are excluded from the legacy
+startup termination sweep. `TEMPORAL_TERMINATE_RUNNING_ON_STARTUP` defaults to
+`false`; enabling it is intended only for legacy installations that explicitly
+prefer termination over durable resumption.
+
+## Delegated-task traces
+
+Each task attempt stores the actual parent, detached runner, and child Temporal
+workflow/run identities registered from `workflow.info()` at child startup.
+Retries and reassignments create immutable attempts instead of overwriting old
+links or results.
+
+`get_team_task_trace` and Task Manager's `inspect_task_trace` first authorize a
+task through its saved workflow, lead, execution, and team membership. They
+then fetch only the persisted Temporal execution link. Results are sanitized,
+cursor-paginated (50 by default, 100 maximum), and limited to normalized
+workflow, child, activity, retry, timer, signal, cancellation, and failure
+metadata. Raw prompts, arguments, secrets, and results are never exposed.
+
+Large histories support bounded grep-style inspection with literal or term
+matching, category filters, contextual events, and opaque continuation cursors.
+The lead searches sanitized projections in chunks instead of loading the full
+Temporal history into its conversation.
+
+The Task Manager Trace tab loads history only when opened. Team Monitor remains
+a lightweight read-only view of generation and child execution state.
+
+Controlled generations do not create trigger-listener workflow executions.
+The `WorkflowControlWorkflow` stores trigger registrations and inbound push
+events in its own history and performs polling cycles as activities. Only a
+real triggered graph execution becomes a child workflow. Legacy deployments
+created before workflow control retain the standalone listener compatibility
+path until they are reset and explicitly started again.
diff --git a/server/config/model_registry.json b/server/config/model_registry.json
index a3845c6f..8ad689f1 100644
--- a/server/config/model_registry.json
+++ b/server/config/model_registry.json
@@ -1,7 +1,7 @@
{
- "_generated": "2026-07-18T13:57:53.993876+00:00",
+ "_generated": "2026-07-19T18:45:02.760858+00:00",
"_source": "openrouter",
- "_model_count": 344,
+ "_model_count": 338,
"models": {
"thinkingmachines/inkling": {
"id": "thinkingmachines/inkling",
@@ -831,8 +831,8 @@
"local_id": "glm-5.2",
"context_length": 1048576,
"max_output_tokens": 131072,
- "input_price_per_mtok": 0.2954,
- "output_price_per_mtok": 0.9284,
+ "input_price_per_mtok": 0.238,
+ "output_price_per_mtok": 0.748,
"supports_thinking": false,
"thinking_type": "none",
"temperature_range": [
@@ -889,8 +889,8 @@
"local_id": "kimi-k2.7-code",
"context_length": 262144,
"max_output_tokens": 262144,
- "input_price_per_mtok": 1.0,
- "output_price_per_mtok": 4.4,
+ "input_price_per_mtok": 0.85,
+ "output_price_per_mtok": 3.8,
"supports_thinking": false,
"thinking_type": "none",
"temperature_range": [
@@ -2474,9 +2474,9 @@
"provider": "moonshotai",
"local_id": "kimi-k2.6",
"context_length": 262144,
- "max_output_tokens": 0,
- "input_price_per_mtok": 0.95,
- "output_price_per_mtok": 4.0,
+ "max_output_tokens": 262144,
+ "input_price_per_mtok": 0.684,
+ "output_price_per_mtok": 3.42,
"supports_thinking": false,
"thinking_type": "none",
"temperature_range": [
@@ -2581,9 +2581,9 @@
"provider": "gemini",
"local_id": "gemma-4-26b-a4b-it",
"context_length": 262144,
- "max_output_tokens": 256000,
- "input_price_per_mtok": 0.1,
- "output_price_per_mtok": 0.3,
+ "max_output_tokens": 16384,
+ "input_price_per_mtok": 0.07,
+ "output_price_per_mtok": 0.34,
"supports_thinking": false,
"thinking_type": "none",
"temperature_range": [
@@ -2655,9 +2655,9 @@
"provider": "gemini",
"local_id": "gemma-4-31b-it",
"context_length": 262144,
- "max_output_tokens": 16384,
- "input_price_per_mtok": 0.12,
- "output_price_per_mtok": 0.37,
+ "max_output_tokens": 262144,
+ "input_price_per_mtok": 0.22,
+ "output_price_per_mtok": 0.55,
"supports_thinking": false,
"thinking_type": "none",
"temperature_range": [
@@ -3144,9 +3144,9 @@
"provider": "nvidia",
"local_id": "nemotron-3-super-120b-a12b",
"context_length": 1000000,
- "max_output_tokens": 0,
- "input_price_per_mtok": 0.21,
- "output_price_per_mtok": 0.455,
+ "max_output_tokens": 16384,
+ "input_price_per_mtok": 0.085,
+ "output_price_per_mtok": 0.4,
"supports_thinking": false,
"thinking_type": "none",
"temperature_range": [
@@ -5392,8 +5392,8 @@
"local_id": "minimax-m2",
"context_length": 204800,
"max_output_tokens": 131072,
- "input_price_per_mtok": 0.255,
- "output_price_per_mtok": 1.02,
+ "input_price_per_mtok": 0.3,
+ "output_price_per_mtok": 1.2,
"supports_thinking": false,
"thinking_type": "none",
"temperature_range": [
@@ -6298,36 +6298,6 @@
"top_p"
]
},
- "qwen/qwen3-next-80b-a3b-instruct:free": {
- "id": "qwen/qwen3-next-80b-a3b-instruct:free",
- "name": "Qwen: Qwen3 Next 80B A3B Instruct (free)",
- "provider": "qwen",
- "local_id": "qwen3-next-80b-a3b-instruct:free",
- "context_length": 262144,
- "max_output_tokens": 0,
- "input_price_per_mtok": 0.0,
- "output_price_per_mtok": 0.0,
- "supports_thinking": false,
- "thinking_type": "none",
- "temperature_range": [
- 0.0,
- 2.0
- ],
- "is_reasoning_model": false,
- "supported_parameters": [
- "frequency_penalty",
- "max_tokens",
- "presence_penalty",
- "response_format",
- "stop",
- "structured_outputs",
- "temperature",
- "tool_choice",
- "tools",
- "top_k",
- "top_p"
- ]
- },
"qwen/qwen-plus-2025-07-28": {
"id": "qwen/qwen-plus-2025-07-28",
"name": "Qwen: Qwen Plus 0728",
@@ -7154,34 +7124,6 @@
"top_p"
]
},
- "qwen/qwen3-coder:free": {
- "id": "qwen/qwen3-coder:free",
- "name": "Qwen: Qwen3 Coder 480B A35B (free)",
- "provider": "qwen",
- "local_id": "qwen3-coder:free",
- "context_length": 1048576,
- "max_output_tokens": 262000,
- "input_price_per_mtok": 0.0,
- "output_price_per_mtok": 0.0,
- "supports_thinking": false,
- "thinking_type": "none",
- "temperature_range": [
- 0.0,
- 2.0
- ],
- "is_reasoning_model": false,
- "supported_parameters": [
- "frequency_penalty",
- "max_tokens",
- "presence_penalty",
- "stop",
- "temperature",
- "tool_choice",
- "tools",
- "top_k",
- "top_p"
- ]
- },
"bytedance/ui-tars-1.5-7b": {
"id": "bytedance/ui-tars-1.5-7b",
"name": "ByteDance: UI-TARS 7B ",
@@ -7337,34 +7279,6 @@
"top_p"
]
},
- "cognitivecomputations/dolphin-mistral-24b-venice-edition:free": {
- "id": "cognitivecomputations/dolphin-mistral-24b-venice-edition:free",
- "name": "Venice: Uncensored (free)",
- "provider": "cognitivecomputations",
- "local_id": "dolphin-mistral-24b-venice-edition:free",
- "context_length": 32768,
- "max_output_tokens": 0,
- "input_price_per_mtok": 0.0,
- "output_price_per_mtok": 0.0,
- "supports_thinking": false,
- "thinking_type": "none",
- "temperature_range": [
- 0.0,
- 2.0
- ],
- "is_reasoning_model": false,
- "supported_parameters": [
- "frequency_penalty",
- "max_tokens",
- "presence_penalty",
- "response_format",
- "stop",
- "structured_outputs",
- "temperature",
- "top_k",
- "top_p"
- ]
- },
"tencent/hunyuan-a13b-instruct": {
"id": "tencent/hunyuan-a13b-instruct",
"name": "Tencent: Hunyuan A13B Instruct",
@@ -7975,9 +7889,9 @@
"provider": "qwen",
"local_id": "qwen3-14b",
"context_length": 131702,
- "max_output_tokens": 8192,
- "input_price_per_mtok": 0.2275,
- "output_price_per_mtok": 0.91,
+ "max_output_tokens": 16384,
+ "input_price_per_mtok": 0.12,
+ "output_price_per_mtok": 0.24,
"supports_thinking": false,
"thinking_type": "none",
"temperature_range": [
@@ -9229,34 +9143,6 @@
"top_p"
]
},
- "meta/llama-3.3-70b-instruct:free": {
- "id": "meta-llama/llama-3.3-70b-instruct:free",
- "name": "Meta: Llama 3.3 70B Instruct (free)",
- "provider": "meta",
- "local_id": "llama-3.3-70b-instruct:free",
- "context_length": 131072,
- "max_output_tokens": 0,
- "input_price_per_mtok": 0.0,
- "output_price_per_mtok": 0.0,
- "supports_thinking": false,
- "thinking_type": "none",
- "temperature_range": [
- 0.0,
- 2.0
- ],
- "is_reasoning_model": false,
- "supported_parameters": [
- "frequency_penalty",
- "max_tokens",
- "presence_penalty",
- "stop",
- "temperature",
- "tool_choice",
- "tools",
- "top_k",
- "top_p"
- ]
- },
"amazon/nova-lite-v1": {
"id": "amazon/nova-lite-v1",
"name": "Amazon: Nova Lite 1.0",
@@ -9673,32 +9559,6 @@
"top_p"
]
},
- "meta/llama-3.2-3b-instruct:free": {
- "id": "meta-llama/llama-3.2-3b-instruct:free",
- "name": "Meta: Llama 3.2 3B Instruct (free)",
- "provider": "meta",
- "local_id": "llama-3.2-3b-instruct:free",
- "context_length": 131072,
- "max_output_tokens": 0,
- "input_price_per_mtok": 0.0,
- "output_price_per_mtok": 0.0,
- "supports_thinking": false,
- "thinking_type": "none",
- "temperature_range": [
- 0.0,
- 2.0
- ],
- "is_reasoning_model": false,
- "supported_parameters": [
- "frequency_penalty",
- "max_tokens",
- "presence_penalty",
- "stop",
- "temperature",
- "top_k",
- "top_p"
- ]
- },
"qwen/qwen-2.5-72b-instruct": {
"id": "qwen/qwen-2.5-72b-instruct",
"name": "Qwen2.5 72B Instruct",
@@ -9893,32 +9753,6 @@
"top_p"
]
},
- "nousresearch/hermes-3-llama-3.1-405b:free": {
- "id": "nousresearch/hermes-3-llama-3.1-405b:free",
- "name": "Nous: Hermes 3 405B Instruct (free)",
- "provider": "nousresearch",
- "local_id": "hermes-3-llama-3.1-405b:free",
- "context_length": 131072,
- "max_output_tokens": 0,
- "input_price_per_mtok": 0.0,
- "output_price_per_mtok": 0.0,
- "supports_thinking": false,
- "thinking_type": "none",
- "temperature_range": [
- 0.0,
- 2.0
- ],
- "is_reasoning_model": false,
- "supported_parameters": [
- "frequency_penalty",
- "max_tokens",
- "presence_penalty",
- "stop",
- "temperature",
- "top_k",
- "top_p"
- ]
- },
"sao10k/l3-lunaris-8b": {
"id": "sao10k/l3-lunaris-8b",
"name": "Sao10K: Llama 3 8B Lunaris",
diff --git a/server/core/database.py b/server/core/database.py
index 99a6e23a..aa95154b 100644
--- a/server/core/database.py
+++ b/server/core/database.py
@@ -40,6 +40,7 @@ from models.database import (
GoogleConnection,
ProxyProviderConfig,
ProxyRoutingRule,
+ WorkflowControlExecution,
)
from models.cache import CacheEntry # SQLite-backed cache for Redis alternative
from core.logging import get_logger
@@ -107,6 +108,7 @@ class Database:
# Add missing columns to existing tables (simple migration)
await self._migrate_user_settings()
await self._migrate_agent_teams()
+ await self._migrate_workflow_controls()
logger.info("Database initialized successfully")
@@ -279,6 +281,8 @@ class Database:
"acceptance_criteria": "JSON", "queue_sequence": "INTEGER DEFAULT 0",
"revision": "INTEGER DEFAULT 0", "current_attempt": "INTEGER DEFAULT 0",
"child_workflow_id": "VARCHAR(500)", "child_run_id": "VARCHAR(255)",
+ "runner_workflow_id": "VARCHAR(500)", "runner_run_id": "VARCHAR(255)",
+ "parent_workflow_id": "VARCHAR(500)", "parent_run_id": "VARCHAR(255)",
"trace_id": "VARCHAR(255)", "cancellation_requested": "BOOLEAN DEFAULT 0",
"cancellation_reason": "VARCHAR(2000)", "usage": "JSON",
}
@@ -292,6 +296,16 @@ class Database:
await conn.execute(text("UPDATE team_tasks SET status='cancelled' WHERE status='skipped'"))
await conn.execute(text("CREATE INDEX IF NOT EXISTS ix_team_tasks_execution_id ON team_tasks(execution_id)"))
await conn.execute(text("CREATE INDEX IF NOT EXISTS ix_team_tasks_root_execution_id ON team_tasks(root_execution_id)"))
+
+ result = await conn.execute(text("PRAGMA table_info(team_task_attempts)"))
+ attempt_columns = {row[1] for row in result.fetchall()}
+ attempt_additions = {
+ "runner_workflow_id": "VARCHAR(500)", "runner_run_id": "VARCHAR(255)",
+ "parent_workflow_id": "VARCHAR(500)", "parent_run_id": "VARCHAR(255)",
+ }
+ for column, definition in attempt_additions.items():
+ if attempt_columns and column not in attempt_columns:
+ await conn.execute(text(f"ALTER TABLE team_task_attempts ADD COLUMN {column} {definition}"))
except Exception as e:
logger.warning(f"Agent-team migration check failed: {e}")
@@ -316,6 +330,27 @@ class Database:
finally:
await session.close()
+ async def _migrate_workflow_controls(self):
+ """Backfill control-plane columns when upgrading an early preview DB."""
+ try:
+ async with self.engine.begin() as conn:
+ result = await conn.execute(text("PRAGMA table_info(workflow_control_executions)"))
+ columns = {row[1] for row in result.fetchall()}
+ additions = {
+ "controller_workflow_id": "VARCHAR(500)",
+ "controller_run_id": "VARCHAR(255)",
+ "resource_manifest": "JSON DEFAULT '{}'",
+ "terminal_reason": "VARCHAR(2000)",
+ "completed_at": "DATETIME",
+ }
+ for column, definition in additions.items():
+ if column not in columns:
+ await conn.execute(text(f"ALTER TABLE workflow_control_executions ADD COLUMN {column} {definition}"))
+ await conn.execute(text("CREATE UNIQUE INDEX IF NOT EXISTS uq_workflow_control_generation ON workflow_control_executions(workflow_id, generation)"))
+ await conn.execute(text("CREATE UNIQUE INDEX IF NOT EXISTS uq_workflow_control_idempotency ON workflow_control_executions(workflow_id, idempotency_key)"))
+ except Exception as exc:
+ logger.warning(f"Workflow-control migration check failed: {exc}")
+
async def run_runtime_mutation(
self,
*,
@@ -1500,10 +1535,6 @@ class Database:
if not schema:
return None
- pending_count = sum(1 for t in tasks if t.status == "pending")
- active_count = sum(1 for t in tasks if t.status == "in_progress")
- completed_count = sum(1 for t in tasks if t.status == "completed")
- failed_count = sum(1 for t in tasks if t.status == "failed")
return {
"node_id": schema.node_id,
"tool_name": schema.tool_name,
@@ -2378,6 +2409,73 @@ class Database:
logger.error("Failed to get compaction history", session_id=session_id, error=str(e))
return []
+ # ============================================================================
+ # Durable workflow control
+ # ============================================================================
+
+ async def get_latest_workflow_control(self, workflow_id: str) -> Optional[WorkflowControlExecution]:
+ async with self.get_session() as session:
+ result = await session.execute(
+ select(WorkflowControlExecution)
+ .where(WorkflowControlExecution.workflow_id == workflow_id)
+ .order_by(WorkflowControlExecution.generation.desc())
+ .limit(1)
+ )
+ return result.scalar_one_or_none()
+
+ async def has_active_workflow_controls(self) -> bool:
+ async with self.get_session() as session:
+ result = await session.execute(
+ select(func.count()).select_from(WorkflowControlExecution).where(
+ WorkflowControlExecution.status.in_({"starting", "running", "pausing", "paused", "resuming"})
+ )
+ )
+ return bool(result.scalar_one())
+
+ async def get_workflow_control_by_idempotency_key(
+ self, workflow_id: str, idempotency_key: str
+ ) -> Optional[WorkflowControlExecution]:
+ async with self.get_session() as session:
+ result = await session.execute(
+ select(WorkflowControlExecution).where(
+ WorkflowControlExecution.workflow_id == workflow_id,
+ WorkflowControlExecution.idempotency_key == idempotency_key,
+ )
+ )
+ return result.scalar_one_or_none()
+
+ async def create_workflow_control(self, control: WorkflowControlExecution) -> WorkflowControlExecution:
+ async with self.get_session() as session:
+ session.add(control)
+ await session.commit()
+ await session.refresh(control)
+ return control
+
+ async def transition_workflow_control(
+ self, control_id: str, *, expected_revision: int, from_statuses: set[str], status: str,
+ values: Optional[Dict[str, Any]] = None,
+ ) -> Optional[WorkflowControlExecution]:
+ """Optimistically transition one generation; return None on conflict."""
+ now = datetime.now(timezone.utc)
+ update_values: Dict[str, Any] = {"status": status, "revision": expected_revision + 1, "updated_at": now}
+ update_values.update(values or {})
+ async with self.get_session() as session:
+ result = await session.execute(
+ update(WorkflowControlExecution)
+ .where(
+ WorkflowControlExecution.id == control_id,
+ WorkflowControlExecution.revision == expected_revision,
+ WorkflowControlExecution.status.in_(from_statuses),
+ )
+ .values(**update_values)
+ )
+ if result.rowcount != 1:
+ await session.rollback()
+ return None
+ await session.commit()
+ refreshed = await session.execute(select(WorkflowControlExecution).where(WorkflowControlExecution.id == control_id))
+ return refreshed.scalar_one()
+
# ============================================================================
# Agent Teams - CRUD Operations
# ============================================================================
@@ -2632,6 +2730,12 @@ class Database:
"revision": task.revision, "current_attempt": task.current_attempt,
"retry_count": task.retry_count, "max_retries": task.max_retries,
"child_workflow_id": task.child_workflow_id, "child_run_id": task.child_run_id,
+ "runner_workflow_id": task.runner_workflow_id, "runner_run_id": task.runner_run_id,
+ "parent_workflow_id": task.parent_workflow_id, "parent_run_id": task.parent_run_id,
+ "trace_availability": (
+ "available" if task.child_workflow_id or task.runner_workflow_id
+ else "execution_not_registered"
+ ),
"trace_id": task.trace_id, "cancellation_requested": task.cancellation_requested,
"cancellation_reason": task.cancellation_reason, "usage": task.usage,
"created_at": task.created_at.isoformat() if task.created_at else None,
@@ -2641,6 +2745,8 @@ class Database:
{"id": a.id, "attempt_number": a.attempt_number, "assignee_node_id": a.assignee_node_id,
"status": a.status, "child_workflow_id": a.child_workflow_id,
"child_run_id": a.child_run_id, "result": a.result, "error": a.error,
+ "runner_workflow_id": a.runner_workflow_id, "runner_run_id": a.runner_run_id,
+ "parent_workflow_id": a.parent_workflow_id, "parent_run_id": a.parent_run_id,
"usage": a.usage, "created_at": a.created_at.isoformat() if a.created_at else None,
"started_at": a.started_at.isoformat() if a.started_at else None,
"completed_at": a.completed_at.isoformat() if a.completed_at else None}
@@ -2712,6 +2818,45 @@ class Database:
await session.refresh(task)
return self._team_task_dict(task)
+ async def register_team_task_execution(
+ self, *, team_id: str, task_id: str, attempt_number: int,
+ runner_workflow_id: Optional[str] = None, runner_run_id: Optional[str] = None,
+ child_workflow_id: Optional[str] = None, child_run_id: Optional[str] = None,
+ parent_workflow_id: Optional[str] = None, parent_run_id: Optional[str] = None,
+ ) -> Optional[Dict[str, Any]]:
+ """Idempotently attach actual Temporal identities to one task attempt."""
+ async with self.get_session() as session:
+ task = (await session.execute(select(TeamTask).where(
+ TeamTask.id == task_id, TeamTask.team_id == team_id,
+ ))).scalar_one_or_none()
+ if task is None or task.current_attempt != attempt_number:
+ return None
+ values = {
+ key: value for key, value in {
+ "runner_workflow_id": runner_workflow_id, "runner_run_id": runner_run_id,
+ "child_workflow_id": child_workflow_id, "child_run_id": child_run_id,
+ "parent_workflow_id": parent_workflow_id, "parent_run_id": parent_run_id,
+ }.items() if value
+ }
+ for key, value in values.items():
+ setattr(task, key, value)
+ attempt_id = f"{task_id}:attempt:{attempt_number}"
+ attempt = (await session.execute(select(TeamTaskAttempt).where(
+ TeamTaskAttempt.id == attempt_id,
+ ))).scalar_one_or_none()
+ if attempt is None:
+ attempt = TeamTaskAttempt(
+ id=attempt_id, task_id=task_id, team_id=team_id,
+ attempt_number=attempt_number, assignee_node_id=task.assigned_to,
+ status=task.status, started_at=task.started_at or datetime.now(timezone.utc),
+ )
+ session.add(attempt)
+ for key, value in values.items():
+ setattr(attempt, key, value)
+ await session.commit()
+ await session.refresh(task)
+ return self._team_task_dict(task, [attempt])
+
async def get_team_tasks(self, team_id: str, status: Optional[str] = None) -> List[Dict[str, Any]]:
"""Get team tasks, optionally filtered by status."""
try:
@@ -2805,6 +2950,11 @@ class Database:
)
)
if completed.rowcount == 1:
+ task = (await session.execute(select(TeamTask).where(TeamTask.id == task_id))).scalar_one()
+ await session.execute(update(TeamTaskAttempt).where(
+ TeamTaskAttempt.id == f"{task_id}:attempt:{task.current_attempt}"
+ ).values(status="submitted", result=result_data, usage=usage,
+ completed_at=datetime.now(timezone.utc)))
return {"success": True}
selected = await session.execute(
@@ -2858,6 +3008,10 @@ class Database:
)
)
if failed.rowcount == 1:
+ task = (await session.execute(select(TeamTask).where(TeamTask.id == task_id))).scalar_one()
+ await session.execute(update(TeamTaskAttempt).where(
+ TeamTaskAttempt.id == f"{task_id}:attempt:{task.current_attempt}"
+ ).values(status="failed", error=error, completed_at=datetime.now(timezone.utc)))
return {"success": True}
selected = await session.execute(
diff --git a/server/main.py b/server/main.py
index 63da6578..2697642e 100644
--- a/server/main.py
+++ b/server/main.py
@@ -335,7 +335,12 @@ async def lifespan(app: FastAPI):
# TEMPORAL_TERMINATE_RUNNING_ON_STARTUP=false.
if settings.temporal_terminate_running_on_startup:
try:
- terminated = await temporal_client_wrapper.terminate_running_workflows()
+ has_controls = await container.database().has_active_workflow_controls()
+ terminated = 0
+ if has_controls:
+ logger.info("Skipping startup Temporal termination sweep; durable workflow controls are active")
+ else:
+ terminated = await temporal_client_wrapper.terminate_running_workflows()
if terminated:
_startup_log(
f"[Temporal] Terminated {terminated} running workflow(s) " "at startup (history preserved)"
diff --git a/server/models/database.py b/server/models/database.py
index 8a4b91b0..43f8f05e 100644
--- a/server/models/database.py
+++ b/server/models/database.py
@@ -3,7 +3,7 @@
from datetime import datetime, timezone
from typing import Optional, Dict, Any
from sqlmodel import SQLModel, Field, Column, DateTime, JSON
-from sqlalchemy import func
+from sqlalchemy import UniqueConstraint, func
class NodeParameter(SQLModel, table=True):
@@ -457,6 +457,35 @@ class AgentTeam(SQLModel, table=True):
completed_at: Optional[datetime] = Field(default=None)
+class WorkflowControlExecution(SQLModel, table=True):
+ """Durable control generation for one saved workflow deployment."""
+
+ __tablename__ = "workflow_control_executions"
+ __table_args__ = (
+ UniqueConstraint("workflow_id", "generation", name="uq_workflow_control_generation"),
+ UniqueConstraint("workflow_id", "idempotency_key", name="uq_workflow_control_idempotency"),
+ )
+
+ id: str = Field(primary_key=True, max_length=255)
+ workflow_id: str = Field(index=True, max_length=255)
+ generation: int = Field(index=True)
+ execution_id: str = Field(index=True, max_length=255)
+ root_execution_id: str = Field(index=True, max_length=255)
+ controller_workflow_id: Optional[str] = Field(default=None, max_length=500)
+ controller_run_id: Optional[str] = Field(default=None, max_length=255)
+ session_id: str = Field(default="default", max_length=255)
+ graph_hash: str = Field(max_length=64)
+ graph_snapshot: Dict[str, Any] = Field(default_factory=dict, sa_column=Column(JSON))
+ resource_manifest: Dict[str, Any] = Field(default_factory=dict, sa_column=Column(JSON))
+ status: str = Field(default="starting", index=True, max_length=20)
+ revision: int = Field(default=0)
+ idempotency_key: str = Field(index=True, max_length=255)
+ terminal_reason: Optional[str] = Field(default=None, max_length=2000)
+ created_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
+ updated_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
+ completed_at: Optional[datetime] = Field(default=None)
+
+
class TeamMember(SQLModel, table=True):
"""Agent team membership.
@@ -507,6 +536,10 @@ class TeamTask(SQLModel, table=True):
current_attempt: int = Field(default=0)
child_workflow_id: Optional[str] = Field(default=None, max_length=500)
child_run_id: Optional[str] = Field(default=None, max_length=255)
+ runner_workflow_id: Optional[str] = Field(default=None, max_length=500)
+ runner_run_id: Optional[str] = Field(default=None, max_length=255)
+ parent_workflow_id: Optional[str] = Field(default=None, max_length=500)
+ parent_run_id: Optional[str] = Field(default=None, max_length=255)
trace_id: Optional[str] = Field(default=None, index=True, max_length=255)
cancellation_requested: bool = Field(default=False)
cancellation_reason: Optional[str] = Field(default=None, max_length=2000)
@@ -537,6 +570,10 @@ class TeamTaskAttempt(SQLModel, table=True):
status: str = Field(default="queued", max_length=20)
child_workflow_id: Optional[str] = Field(default=None, max_length=500)
child_run_id: Optional[str] = Field(default=None, max_length=255)
+ runner_workflow_id: Optional[str] = Field(default=None, max_length=500)
+ runner_run_id: Optional[str] = Field(default=None, max_length=255)
+ parent_workflow_id: Optional[str] = Field(default=None, max_length=500)
+ parent_run_id: Optional[str] = Field(default=None, max_length=255)
result: Optional[Dict[str, Any]] = Field(default=None, sa_column=Column(JSON))
error: Optional[str] = Field(default=None, max_length=5000)
usage: Optional[Dict[str, Any]] = Field(default=None, sa_column=Column(JSON))
diff --git a/server/nodes/tool/task_manager/__init__.py b/server/nodes/tool/task_manager/__init__.py
index f839f4ef..b63c3e1d 100644
--- a/server/nodes/tool/task_manager/__init__.py
+++ b/server/nodes/tool/task_manager/__init__.py
@@ -11,6 +11,7 @@ from services.plugin import NodeContext, Operation, TaskQueue, ToolNode
TaskOperation = Literal[
"assign_task", "list_tasks", "get_task", "modify_task", "cancel_task",
"retry_task", "reassign_task", "accept_task", "finish_team", "mark_done",
+ "inspect_task_trace",
]
@@ -28,6 +29,16 @@ class TaskManagerParams(BaseModel):
reason: Optional[str] = Field(default=None, max_length=2000)
status_filter: Optional[str] = None
include_history: bool = False
+ attempt: Optional[int] = Field(default=None, ge=0)
+ cursor: Optional[str] = None
+ limit: int = Field(default=50, ge=1, le=100)
+ detail: Literal["summary", "failures", "timeline", "search"] = "summary"
+ query: Optional[str] = Field(default=None, max_length=200)
+ search_mode: Literal["literal", "all_terms", "any_terms"] = "literal"
+ case_sensitive: bool = False
+ context_lines: int = Field(default=2, ge=0, le=5)
+ scan_limit: int = Field(default=250, ge=1, le=500)
+ categories: Optional[List[Literal["failure", "activity", "child", "signal", "timer", "workflow"]]] = None
model_config = ConfigDict(extra="allow")
@@ -112,6 +123,7 @@ async def _execute_task_manager(args: Dict[str, Any], config: Dict[str, Any]) ->
allowed_operations = {
"assign_task", "list_tasks", "get_task", "modify_task", "cancel_task",
"retry_task", "reassign_task", "accept_task", "finish_team", "mark_done",
+ "inspect_task_trace",
}
if operation not in allowed_operations:
raise ValueError(f"Unknown Task Manager operation: {operation}")
@@ -133,6 +145,22 @@ async def _execute_task_manager(args: Dict[str, Any], config: Dict[str, Any]) ->
raise ValueError("get_task requires task_id")
task = await service.get_durable_task(**scope, task_id=task_id)
return {"success": True, "operation": operation, "task": task}
+ if operation == "inspect_task_trace":
+ task_id = str(args.get("task_id") or "")
+ if not task_id:
+ raise ValueError("inspect_task_trace requires task_id")
+ from services.team_task_trace import get_team_task_trace_service
+
+ trace = await get_team_task_trace_service().get_trace(
+ **scope, task_id=task_id, attempt=args.get("attempt"),
+ cursor=args.get("cursor"), limit=int(args.get("limit") or 50),
+ detail=str(args.get("detail") or "summary"),
+ query=args.get("query"), search_mode=str(args.get("search_mode") or "literal"),
+ case_sensitive=bool(args.get("case_sensitive", False)),
+ context_lines=int(args.get("context_lines", 2)),
+ scan_limit=int(args.get("scan_limit", 250)), categories=args.get("categories"),
+ )
+ return {"success": True, "operation": operation, "trace": trace}
if operation == "assign_task":
if not args.get("title") or not args.get("mission"):
raise ValueError("assign_task requires title and mission")
diff --git a/server/services/agent_teams/handlers.py b/server/services/agent_teams/handlers.py
index bfc5990a..d9fbca49 100644
--- a/server/services/agent_teams/handlers.py
+++ b/server/services/agent_teams/handlers.py
@@ -179,6 +179,24 @@ async def handle_get_team_task(data: Dict[str, Any], websocket: WebSocket) -> Di
return {"task": task}
+@ws_handler("workflow_id", "team_lead_node_id", "task_id")
+async def handle_get_team_task_trace(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
+ """Read Temporal history through the task's trusted execution link."""
+ from services.team_task_trace import get_team_task_trace_service
+
+ trace = await get_team_task_trace_service().get_trace(
+ workflow_id=data["workflow_id"], team_lead_node_id=data["team_lead_node_id"],
+ execution_id=data.get("execution_id"), task_id=data["task_id"],
+ attempt=data.get("attempt"), cursor=data.get("cursor"),
+ limit=int(data.get("limit") or 50), detail=str(data.get("detail") or "summary"),
+ query=data.get("query"), search_mode=str(data.get("search_mode") or "literal"),
+ case_sensitive=bool(data.get("case_sensitive", False)),
+ context_lines=int(data.get("context_lines", 2)),
+ scan_limit=int(data.get("scan_limit", 250)), categories=data.get("categories"),
+ )
+ return {"trace": trace}
+
+
@ws_handler("workflow_id", "team_lead_node_id", "task_id", "operation")
async def handle_manage_team_task(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
from services.agent_team import get_agent_team_service
@@ -246,6 +264,7 @@ WS_HANDLERS: Dict[str, Any] = {
"get_team_tasks": handle_get_team_tasks,
"assign_team_task": handle_assign_team_task,
"get_team_task": handle_get_team_task,
+ "get_team_task_trace": handle_get_team_task_trace,
"manage_team_task": handle_manage_team_task,
"finish_team": handle_finish_team,
"send_team_message": handle_send_team_message,
@@ -263,6 +282,7 @@ __all__ = [
"handle_get_team",
"handle_get_team_messages",
"handle_get_team_status",
+ "handle_get_team_task_trace",
"handle_get_team_tasks",
"handle_send_team_message",
]
diff --git a/server/services/deployment/control.py b/server/services/deployment/control.py
new file mode 100644
index 00000000..1bf3a479
--- /dev/null
+++ b/server/services/deployment/control.py
@@ -0,0 +1,100 @@
+"""Durable, revision-guarded workflow deployment control plane."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import uuid
+from datetime import datetime, timezone
+from typing import Any, Dict, Optional
+
+from models.database import WorkflowControlExecution
+from sqlalchemy.exc import IntegrityError
+
+
+ACTIVE_STATES = {"starting", "running", "pausing", "paused", "resuming", "resetting"}
+
+
+def _graph_hash(nodes: list[dict], edges: list[dict]) -> str:
+ value = json.dumps({"nodes": nodes, "edges": edges}, sort_keys=True, separators=(",", ":"), default=str)
+ return hashlib.sha256(value.encode()).hexdigest()
+
+
+def serialize_control(control: Optional[WorkflowControlExecution]) -> Dict[str, Any]:
+ if control is None:
+ return {
+ "state": "never_started", "revision": 0, "generation": 0,
+ "can_start": True, "can_pause": False, "can_resume": False, "can_reset": False,
+ }
+ state = "ready" if control.status == "reset" else control.status
+ return {
+ "workflow_id": control.workflow_id,
+ "generation": control.generation,
+ "execution_id": control.execution_id,
+ "root_execution_id": control.root_execution_id,
+ "controller_workflow_id": control.controller_workflow_id,
+ "controller_run_id": control.controller_run_id,
+ "state": state,
+ "revision": control.revision,
+ "can_start": state == "ready",
+ "can_pause": state == "running",
+ "can_resume": state == "paused",
+ "can_reset": state != "ready",
+ "created_at": control.created_at.isoformat() if control.created_at else None,
+ "updated_at": control.updated_at.isoformat() if control.updated_at else None,
+ "terminal_reason": control.terminal_reason,
+ }
+
+
+class WorkflowControlService:
+ def __init__(self, database: Any):
+ self.database = database
+
+ async def get_status(self, workflow_id: str) -> Dict[str, Any]:
+ return serialize_control(await self.database.get_latest_workflow_control(workflow_id))
+
+ async def begin_generation(
+ self, *, workflow_id: str, nodes: list[dict], edges: list[dict], session_id: str,
+ idempotency_key: str, reset: bool = False,
+ ) -> tuple[WorkflowControlExecution, bool]:
+ existing = await self.database.get_workflow_control_by_idempotency_key(workflow_id, idempotency_key)
+ if existing:
+ return existing, False
+ latest = await self.database.get_latest_workflow_control(workflow_id)
+ if latest and latest.status != "reset" and not reset:
+ raise ValueError("workflow_already_started")
+ generation = (latest.generation + 1) if latest else 1
+ execution_id = f"{workflow_id}:g{generation}:{uuid.uuid4().hex[:12]}"
+ control = WorkflowControlExecution(
+ id=f"workflow-control:{workflow_id}:{generation}", workflow_id=workflow_id,
+ generation=generation, execution_id=execution_id, root_execution_id=execution_id,
+ controller_workflow_id=f"workflow-control-{workflow_id}-g{generation}",
+ session_id=session_id, graph_hash=_graph_hash(nodes, edges),
+ graph_snapshot={"nodes": nodes, "edges": edges}, idempotency_key=idempotency_key,
+ )
+ try:
+ return await self.database.create_workflow_control(control), True
+ except IntegrityError:
+ # A concurrent request may win either unique key. Idempotent
+ # duplicates return its generation; a distinct request conflicts.
+ winner = await self.database.get_workflow_control_by_idempotency_key(workflow_id, idempotency_key)
+ if winner is not None:
+ return winner, False
+ raise ValueError("workflow_control_generation_conflict")
+
+ async def transition(
+ self, control: WorkflowControlExecution, *, expected_revision: int, from_statuses: set[str],
+ status: str, values: Optional[Dict[str, Any]] = None,
+ ) -> WorkflowControlExecution:
+ updated = await self.database.transition_workflow_control(
+ control.id, expected_revision=expected_revision, from_statuses=from_statuses, status=status, values=values,
+ )
+ if updated is None:
+ raise ValueError("control_revision_conflict")
+ return updated
+
+ async def fail(self, control: WorkflowControlExecution, reason: str) -> WorkflowControlExecution:
+ return await self.transition(
+ control, expected_revision=control.revision, from_statuses={control.status}, status="failed",
+ values={"terminal_reason": reason, "completed_at": datetime.now(timezone.utc)},
+ )
diff --git a/server/services/deployment/handlers.py b/server/services/deployment/handlers.py
index 18c83aca..89f72203 100644
--- a/server/services/deployment/handlers.py
+++ b/server/services/deployment/handlers.py
@@ -22,12 +22,15 @@ from __future__ import annotations
import asyncio
import time
+import uuid
+from datetime import datetime, timezone
from typing import Any, Dict, Optional
from fastapi import WebSocket
from core.logging import get_logger
from services.ws_handler_registry import ws_handler
+from services.deployment.control import WorkflowControlService, serialize_control
# ``core.container`` and ``services.status_broadcaster`` are lazy-imported
# inside each handler body. This module is imported transitively via
@@ -370,12 +373,271 @@ async def handle_update_deployment_settings(
}
+def _control_service():
+ from core.container import container
+
+ return WorkflowControlService(container.database())
+
+
+async def _start_controller(control) -> Optional[str]:
+ """Start the durable Temporal controller when Temporal is available."""
+ from core.container import container
+ from temporalio.common import SearchAttributeKey, SearchAttributePair, TypedSearchAttributes
+
+ wrapper = container.temporal_client()
+ if wrapper is None or wrapper.client is None:
+ return None
+ handle = await wrapper.client.start_workflow(
+ "WorkflowControlWorkflow",
+ args=[{"generation": control.generation, "state": "running"}],
+ id=control.controller_workflow_id,
+ task_queue="machina-tasks",
+ search_attributes=TypedSearchAttributes([
+ SearchAttributePair(
+ SearchAttributeKey.for_keyword("EventWorkflowId"), control.workflow_id,
+ )
+ ]),
+ )
+ return getattr(handle, "result_run_id", None) or getattr(handle, "first_execution_run_id", None)
+
+
+async def _signal_controller(control, signal_name: str) -> None:
+ from core.container import container
+
+ wrapper = container.temporal_client()
+ if wrapper is not None and wrapper.client is not None and control.controller_workflow_id:
+ await wrapper.client.get_workflow_handle(
+ control.controller_workflow_id, run_id=control.controller_run_id
+ ).signal(signal_name)
+
+
+async def _signal_generation_workflows(workflow_id: str, signal_name: str) -> int:
+ """Best-effort cooperative fan-out to this deployment's live executions.
+
+ Visibility is discovery only; durable control state remains authoritative.
+ Every matching workflow receives an idempotent pause/resume flag mutation.
+ """
+ from core.container import container
+
+ wrapper = container.temporal_client()
+ if wrapper is None or wrapper.client is None:
+ return 0
+ query = (
+ f"EventWorkflowId='{workflow_id}' "
+ "AND ExecutionStatus='Running'"
+ )
+ signalled = 0
+ try:
+ async for execution in wrapper.client.list_workflows(query=query):
+ try:
+ await wrapper.client.get_workflow_handle(
+ execution.id, run_id=execution.run_id
+ ).signal(signal_name)
+ signalled += 1
+ except Exception as exc:
+ logger.warning(
+ "Workflow control signal failed",
+ workflow_id=workflow_id,
+ temporal_workflow_id=execution.id,
+ signal=signal_name,
+ error=str(exc),
+ )
+ except Exception as exc:
+ logger.warning(
+ "Workflow control visibility fan-out failed",
+ workflow_id=workflow_id,
+ signal=signal_name,
+ error=str(exc),
+ )
+ return signalled
+
+
+async def _terminate_generation_workflows(workflow_id: str) -> int:
+ """Immediately terminate every visible execution in one application tree."""
+ from core.container import container
+
+ wrapper = container.temporal_client()
+ if wrapper is None or wrapper.client is None:
+ return 0
+ terminated = 0
+ try:
+ async for execution in wrapper.client.list_workflows(
+ query=f"EventWorkflowId='{workflow_id}' AND ExecutionStatus='Running'"
+ ):
+ try:
+ await wrapper.client.get_workflow_handle(
+ execution.id, run_id=execution.run_id
+ ).terminate(reason="workflow_reset")
+ terminated += 1
+ except Exception as exc:
+ logger.warning(
+ "Workflow reset termination failed",
+ workflow_id=workflow_id,
+ temporal_workflow_id=execution.id,
+ error=str(exc),
+ )
+ except Exception as exc:
+ logger.warning(
+ "Workflow reset visibility scan failed",
+ workflow_id=workflow_id,
+ error=str(exc),
+ )
+ return terminated
+
+
+def _expected_revision(data: Dict[str, Any], control) -> int:
+ supplied = data.get("expected_revision")
+ return control.revision if supplied is None else int(supplied)
+
+
+async def _set_cron_pause(workflow_id: str, *, paused: bool) -> int:
+ from core.container import container
+ from services.temporal.schedules import set_cron_schedules_paused
+
+ wrapper = container.temporal_client()
+ if wrapper is None or wrapper.client is None:
+ return 0
+ return await set_cron_schedules_paused(wrapper.client, workflow_id, paused=paused)
+
+
+async def _with_runtime_counts(payload: Dict[str, Any], workflow_id: str) -> Dict[str, Any]:
+ from core.container import container
+
+ status = container.workflow_service().get_deployment_status(workflow_id)
+ return {
+ **payload,
+ "active_count": status.get("active_runs", 0),
+ "in_flight_count": status.get("active_runs", 0),
+ "queued_count": 0,
+ }
+
+
+@ws_handler("workflow_id")
+async def handle_get_workflow_control_status(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
+ return await _with_runtime_counts(await _control_service().get_status(data["workflow_id"]), data["workflow_id"])
+
+
+@ws_handler("workflow_id")
+async def handle_start_workflow(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
+ """Create generation one and retain deploy_workflow wire compatibility."""
+ workflow_id = data["workflow_id"]
+ key = data.get("idempotency_key") or f"start:{workflow_id}:{uuid.uuid4().hex}"
+ service = _control_service()
+ control, created = await service.begin_generation(
+ workflow_id=workflow_id, nodes=data.get("nodes", []), edges=data.get("edges", []),
+ session_id=data.get("session_id", "default"), idempotency_key=key,
+ )
+ if not created:
+ return await _with_runtime_counts({"success": True, "idempotent": True, **serialize_control(control)}, workflow_id)
+ try:
+ run_id = await _start_controller(control)
+ if run_id:
+ control = await service.transition(
+ control, expected_revision=control.revision, from_statuses={"starting"}, status="starting",
+ values={"controller_run_id": run_id},
+ )
+ deployed = await handle_deploy_workflow(data, websocket)
+ if not deployed.get("success"):
+ await service.fail(control, str(deployed.get("error", "deployment_failed")))
+ return deployed
+ control = await service.transition(control, expected_revision=control.revision, from_statuses={"starting"}, status="running")
+ return await _with_runtime_counts({"success": True, **serialize_control(control)}, workflow_id)
+ except Exception as exc:
+ await service.fail(control, str(exc))
+ raise
+
+
+@ws_handler("workflow_id")
+async def handle_pause_workflow(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
+ from core.container import container
+
+ workflow_id = data["workflow_id"]
+ service = _control_service()
+ control = await service.database.get_latest_workflow_control(workflow_id)
+ if control is None:
+ return {"success": False, "error": "workflow_never_started"}
+ control = await service.transition(
+ control, expected_revision=_expected_revision(data, control), from_statuses={"running"}, status="pausing"
+ )
+ container.workflow_service().pause_deployment(workflow_id)
+ await _signal_controller(control, "pause")
+ paused_schedules = await _set_cron_pause(workflow_id, paused=True)
+ paused_triggers = await container.workflow_service().update_trigger_pause_status(workflow_id, paused=True)
+ signalled = await _signal_generation_workflows(workflow_id, "pause")
+ control = await service.transition(control, expected_revision=control.revision, from_statuses={"pausing"}, status="paused")
+ return await _with_runtime_counts({
+ "success": True, "signalled_executions": signalled,
+ "paused_schedules": paused_schedules, "paused_triggers": paused_triggers,
+ **serialize_control(control),
+ }, workflow_id)
+
+
+@ws_handler("workflow_id")
+async def handle_resume_workflow(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
+ from core.container import container
+
+ workflow_id = data["workflow_id"]
+ service = _control_service()
+ control = await service.database.get_latest_workflow_control(workflow_id)
+ if control is None:
+ return {"success": False, "error": "workflow_never_started"}
+ control = await service.transition(
+ control, expected_revision=_expected_revision(data, control), from_statuses={"paused"}, status="resuming"
+ )
+ await _signal_controller(control, "resume")
+ resumed_schedules = await _set_cron_pause(workflow_id, paused=False)
+ signalled = await _signal_generation_workflows(workflow_id, "resume")
+ queued = await container.workflow_service().resume_deployment(workflow_id)
+ resumed_triggers = await container.workflow_service().update_trigger_pause_status(workflow_id, paused=False)
+ control = await service.transition(control, expected_revision=control.revision, from_statuses={"resuming"}, status="running")
+ return await _with_runtime_counts({
+ "success": True, "resumed_queued_events": queued, "signalled_executions": signalled,
+ "resumed_schedules": resumed_schedules, "resumed_triggers": resumed_triggers,
+ **serialize_control(control),
+ }, workflow_id)
+
+
+@ws_handler("workflow_id")
+async def handle_reset_workflow(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
+ workflow_id = data["workflow_id"]
+ service = _control_service()
+ current = await service.database.get_latest_workflow_control(workflow_id)
+ if current is not None:
+ current = await service.transition(
+ current, expected_revision=_expected_revision(data, current),
+ from_statuses={"starting", "running", "pausing", "paused", "resuming", "failed"}, status="resetting",
+ values={"terminal_reason": "workflow_reset", "completed_at": datetime.now(timezone.utc)},
+ )
+ try:
+ await _signal_controller(current, "reset")
+ finally:
+ terminated = await _terminate_generation_workflows(workflow_id)
+ await handle_cancel_deployment({"workflow_id": workflow_id}, websocket)
+ else:
+ terminated = 0
+ if current is None:
+ return await _with_runtime_counts(await service.get_status(workflow_id), workflow_id)
+ current = await service.transition(
+ current, expected_revision=current.revision, from_statuses={"resetting"}, status="reset",
+ values={"terminal_reason": "workflow_reset", "completed_at": datetime.now(timezone.utc)},
+ )
+ return await _with_runtime_counts(
+ {"success": True, "terminated_executions": terminated, **serialize_control(current)},
+ workflow_id,
+ )
+
+
WS_HANDLERS: Dict[str, Any] = {
"deploy_workflow": handle_deploy_workflow,
"cancel_deployment": handle_cancel_deployment,
"get_deployment_status": handle_get_deployment_status,
"get_workflow_lock": handle_get_workflow_lock,
"update_deployment_settings": handle_update_deployment_settings,
+ "start_workflow": handle_start_workflow,
+ "pause_workflow": handle_pause_workflow,
+ "resume_workflow": handle_resume_workflow,
+ "reset_workflow": handle_reset_workflow,
+ "get_workflow_control_status": handle_get_workflow_control_status,
}
diff --git a/server/services/deployment/manager.py b/server/services/deployment/manager.py
index 8b4065ff..5e2f8d8c 100644
--- a/server/services/deployment/manager.py
+++ b/server/services/deployment/manager.py
@@ -7,6 +7,7 @@ Implements n8n/Conductor pattern where:
"""
import asyncio
+import inspect
import json
import time
from datetime import datetime
@@ -15,6 +16,7 @@ from typing import Dict, Any, List, Optional, Callable, TYPE_CHECKING
from core.logging import get_logger
from constants import POLLING_TRIGGER_TYPES, TOOLKIT_NODE_TYPES, WORKFLOW_TRIGGER_TYPES
from services import event_waiter
+from services.events.envelope import equivalent_event_types
from .state import DeploymentState, TriggerInfo
from .triggers import TriggerManager
@@ -77,6 +79,8 @@ class DeploymentManager:
self._trigger_managers: Dict[str, TriggerManager] = {}
self._active_runs: Dict[str, Dict[str, asyncio.Task]] = {} # workflow_id -> {run_id: task}
self._run_counters: Dict[str, int] = {}
+ self._paused_workflows: set[str] = set()
+ self._paused_events: Dict[str, List[tuple[str, Dict[str, Any]]]] = {}
self._status_callbacks: Dict[str, Callable] = {}
self._main_loop: Optional[asyncio.AbstractEventLoop] = None
@@ -96,6 +100,47 @@ class DeploymentManager:
"""Get list of deployed workflow IDs."""
return [wid for wid, state in self._deployments.items() if state.is_running]
+ def pause(self, workflow_id: str) -> bool:
+ """Cooperatively stop admitting new runs for a deployment."""
+ if not self.is_workflow_deployed(workflow_id):
+ return False
+ self._paused_workflows.add(workflow_id)
+ return True
+
+ async def resume(self, workflow_id: str) -> int:
+ """Resume admission and drain paused trigger events in FIFO order."""
+ if workflow_id not in self._paused_workflows:
+ return 0
+ self._paused_workflows.discard(workflow_id)
+ queued = self._paused_events.pop(workflow_id, [])
+ for trigger_node_id, trigger_data in queued:
+ await self._spawn_run(trigger_node_id, trigger_data, workflow_id=workflow_id)
+ return len(queued)
+
+ async def update_trigger_pause_status(self, workflow_id: str, *, paused: bool) -> int:
+ """Reflect deployment admission state on every armed trigger node."""
+ state = self._deployments.get(workflow_id)
+ if state is None:
+ return 0
+ trigger_ids: list[str] = []
+ for node in state.nodes:
+ node_type = node.get("type", "")
+ if node_type == "start":
+ continue
+ if (
+ node_type in WORKFLOW_TRIGGER_TYPES
+ or event_waiter.get_trigger_config(node_type) is not None
+ or self._is_polling_trigger_class(node_type)
+ ):
+ trigger_ids.append(node["id"])
+ for node_id in trigger_ids:
+ await self._broadcaster.update_node_status(
+ node_id, "idle" if paused else "waiting",
+ {"paused": paused, "message": "Paused" if paused else "Waiting for events"},
+ workflow_id=workflow_id,
+ )
+ return len(trigger_ids)
+
# =========================================================================
# DEPLOYMENT LIFECYCLE
# =========================================================================
@@ -740,24 +785,44 @@ class DeploymentManager:
event_workflow_id_key = SearchAttributeKey.for_keyword("EventWorkflowId")
event_trigger_kind_key = SearchAttributeKey.for_keyword("EventTriggerKind")
- await wrapper.client.start_workflow(
- workflow_type_name,
- args=[listener_args],
- id=listener_id,
- task_queue="machina-tasks",
- id_conflict_policy=WorkflowIDConflictPolicy.USE_EXISTING,
- search_attributes=TypedSearchAttributes(
- [
- # MUST be the CloudEvents type the producer emits on
- # outgoing envelopes — dispatch.emit's Visibility query
- # substitutes ``event.type`` into the EventType filter.
- SearchAttributePair(event_type_key, cloudevent_type),
- SearchAttributePair(trigger_node_id_key, node_id),
- SearchAttributePair(event_workflow_id_key, workflow_id),
- SearchAttributePair(event_trigger_kind_key, trigger_kind),
- ]
- ),
- )
+ control_lookup = container.database().get_latest_workflow_control(workflow_id)
+ control = await control_lookup if inspect.isawaitable(control_lookup) else None
+ if control is not None and control.controller_workflow_id and control.status in {
+ "starting", "running", "pausing", "paused", "resuming",
+ }:
+ # The controller owns listeners as child workflows. This keeps
+ # trigger activity in the deployment's Temporal execution tree
+ # instead of presenting each listener as an unrelated root trace.
+ await wrapper.client.get_workflow_handle(
+ control.controller_workflow_id, run_id=control.controller_run_id,
+ ).signal("register_trigger", {
+ "listener_id": listener_id,
+ "workflow_type": workflow_type_name,
+ "listener_args": listener_args,
+ "event_type": cloudevent_type,
+ "event_types": list(equivalent_event_types(cloudevent_type)),
+ "trigger_node_id": node_id,
+ "workflow_id": workflow_id,
+ "trigger_kind": trigger_kind,
+ })
+ else:
+ # Compatibility path for legacy deployments without a durable
+ # workflow controller.
+ await wrapper.client.start_workflow(
+ workflow_type_name,
+ args=[listener_args],
+ id=listener_id,
+ task_queue="machina-tasks",
+ id_conflict_policy=WorkflowIDConflictPolicy.USE_EXISTING,
+ search_attributes=TypedSearchAttributes(
+ [
+ SearchAttributePair(event_type_key, cloudevent_type),
+ SearchAttributePair(trigger_node_id_key, node_id),
+ SearchAttributePair(event_workflow_id_key, workflow_id),
+ SearchAttributePair(event_trigger_kind_key, trigger_kind),
+ ]
+ ),
+ )
# Broadcast waiting status (parity with legacy path).
await self._broadcaster.update_node_status(
@@ -954,6 +1019,11 @@ class DeploymentManager:
if not workflow_id or not self.is_workflow_deployed(workflow_id):
return None
+ if workflow_id in self._paused_workflows:
+ self._paused_events.setdefault(workflow_id, []).append((trigger_node_id, trigger_data))
+ logger.info("Deployment event queued while paused", workflow_id=workflow_id, trigger_node_id=trigger_node_id)
+ return None
+
state = self._deployments[workflow_id]
# Check concurrent limit for this workflow
@@ -1083,8 +1153,6 @@ class DeploymentManager:
"""Get all downstream nodes from a trigger."""
downstream_ids = set()
node_types = {n["id"]: n.get("type", "") for n in nodes}
- nodes_with_inputs = {e.get("target") for e in edges if e.get("target")}
-
def collect(current_id: str):
for edge in edges:
if edge.get("source") != current_id:
diff --git a/server/services/events/dispatch.py b/server/services/events/dispatch.py
index b7c60f51..d7815a56 100644
--- a/server/services/events/dispatch.py
+++ b/server/services/events/dispatch.py
@@ -47,6 +47,7 @@ _DEFAULT_WIRE_ROUTING_KEY = "cloudevent"
# encoding; see https://docs.temporal.io/list-filter.
_RUNNING_CONSUMERS_QUERY = "EventType='{event_type}' AND ExecutionStatus='Running'"
_RUNNING_COMPAT_CONSUMERS_QUERY = "({event_type_clauses}) AND ExecutionStatus='Running'"
+_RUNNING_CONTROLLERS_QUERY = "WorkflowType='WorkflowControlWorkflow' AND ExecutionStatus='Running'"
# Signal handler name on consumer workflows. Plain identifier rather
# than CloudEvents type because Temporal Signal names are Python
@@ -126,6 +127,10 @@ async def _signal_running_consumers(event: WorkflowEvent) -> None:
query = _RUNNING_COMPAT_CONSUMERS_QUERY.format(event_type_clauses=clauses)
try:
+ # Controlled deployments keep trigger definitions in their controller
+ # history. Controllers filter the event against those definitions, so
+ # no TriggerListenerWorkflow/ PollingTriggerWorkflow execution exists.
+ query = f"({query}) OR ({_RUNNING_CONTROLLERS_QUERY})"
consumers = []
async for wf in client.list_workflows(query=query):
consumers.append(wf)
diff --git a/server/services/team_task_trace.py b/server/services/team_task_trace.py
new file mode 100644
index 00000000..3d0442a4
--- /dev/null
+++ b/server/services/team_task_trace.py
@@ -0,0 +1,255 @@
+"""Authorized, sanitized Temporal histories for durable team-task attempts."""
+
+from __future__ import annotations
+
+import base64
+import re
+import uuid
+from typing import Any, Dict, Iterable, Optional
+
+from temporalio.api.enums.v1 import EventType
+
+_SECRET = re.compile(
+ r"(?i)(api[_-]?key|authorization|bearer|password|secret|session[_-]?token)"
+ r"(\s*[:=]\s*|\s+)[^\s,;]+"
+)
+_FAILURE_EVENT_NAMES = {"ACTIVITY_TASK_FAILED", "CHILD_WORKFLOW_EXECUTION_FAILED",
+ "WORKFLOW_EXECUTION_FAILED", "WORKFLOW_TASK_FAILED",
+ "ACTIVITY_TASK_TIMED_OUT", "CHILD_WORKFLOW_EXECUTION_TIMED_OUT"}
+
+
+def _redact(value: Any, limit: int = 2000) -> str:
+ text = str(value or "")[:limit]
+ return _SECRET.sub(lambda match: f"{match.group(1)}{match.group(2)}[REDACTED]", text)
+
+
+def _cursor_decode(cursor: Optional[str]) -> Optional[bytes]:
+ if not cursor:
+ return None
+ try:
+ return base64.urlsafe_b64decode(cursor.encode("ascii"))
+ except Exception as exc:
+ raise ValueError("Invalid trace cursor") from exc
+
+
+def _cursor_encode(token: Optional[bytes]) -> Optional[str]:
+ return base64.urlsafe_b64encode(token).decode("ascii") if token else None
+
+
+def _event_name(event: Any) -> str:
+ try:
+ return EventType.Name(event.event_type).removeprefix("EVENT_TYPE_")
+ except Exception:
+ return "UNKNOWN"
+
+
+def _normalize_event(event: Any) -> Dict[str, Any]:
+ name = _event_name(event)
+ result: Dict[str, Any] = {
+ "event_id": int(getattr(event, "event_id", 0)),
+ "type": name.lower(),
+ "category": (
+ "failure" if name in _FAILURE_EVENT_NAMES
+ else "activity" if "ACTIVITY" in name
+ else "child" if "CHILD_WORKFLOW" in name
+ else "signal" if "SIGNAL" in name
+ else "timer" if "TIMER" in name
+ else "workflow"
+ ),
+ }
+ event_time = getattr(event, "event_time", None)
+ if event_time:
+ try:
+ result["timestamp"] = event_time.ToDatetime().isoformat()
+ except Exception:
+ pass
+ try:
+ attr_name = event.WhichOneof("attributes")
+ attrs = getattr(event, attr_name) if attr_name else None
+ except Exception:
+ attrs = None
+ if attrs is not None:
+ activity_type = getattr(getattr(attrs, "activity_type", None), "name", "")
+ workflow_type = getattr(getattr(attrs, "workflow_type", None), "name", "")
+ execution = getattr(attrs, "workflow_execution", None)
+ failure = getattr(attrs, "failure", None)
+ if activity_type:
+ result["activity_type"] = _redact(activity_type, 255)
+ if workflow_type:
+ result["workflow_type"] = _redact(workflow_type, 255)
+ if execution and getattr(execution, "workflow_id", ""):
+ result["child_workflow_id"] = _redact(execution.workflow_id, 500)
+ result["child_run_id"] = _redact(getattr(execution, "run_id", ""), 255)
+ attempt = getattr(attrs, "attempt", 0)
+ if attempt:
+ result["attempt"] = int(attempt)
+ retry_state = getattr(attrs, "retry_state", 0)
+ if retry_state:
+ result["retry_state"] = int(retry_state)
+ if failure and getattr(failure, "message", ""):
+ result["failure"] = {"message": _redact(failure.message)}
+ return result
+
+
+def _search_text(event: Dict[str, Any]) -> str:
+ """Build a bounded searchable projection without exposing raw payloads."""
+ values: list[str] = []
+
+ def collect(value: Any) -> None:
+ if isinstance(value, dict):
+ for nested in value.values():
+ collect(nested)
+ elif isinstance(value, (str, int)):
+ values.append(str(value))
+
+ collect(event)
+ return " ".join(values)[:5000]
+
+
+def _matches_search(
+ event: Dict[str, Any], query: str, *, mode: str, case_sensitive: bool,
+) -> bool:
+ haystack = _search_text(event)
+ needle = query.strip()
+ if not case_sensitive:
+ haystack, needle = haystack.casefold(), needle.casefold()
+ if mode == "literal":
+ return needle in haystack
+ terms = [term for term in needle.split() if term]
+ return bool(terms) and (all(term in haystack for term in terms) if mode == "all_terms"
+ else any(term in haystack for term in terms))
+
+
+def _contextual_events(
+ events: list[Dict[str, Any]], matched_indexes: Iterable[int], context_lines: int,
+) -> list[Dict[str, Any]]:
+ matched = set(matched_indexes)
+ selected: set[int] = set()
+ for index in matched:
+ selected.update(range(max(0, index - context_lines), min(len(events), index + context_lines + 1)))
+ return [{**events[index], "match": index in matched} for index in sorted(selected)]
+
+
+class TeamTaskTraceService:
+ def __init__(self, team_service: Any, temporal_wrapper: Any):
+ self.team_service = team_service
+ self.temporal_wrapper = temporal_wrapper
+
+ async def get_trace(
+ self, *, workflow_id: str, team_lead_node_id: str,
+ execution_id: Optional[str], task_id: str, attempt: Optional[int] = None,
+ cursor: Optional[str] = None, limit: int = 50, detail: str = "summary",
+ query: Optional[str] = None, search_mode: str = "literal",
+ case_sensitive: bool = False, context_lines: int = 2, scan_limit: int = 250,
+ categories: Optional[list[str]] = None,
+ ) -> Dict[str, Any]:
+ if detail not in {"summary", "failures", "timeline", "search"}:
+ raise ValueError("detail must be summary, failures, timeline, or search")
+ if detail == "search" and not str(query or "").strip():
+ raise ValueError("trace search requires a non-empty query")
+ if search_mode not in {"literal", "all_terms", "any_terms"}:
+ raise ValueError("search_mode must be literal, all_terms, or any_terms")
+ if query is not None and len(query) > 200:
+ raise ValueError("trace search query is limited to 200 characters")
+ limit = max(1, min(int(limit), 100))
+ context_lines = max(0, min(int(context_lines), 5))
+ scan_limit = max(1, min(int(scan_limit), 500))
+ allowed_categories = {"failure", "activity", "child", "signal", "timer", "workflow"}
+ category_filter = set(categories or [])
+ if not category_filter.issubset(allowed_categories):
+ raise ValueError("invalid trace category filter")
+ task = await self.team_service.get_durable_task(
+ workflow_id=workflow_id, team_lead_node_id=team_lead_node_id,
+ execution_id=execution_id, task_id=task_id,
+ )
+ attempts = task.get("attempts") or []
+ attempt_number = task.get("current_attempt", 0) if attempt is None else int(attempt)
+ selected = next((item for item in attempts if item["attempt_number"] == attempt_number), None)
+ if selected is None and attempt_number == task.get("current_attempt", 0):
+ selected = task
+ workflow_identity = selected.get("child_workflow_id") or selected.get("runner_workflow_id")
+ run_identity = selected.get("child_run_id") or selected.get("runner_run_id")
+ base = {
+ "task_id": task_id, "attempt": attempt_number, "detail": detail,
+ "trace_id": task.get("trace_id"),
+ "execution": {
+ "workflow_id": workflow_identity, "run_id": run_identity,
+ "runner_workflow_id": selected.get("runner_workflow_id"),
+ "runner_run_id": selected.get("runner_run_id"),
+ "child_workflow_id": selected.get("child_workflow_id"),
+ "child_run_id": selected.get("child_run_id"),
+ "parent_workflow_id": selected.get("parent_workflow_id"),
+ "parent_run_id": selected.get("parent_run_id"),
+ },
+ }
+ await self.team_service.database.add_agent_message(
+ team_id=task["team_id"], from_agent=team_lead_node_id,
+ to_agent=team_lead_node_id, message_type="trace_inspection",
+ content=f"Inspected task {task_id} attempt {attempt_number} ({detail})",
+ event_id=f"trace-inspection:{uuid.uuid4().hex}",
+ extra_data={"task_id": task_id, "attempt": attempt_number, "detail": detail,
+ "search": detail == "search", "query_length": len(query or "")},
+ )
+ if not workflow_identity:
+ return {**base, "status": "execution_not_registered", "events": [], "next_cursor": None}
+ client = self.temporal_wrapper.client if self.temporal_wrapper else None
+ if client is None:
+ return {**base, "status": "temporal_unavailable", "events": [], "next_cursor": None}
+ try:
+ handle = client.get_workflow_handle(workflow_identity, run_id=run_identity)
+ page_size = min(100, scan_limit if detail == "search" else limit)
+ iterator = handle.fetch_history_events(
+ page_size=page_size, next_page_token=_cursor_decode(cursor), skip_archival=False,
+ )
+ events: list[Dict[str, Any]] = []
+ matched_indexes: list[int] = []
+ scan_cap = scan_limit if detail == "search" else limit
+ scanned = 0
+ while scanned < scan_cap:
+ remaining = scan_cap - scanned
+ await iterator.fetch_next_page(page_size=min(100, remaining))
+ raw_page = list(iterator.current_page or [])
+ scanned += len(raw_page)
+ page = [_normalize_event(event) for event in raw_page]
+ if category_filter:
+ page = [event for event in page if event["category"] in category_filter]
+ start = len(events)
+ events.extend(page)
+ if detail == "search":
+ matched_indexes.extend(
+ start + index for index, event in enumerate(page)
+ if _matches_search(event, str(query), mode=search_mode, case_sensitive=case_sensitive)
+ )
+ if len(matched_indexes) >= limit:
+ break
+ else:
+ break
+ if not iterator.next_page_token or not raw_page:
+ break
+ if detail == "failures":
+ events = [event for event in events if event["category"] == "failure"]
+ elif detail == "summary":
+ events = [event for event in events if event["category"] in {"failure", "activity", "child"}]
+ elif detail == "search":
+ events = _contextual_events(events, matched_indexes, context_lines)
+ base["search"] = {
+ "mode": search_mode, "case_sensitive": case_sensitive,
+ "context_lines": context_lines, "scanned_events": scanned,
+ "matched_events": len(matched_indexes),
+ }
+ return {**base, "status": "available", "events": events,
+ "next_cursor": _cursor_encode(iterator.next_page_token)}
+ except ValueError:
+ raise
+ except Exception as exc:
+ name = type(exc).__name__.lower()
+ status = "retention_expired" if "notfound" in name or "not found" in str(exc).lower() else "temporal_unavailable"
+ return {**base, "status": status, "events": [], "next_cursor": None,
+ "error": _redact(exc, 500)}
+
+
+def get_team_task_trace_service() -> TeamTaskTraceService:
+ from core.container import container
+ from services.agent_team import get_agent_team_service
+
+ return TeamTaskTraceService(get_agent_team_service(), container.temporal_client())
diff --git a/server/services/temporal/agent_activities.py b/server/services/temporal/agent_activities.py
index e65cfa95..617cb6c9 100644
--- a/server/services/temporal/agent_activities.py
+++ b/server/services/temporal/agent_activities.py
@@ -1228,6 +1228,33 @@ async def finish_agent_delegation(payload: Dict[str, Any]) -> Dict[str, Any]:
return {"team_id": team_id, "team_task_id": task_id, "status": target_status}
[email protected](name="agent.register_task_execution.v1")
+async def register_task_execution(payload: Dict[str, Any]) -> Dict[str, Any]:
+ """Persist actual runner/child Temporal identities for trace inspection."""
+ from services.agent_team import get_agent_team_service
+
+ team_id = str(payload.get("team_id") or "")
+ task_id = str(payload.get("team_task_id") or "")
+ if not team_id or not task_id:
+ raise ValueError("team_id and team_task_id are required")
+ task = await get_agent_team_service().database.get_durable_team_task(team_id, task_id)
+ if not task:
+ raise ValueError("Delegated team task does not exist")
+ registered = await get_agent_team_service().database.register_team_task_execution(
+ team_id=team_id, task_id=task_id,
+ attempt_number=int(payload.get("attempt_number", task.get("current_attempt", 0))),
+ runner_workflow_id=payload.get("runner_workflow_id"),
+ runner_run_id=payload.get("runner_run_id"),
+ child_workflow_id=payload.get("child_workflow_id"),
+ child_run_id=payload.get("child_run_id"),
+ parent_workflow_id=payload.get("parent_workflow_id"),
+ parent_run_id=payload.get("parent_run_id"),
+ )
+ if not registered:
+ raise RuntimeError("Failed to register delegated Temporal execution")
+ return {"team_id": team_id, "team_task_id": task_id, "registered": True}
+
+
@activity.defn(name="agent.finalize_team.v1")
async def finalize_agent_team(payload: Dict[str, Any]) -> Dict[str, Any]:
"""Finalize a lead's team after all delegated tasks become terminal."""
@@ -1274,6 +1301,7 @@ def collect_agent_activities() -> List[Any]:
queue_agent_delegation,
acquire_subagent_permit,
release_subagent_permit,
+ register_task_execution,
finish_agent_delegation,
finalize_agent_team,
]
diff --git a/server/services/temporal/agent_workflow.py b/server/services/temporal/agent_workflow.py
index b3acbf17..b0bc7b81 100644
--- a/server/services/temporal/agent_workflow.py
+++ b/server/services/temporal/agent_workflow.py
@@ -198,6 +198,21 @@ def _duplicate_visible_tool_name_error(tools: List[Dict[str, Any]]) -> Optional[
class DelegatedTaskWorkflow:
"""Own one queued delegation after the lead returns to its caller."""
+ def __init__(self) -> None:
+ self._control_paused = False
+
+ @workflow.signal
+ async def pause(self) -> None:
+ self._control_paused = True
+
+ @workflow.signal
+ async def resume(self) -> None:
+ self._control_paused = False
+
+ async def _wait_until_resumed(self) -> None:
+ if self._control_paused:
+ await workflow.wait_condition(lambda: not self._control_paused)
+
@workflow.run
async def run(self, request: Dict[str, Any]) -> Dict[str, Any]:
lifecycle = request["lifecycle"]
@@ -206,6 +221,16 @@ class DelegatedTaskWorkflow:
acquired = False
began = False
try:
+ await self._wait_until_resumed()
+ info = workflow.info()
+ await workflow.execute_activity(
+ "agent.register_task_execution.v1",
+ args=[{**lifecycle, "runner_workflow_id": info.workflow_id,
+ "runner_run_id": info.run_id,
+ "child_workflow_id": request["child_workflow_id"]}],
+ start_to_close_timeout=PERSIST_TURN_TIMEOUT,
+ retry_policy=AGENT_ACTIVITY_RETRY,
+ )
await workflow.execute_activity(
"agent.acquire_subagent_permit.v1",
args=[{"root_execution_id": root_id, "permit_id": task_id,
@@ -221,11 +246,22 @@ class DelegatedTaskWorkflow:
retry_policy=AGENT_ACTIVITY_RETRY,
)
began = True
- result = await workflow.execute_child_workflow(
+ await self._wait_until_resumed()
+ child_handle = await workflow.start_child_workflow(
"AgentWorkflow", args=[request["child_context"]],
id=request["child_workflow_id"],
execution_timeout=timedelta(hours=1), run_timeout=timedelta(hours=1),
)
+ await workflow.execute_activity(
+ "agent.register_task_execution.v1",
+ args=[{**lifecycle, "runner_workflow_id": info.workflow_id,
+ "runner_run_id": info.run_id,
+ "child_workflow_id": child_handle.id,
+ "child_run_id": child_handle.first_execution_run_id}],
+ start_to_close_timeout=PERSIST_TURN_TIMEOUT,
+ retry_policy=AGENT_ACTIVITY_RETRY,
+ )
+ result = await child_handle
succeeded = bool(result.get("success", True)) if isinstance(result, dict) else True
_response, summary = _normalise_delegated_result(result)
return await workflow.execute_activity(
@@ -270,6 +306,21 @@ class AgentWorkflow:
as F4.A per-type activities.
"""
+ def __init__(self) -> None:
+ self._control_paused = False
+
+ @workflow.signal
+ async def pause(self) -> None:
+ self._control_paused = True
+
+ @workflow.signal
+ async def resume(self) -> None:
+ self._control_paused = False
+
+ async def _wait_until_resumed(self) -> None:
+ if self._control_paused:
+ await workflow.wait_condition(lambda: not self._control_paused)
+
@workflow.run
async def run(self, context: Dict[str, Any]) -> Dict[str, Any]:
"""Run the agent loop.
@@ -471,6 +522,7 @@ class AgentWorkflow:
# ---- Main loop --------------------------------------------------
for iteration in range(max_iterations):
+ await self._wait_until_resumed()
workflow.logger.info(f"AgentWorkflow iteration {iteration} " f"(messages={len(messages)} tools={len(tools)})")
# CloudEvents-shaped agent_progress per LLM turn. Mirrors the
@@ -682,6 +734,8 @@ class AgentWorkflow:
),
"workflow_id": payload.get("workflow_id"),
"parent_agent_workflow_id": workflow.info().workflow_id,
+ "parent_workflow_id": workflow.info().workflow_id,
+ "parent_run_id": workflow.info().run_id,
"task": task,
"root_execution_id": root_execution_id,
"delegation_depth": delegation_depth + 1,
@@ -764,6 +818,17 @@ class AgentWorkflow:
execution_timeout=timedelta(hours=1),
run_timeout=timedelta(hours=1),
)
+ if team_id:
+ child_handle = delegation_handles[call_index]
+ await workflow.execute_activity(
+ "agent.register_task_execution.v1",
+ args=[{**lifecycle_payload,
+ "child_workflow_id": child_handle.id,
+ "child_run_id": child_handle.first_execution_run_id}],
+ activity_id=f"register-delegation-{iteration + 1}-{call_index + 1}",
+ start_to_close_timeout=PERSIST_TURN_TIMEOUT,
+ retry_policy=AGENT_ACTIVITY_RETRY,
+ )
except BaseException:
if team_id and call_index in delegation_permits:
await workflow.execute_activity(
@@ -844,6 +909,8 @@ class AgentWorkflow:
),
"workflow_id": payload.get("workflow_id"),
"parent_agent_workflow_id": workflow.info().workflow_id,
+ "parent_workflow_id": workflow.info().workflow_id,
+ "parent_run_id": workflow.info().run_id,
"task": mission,
"root_execution_id": root_execution_id,
"delegation_depth": delegation_depth + 1,
diff --git a/server/services/temporal/client.py b/server/services/temporal/client.py
index 3aa6c688..7107c94e 100644
--- a/server/services/temporal/client.py
+++ b/server/services/temporal/client.py
@@ -8,6 +8,7 @@ from datetime import timedelta
from typing import Optional
from temporalio.api.workflowservice.v1 import DescribeNamespaceRequest
from temporalio.client import Client
+from temporalio.contrib.opentelemetry import TracingInterceptor
from temporalio.runtime import LoggingConfig, Runtime, TelemetryConfig
from core.config import Settings
@@ -91,6 +92,7 @@ class TemporalClientWrapper:
self.server_address,
namespace=self.namespace,
runtime=self._runtime,
+ interceptors=[TracingInterceptor()],
)
# Verify namespace is ready (gRPC port may accept connections
# before the server finishes registering namespaces)
@@ -183,7 +185,7 @@ class TemporalClientWrapper:
try:
count = 0
async for wf in self._client.list_workflows(
- "ExecutionStatus = 'Running'",
+ "ExecutionStatus = 'Running' AND WorkflowType != 'WorkflowControlWorkflow'",
):
try:
handle = self._client.get_workflow_handle(
diff --git a/server/services/temporal/executor.py b/server/services/temporal/executor.py
index edef7d92..a806529e 100644
--- a/server/services/temporal/executor.py
+++ b/server/services/temporal/executor.py
@@ -10,6 +10,11 @@ from datetime import datetime
from typing import Any, Dict, List, Optional, Callable
from temporalio.client import Client
+from temporalio.common import (
+ SearchAttributeKey,
+ SearchAttributePair,
+ TypedSearchAttributes,
+)
from core.logging import get_logger
from .workflow import MachinaWorkflow
@@ -97,6 +102,14 @@ class TemporalExecutor:
},
id=execution_id,
task_queue=self.task_queue,
+ search_attributes=TypedSearchAttributes(
+ [
+ SearchAttributePair(
+ SearchAttributeKey.for_keyword("EventWorkflowId"),
+ workflow_id,
+ )
+ ]
+ ),
)
execution_time = time.time() - start_time
diff --git a/server/services/temporal/polling_trigger_workflow.py b/server/services/temporal/polling_trigger_workflow.py
index f2203a52..64961c93 100644
--- a/server/services/temporal/polling_trigger_workflow.py
+++ b/server/services/temporal/polling_trigger_workflow.py
@@ -74,6 +74,15 @@ class PollingTriggerWorkflow:
def __init__(self) -> None:
self._seen_event_ids: Set[str] = set()
self._processed_count: int = 0
+ self._control_paused = False
+
+ @workflow.signal
+ async def pause(self) -> None:
+ self._control_paused = True
+
+ @workflow.signal
+ async def resume(self) -> None:
+ self._control_paused = False
@workflow.run
async def run(self, listener_data: Dict[str, Any]) -> Dict[str, Any]:
@@ -118,6 +127,8 @@ class PollingTriggerWorkflow:
)
while True:
+ if self._control_paused:
+ await workflow.wait_condition(lambda: not self._control_paused)
if is_baseline:
# Establish seen baseline immediately on first run so we
# don't re-emit items the user has had since before deploy.
diff --git a/server/services/temporal/schedules.py b/server/services/temporal/schedules.py
index 82cd2f66..cdcedf4a 100644
--- a/server/services/temporal/schedules.py
+++ b/server/services/temporal/schedules.py
@@ -232,8 +232,38 @@ async def delete_cron_schedules_for_deployment(
return deleted
+async def set_cron_schedules_paused(
+ client: Client, deployment_workflow_id: str, *, paused: bool,
+) -> int:
+ """Pause or unpause every cron Schedule owned by one deployment."""
+ query = f"EventWorkflowId='{deployment_workflow_id}' AND EventTriggerKind='cron'"
+ changed = 0
+ try:
+ iterator = await client.list_schedules(query=query)
+ async for desc in iterator:
+ try:
+ handle = client.get_schedule_handle(desc.id)
+ if paused:
+ await handle.pause(note="OpenCompany workflow paused")
+ else:
+ await handle.unpause(note="OpenCompany workflow resumed")
+ changed += 1
+ except Exception as exc: # noqa: BLE001
+ logger.warning(
+ f"Failed to {'pause' if paused else 'resume'} cron Schedule {desc.id!r}: {exc}",
+ deployment_workflow_id=deployment_workflow_id,
+ )
+ except Exception as exc: # noqa: BLE001
+ logger.warning(
+ f"Cron Schedule pause sweep failed: {exc}",
+ deployment_workflow_id=deployment_workflow_id,
+ )
+ return changed
+
+
__all__ = [
"cron_schedule_id",
"create_cron_schedule",
"delete_cron_schedules_for_deployment",
+ "set_cron_schedules_paused",
]
diff --git a/server/services/temporal/trigger_listener_workflow.py b/server/services/temporal/trigger_listener_workflow.py
index a25f9304..d0375bfc 100644
--- a/server/services/temporal/trigger_listener_workflow.py
+++ b/server/services/temporal/trigger_listener_workflow.py
@@ -61,6 +61,15 @@ class TriggerListenerWorkflow:
self._seen_event_ids: Set[str] = set()
self._matched_events: List[Dict[str, Any]] = []
self._processed_count: int = 0
+ self._control_paused = False
+
+ @workflow.signal
+ async def pause(self) -> None:
+ self._control_paused = True
+
+ @workflow.signal
+ async def resume(self) -> None:
+ self._control_paused = False
@workflow.signal
async def on_event(self, event_payload: Dict[str, Any]) -> None:
@@ -109,7 +118,9 @@ class TriggerListenerWorkflow:
)
while True:
- await workflow.wait_condition(lambda: bool(self._matched_events))
+ await workflow.wait_condition(
+ lambda: bool(self._matched_events) and not self._control_paused
+ )
event = self._matched_events.pop(0) if self._matched_events else None
if event is None:
continue
@@ -133,6 +144,7 @@ class TriggerListenerWorkflow:
self,
event: Dict[str, Any],
listener_data: Dict[str, Any],
+ admission_check=None,
) -> None:
"""Build the filtered downstream graph + start a child MachinaWorkflow.
@@ -213,6 +225,9 @@ class TriggerListenerWorkflow:
event_type=event.get("type", ""),
)
+ if admission_check is not None:
+ await admission_check()
+
await workflow.start_child_workflow(
"MachinaWorkflow",
args=[
diff --git a/server/services/temporal/worker.py b/server/services/temporal/worker.py
index 6c260baa..e275f282 100644
--- a/server/services/temporal/worker.py
+++ b/server/services/temporal/worker.py
@@ -21,6 +21,7 @@ from typing import Optional
import aiohttp
from opentelemetry import trace
+from temporalio.contrib.opentelemetry import TracingInterceptor
from temporalio.client import Client
from temporalio.runtime import LoggingConfig, Runtime, TelemetryConfig
from temporalio.worker import PollerBehaviorAutoscaling, Worker
@@ -30,6 +31,7 @@ from ._interceptors import ObservabilityWorkerInterceptor
from .workflow import MachinaWorkflow
from .trigger_listener_workflow import TriggerListenerWorkflow
from .polling_trigger_workflow import PollingTriggerWorkflow
+from .workflow_control_workflow import WorkflowControlWorkflow
from .plugin_registry import temporal_plugins
from .activities import (
NodeExecutionActivities,
@@ -219,6 +221,7 @@ class TemporalWorkerManager:
DelegatedTaskWorkflow,
TriggerListenerWorkflow,
PollingTriggerWorkflow,
+ WorkflowControlWorkflow,
],
activities=[
self._activities.execute_node_activity,
@@ -234,7 +237,7 @@ class TemporalWorkerManager:
max_concurrent_workflow_tasks=10,
graceful_shutdown_timeout=_graceful_shutdown_timeout(),
identity=_worker_identity(self.task_queue),
- interceptors=[ObservabilityWorkerInterceptor()],
+ interceptors=[TracingInterceptor(), ObservabilityWorkerInterceptor()],
# Wave 18.2: sticky cache sized by deployment mode so
# cached workflows skip Event-History replay without
# blowing laptop RAM.
@@ -494,7 +497,7 @@ class TemporalWorkerPool:
activities=activities,
graceful_shutdown_timeout=_graceful_shutdown_timeout(),
identity=_worker_identity(queue),
- interceptors=[ObservabilityWorkerInterceptor()],
+ interceptors=[TracingInterceptor(), ObservabilityWorkerInterceptor()],
# Wave 18.1: per-queue activities/second ceiling.
max_activities_per_second=rate_limit,
# Wave 18.3: activity-only workers need a small
@@ -584,7 +587,12 @@ async def run_standalone_worker(
for attempt in range(1, 6):
try:
logger.info(f"Connecting to Temporal server (attempt {attempt}/5)")
- client = await Client.connect(server_address, namespace=namespace, runtime=runtime)
+ client = await Client.connect(
+ server_address,
+ namespace=namespace,
+ runtime=runtime,
+ interceptors=[TracingInterceptor()],
+ )
logger.info("Connected to Temporal server")
break
except Exception as e:
@@ -614,6 +622,7 @@ async def run_standalone_worker(
MachinaWorkflow,
TriggerListenerWorkflow,
PollingTriggerWorkflow,
+ WorkflowControlWorkflow,
],
activities=[
activities.execute_node_activity,
@@ -624,6 +633,7 @@ async def run_standalone_worker(
max_concurrent_activities=pool_size,
max_concurrent_workflow_tasks=10,
graceful_shutdown_timeout=_graceful_shutdown_timeout(),
+ interceptors=[TracingInterceptor(), ObservabilityWorkerInterceptor()],
)
logger.info("Worker running. Press Ctrl+C to stop.")
@@ -667,6 +677,7 @@ async def create_worker(
MachinaWorkflow,
TriggerListenerWorkflow,
PollingTriggerWorkflow,
+ WorkflowControlWorkflow,
],
activities=[
activities.execute_node_activity,
@@ -677,6 +688,7 @@ async def create_worker(
max_concurrent_activities=100,
max_concurrent_workflow_tasks=10,
graceful_shutdown_timeout=_graceful_shutdown_timeout(),
+ interceptors=[TracingInterceptor(), ObservabilityWorkerInterceptor()],
)
diff --git a/server/services/temporal/workflow.py b/server/services/temporal/workflow.py
index 6b2e3d01..67820502 100644
--- a/server/services/temporal/workflow.py
+++ b/server/services/temporal/workflow.py
@@ -130,6 +130,16 @@ class MachinaWorkflow:
# flag is on AND a producer signals this workflow.
self._seen_event_ids: Set[str] = set()
self._matched_events: List[Dict[str, Any]] = []
+ self._control_paused = False
+
+ @workflow.signal
+ async def pause(self) -> None:
+ """Cooperatively gate new node/activity scheduling."""
+ self._control_paused = True
+
+ @workflow.signal
+ async def resume(self) -> None:
+ self._control_paused = False
@workflow.signal
async def on_event(self, event_payload: Dict[str, Any]) -> None:
@@ -317,6 +327,11 @@ class MachinaWorkflow:
loop_count = 0
while True:
loop_count += 1
+ # Avoid emitting a redundant wait command on the normal running
+ # path. Once paused, the condition is durable and replay-safe;
+ # the resume signal flips the flag and releases scheduling.
+ if self._control_paused:
+ await workflow.wait_condition(lambda: not self._control_paused)
# Find ready nodes (all deps completed, not running/completed)
ready = self._find_ready_nodes(deps, completed, running, node_map)
workflow.logger.debug(f"Loop {loop_count}: ready={len(ready)}, running={len(running)}, completed={len(completed)}")
@@ -598,7 +613,6 @@ class MachinaWorkflow:
config_ids = set()
for edge in edges:
- handle = edge.get("targetHandle", "")
source_id = edge.get("source")
# Edges to config handles mean source is a config node
diff --git a/server/services/temporal/workflow_control_workflow.py b/server/services/temporal/workflow_control_workflow.py
new file mode 100644
index 00000000..77d6d424
--- /dev/null
+++ b/server/services/temporal/workflow_control_workflow.py
@@ -0,0 +1,169 @@
+"""Long-lived deployment controller and trigger hub for one generation."""
+
+import asyncio
+from datetime import timedelta
+from typing import Any, Dict
+
+from temporalio import workflow
+
+from services.temporal._retry_policies import DEFAULT_ACTIVITY_RETRY
+
+
[email protected](name="WorkflowControlWorkflow", sandboxed=False)
+class WorkflowControlWorkflow:
+ """Own control state and trigger scheduling without listener workflows.
+
+ Trigger definitions and inbound events are recorded as signals in this
+ workflow's own history. Only an actual triggered graph run becomes a child
+ workflow, so Temporal's workflow list has no per-trigger listener rows.
+ """
+
+ def __init__(self) -> None:
+ self._state = "running"
+ self._revision = 0
+ self._closed = False
+ self._triggers: dict[str, Dict[str, Any]] = {}
+ self._events: list[tuple[str, Dict[str, Any]]] = []
+ self._seen_event_ids: set[str] = set()
+ self._poll_tasks: dict[str, asyncio.Task] = {}
+
+ @workflow.signal
+ async def pause(self) -> None:
+ if self._state == "running":
+ self._state = "paused"
+ self._revision += 1
+
+ @workflow.signal
+ async def resume(self) -> None:
+ if self._state == "paused":
+ self._state = "running"
+ self._revision += 1
+
+ @workflow.signal
+ async def reset(self) -> None:
+ self._state = "resetting"
+ self._revision += 1
+ self._closed = True
+ for task in self._poll_tasks.values():
+ task.cancel()
+
+ @workflow.signal
+ async def register_trigger(self, spec: Dict[str, Any]) -> None:
+ listener_id = str(spec["listener_id"])
+ if listener_id in self._triggers:
+ return
+ self._triggers[listener_id] = spec
+ if spec["workflow_type"] == "PollingTriggerWorkflow":
+ self._poll_tasks[listener_id] = asyncio.create_task(self._poll_trigger(listener_id, spec))
+
+ @workflow.signal
+ async def on_event(self, event: Dict[str, Any]) -> None:
+ event_id = str(event.get("id") or "")
+ event_type = str(event.get("type") or "")
+ if not event_id or not event_type:
+ return
+ for listener_id, spec in self._triggers.items():
+ if spec["workflow_type"] == "PollingTriggerWorkflow":
+ continue
+ if event_type not in set(spec.get("event_types") or [spec.get("event_type")]):
+ continue
+ dedup_key = f"{listener_id}:{event_id}"
+ if dedup_key not in self._seen_event_ids:
+ self._seen_event_ids.add(dedup_key)
+ self._events.append((listener_id, event))
+
+ @workflow.query
+ def status(self) -> Dict[str, Any]:
+ return {
+ "state": self._state, "revision": self._revision,
+ "triggers": {key: value["trigger_node_id"] for key, value in self._triggers.items()},
+ "queued_events": len(self._events),
+ }
+
+ @workflow.run
+ async def run(self, control_data: Dict[str, Any]) -> Dict[str, Any]:
+ self._state = control_data.get("state", "running")
+ while not self._closed:
+ await workflow.wait_condition(
+ lambda: self._closed or (self._state == "running" and bool(self._events))
+ )
+ if self._closed:
+ break
+ listener_id, event = self._events.pop(0)
+ spec = self._triggers.get(listener_id)
+ if spec is not None:
+ await self._spawn_push_run(event, spec)
+ return {"state": self._state, "generation": control_data.get("generation")}
+
+ async def _spawn_push_run(self, event: Dict[str, Any], spec: Dict[str, Any]) -> None:
+ from services.temporal.trigger_listener_workflow import TriggerListenerWorkflow
+
+ listener = TriggerListenerWorkflow()
+ await listener._spawn_child_run(
+ event, spec["listener_args"], admission_check=self._wait_until_running,
+ )
+
+ async def _wait_until_running(self) -> None:
+ if self._state != "running":
+ await workflow.wait_condition(lambda: self._closed or self._state == "running")
+ if self._closed:
+ raise asyncio.CancelledError
+
+ async def _poll_trigger(self, listener_id: str, spec: Dict[str, Any]) -> None:
+ from services.temporal.polling_trigger_workflow import (
+ PollingTriggerWorkflow,
+ _ACTIVITY_TIMEOUT_MULT,
+ _DEFAULT_POLL_INTERVAL_S,
+ )
+
+ listener_data = spec["listener_args"]
+ node_type = listener_data["node_type"]
+ activity_name = f"poll.{node_type}.v{listener_data.get('version', 1)}"
+ params = listener_data.get("filter_params", {}) or {}
+ poll_interval = int(params.get("poll_interval") or _DEFAULT_POLL_INTERVAL_S)
+ activity_timeout_s = max(30, poll_interval * _ACTIVITY_TIMEOUT_MULT)
+ seen_ids: set[str] = set(listener_data.get("seen_ids") or [])
+ baseline = not seen_ids
+ runner = PollingTriggerWorkflow()
+
+ while not self._closed and listener_id in self._triggers:
+ if self._state != "running":
+ await workflow.wait_condition(lambda: self._closed or self._state == "running")
+ if self._closed:
+ return
+ if baseline:
+ baseline = False
+ baseline_only = True
+ else:
+ await workflow.sleep(timedelta(seconds=poll_interval))
+ if self._state != "running":
+ continue
+ baseline_only = False
+ payload = {
+ "node_id": listener_data["trigger_node_id"], "params": params,
+ "seen_ids": list(seen_ids), "baseline_only": baseline_only,
+ }
+ try:
+ result = await workflow.execute_activity(
+ activity_name, payload, activity_id=listener_data["trigger_node_id"],
+ start_to_close_timeout=timedelta(seconds=activity_timeout_s),
+ retry_policy=DEFAULT_ACTIVITY_RETRY,
+ )
+ except Exception as exc: # noqa: BLE001
+ workflow.logger.error(f"Controlled polling trigger failed: {exc}")
+ continue
+ seen_ids = set(result.get("seen_ids") or [])
+ for event in result.get("events") or []:
+ event_id = str(event.get("id") or "")
+ dedup_key = f"{listener_id}:{event_id}"
+ if not event_id or dedup_key in self._seen_event_ids:
+ continue
+ self._seen_event_ids.add(dedup_key)
+ if self._state != "running":
+ await workflow.wait_condition(lambda: self._closed or self._state == "running")
+ if self._closed:
+ return
+ await runner._spawn_child_run(event, listener_data)
+
+
+__all__ = ["WorkflowControlWorkflow"]
diff --git a/server/services/workflow.py b/server/services/workflow.py
index 9580b4c6..c13c59ed 100644
--- a/server/services/workflow.py
+++ b/server/services/workflow.py
@@ -561,6 +561,15 @@ class WorkflowService:
"""Get list of deployed workflow IDs."""
return self._get_deployment_manager().get_deployed_workflows()
+ def pause_deployment(self, workflow_id: str) -> bool:
+ return self._get_deployment_manager().pause(workflow_id)
+
+ async def resume_deployment(self, workflow_id: str) -> int:
+ return await self._get_deployment_manager().resume(workflow_id)
+
+ async def update_trigger_pause_status(self, workflow_id: str, *, paused: bool) -> int:
+ return await self._get_deployment_manager().update_trigger_pause_status(workflow_id, paused=paused)
+
# =========================================================================
# OUTPUT STORAGE
# =========================================================================
diff --git a/server/skills/assistant/task-manager/SKILL.md b/server/skills/assistant/task-manager/SKILL.md
index 301ce24a..93a8a1a8 100644
--- a/server/skills/assistant/task-manager/SKILL.md
+++ b/server/skills/assistant/task-manager/SKILL.md
@@ -4,7 +4,7 @@ description: Coordinate durable team tasks by assigning connected teammates, rev
allowed-tools: task_manager
metadata:
author: opencompany
- version: "2.2"
+ version: "2.3"
category: automation
---
@@ -27,8 +27,14 @@ to this lead's `input-teammates` handle.
three descendants across the whole agent tree; excess tasks remain queued.
After `assign_task` returns `queued`, report that delegation started and
return immediately. Do not poll or wait in the assigning invocation.
-5. When a teammate submits work, inspect the complete result, error, attempt
- history, and acceptance criteria. Copy `task.id` to `task_id` and
+5. When a teammate submits work or an attempt fails, call `get_task`, then use
+ `inspect_task_trace` before retrying, reassigning, or reporting a terminal
+ failure. Start with `detail="summary"`; use `failures` or a paginated
+ `timeline` only when the summary does not explain the outcome. For large
+ histories use `detail="search"` with a narrow `query`, inspect the returned
+ context lines, and follow `next_cursor` until the match is found or the
+ cursor is empty. Prefer `categories` and `all_terms` over broad searches.
+ Copy `task.id` to `task_id` and
`task.revision` to `expected_revision` for the review mutation.
6. Choose exactly one review outcome:
- `accept_task` when the result satisfies the task;
@@ -47,6 +53,7 @@ to this lead's `input-teammates` handle.
| `assign_task` | Persist and queue a bounded mission | `title`, `mission`, `assignee_node_id` or `delegate_name`, optional `context`, `acceptance_criteria`, `depends_on` |
| `list_tasks` | Inspect current or historical tasks | optional `status_filter`, `include_history` |
| `get_task` | Review one task and all attempts | `task_id` |
+| `inspect_task_trace` | Inspect or grep the task attempt's sanitized Temporal events | `task_id`; optional `attempt`, `detail`, `cursor`, `limit`; search supports `query`, `search_mode`, `categories`, `context_lines`, `scan_limit` |
| `modify_task` | Change queued/blocked work | `task_id`, `expected_revision`, changed task fields |
| `cancel_task` | Cancel queued or running work | `task_id`, `expected_revision`, `reason` |
| `retry_task` | Queue a new attempt | `task_id`, `expected_revision`, optional revision context |
@@ -76,6 +83,13 @@ Do not treat `submitted` as finished team work. Do not poll a running task;
`taskTrigger` starts a separate review invocation carrying the owning execution.
Do not invent teammate IDs, team IDs, or
execution IDs—the runtime supplies authority and rejects cross-team access.
+Trace output is intentionally sanitized and scoped to the persisted task
+execution. Never request or infer raw Temporal workflow IDs.
+
+Trace search is a bounded grep-like read over normalized event metadata, never
+raw payloads. A call scans at most 500 events and returns only matches plus up
+to five surrounding events. Continue from `next_cursor`; do not restart from
+the beginning or load the entire timeline when a targeted search is sufficient.
## Team Monitor interpretation
diff --git a/server/tests/nodes/test_ai_tools.py b/server/tests/nodes/test_ai_tools.py
index fe093a6c..b58348b6 100644
--- a/server/tests/nodes/test_ai_tools.py
+++ b/server/tests/nodes/test_ai_tools.py
@@ -322,6 +322,30 @@ class TestTaskManager:
with pytest.raises(ValueError, match="Unknown Task Manager operation"):
await _execute_task_manager({"operation": "self_destruct"}, self._config())
+ async def test_inspect_trace_uses_trusted_lead_scope(self, durable_service):
+ from nodes.tool.task_manager import _execute_task_manager
+
+ trace_service = MagicMock()
+ trace_service.get_trace = AsyncMock(return_value={
+ "status": "available", "events": [], "next_cursor": None,
+ })
+ with patch(
+ "services.team_task_trace.get_team_task_trace_service",
+ return_value=trace_service,
+ ):
+ result = await _execute_task_manager(
+ {"operation": "inspect_task_trace", "task_id": "task-1", "detail": "failures"},
+ self._config(),
+ )
+
+ assert result["trace"]["status"] == "available"
+ trace_service.get_trace.assert_awaited_once_with(
+ workflow_id="workflow-1", team_lead_node_id="lead-1", execution_id="run-1",
+ task_id="task-1", attempt=None, cursor=None, limit=50, detail="failures",
+ query=None, search_mode="literal", case_sensitive=False,
+ context_lines=2, scan_limit=250, categories=None,
+ )
+
# ============================================================================
# writeTodos
diff --git a/server/tests/services/test_team_task_trace.py b/server/tests/services/test_team_task_trace.py
new file mode 100644
index 00000000..4fb47c6f
--- /dev/null
+++ b/server/tests/services/test_team_task_trace.py
@@ -0,0 +1,169 @@
+from types import SimpleNamespace
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+from temporalio.api.enums.v1 import EventType
+
+from services.team_task_trace import TeamTaskTraceService, _redact
+
+
+def test_trace_redaction_removes_secret_values():
+ text = _redact("authorization: Bearer-abc api_key=top-secret ordinary context")
+ assert "Bearer-abc" not in text
+ assert "top-secret" not in text
+ assert text.count("[REDACTED]") == 2
+
+
+def _task(**execution):
+ return {
+ "id": "task-1", "team_id": "team-1", "current_attempt": 0,
+ "trace_id": "trace-1", "attempts": [{"attempt_number": 0, **execution}],
+ }
+
+
+def _team_service(task):
+ service = MagicMock()
+ service.get_durable_task = AsyncMock(return_value=task)
+ service.database.add_agent_message = AsyncMock(return_value={"id": 1})
+ return service
+
+
+async def test_unregistered_attempt_returns_explicit_status_without_temporal_lookup():
+ team_service = _team_service(_task())
+ trace = await TeamTaskTraceService(team_service, SimpleNamespace(client=MagicMock())).get_trace(
+ workflow_id="wf-1", team_lead_node_id="lead-1", execution_id="exec-1",
+ task_id="task-1",
+ )
+
+ assert trace["status"] == "execution_not_registered"
+ assert trace["events"] == []
+ team_service.database.add_agent_message.assert_awaited_once()
+
+
+async def test_history_is_paginated_sanitized_and_resolved_from_persisted_identity():
+ event = SimpleNamespace(
+ event_id=7,
+ event_type=EventType.Value("EVENT_TYPE_ACTIVITY_TASK_STARTED"),
+ event_time=None,
+ WhichOneof=lambda _name: None,
+ )
+ iterator = MagicMock()
+ iterator.current_page = [event]
+ iterator.next_page_token = b"next"
+ iterator.fetch_next_page = AsyncMock()
+ handle = MagicMock()
+ handle.fetch_history_events.return_value = iterator
+ client = MagicMock()
+ client.get_workflow_handle.return_value = handle
+ team_service = _team_service(_task(child_workflow_id="child-1", child_run_id="run-1"))
+
+ trace = await TeamTaskTraceService(team_service, SimpleNamespace(client=client)).get_trace(
+ workflow_id="wf-1", team_lead_node_id="lead-1", execution_id="exec-1",
+ task_id="task-1", detail="timeline", limit=500,
+ )
+
+ client.get_workflow_handle.assert_called_once_with("child-1", run_id="run-1")
+ handle.fetch_history_events.assert_called_once_with(
+ page_size=100, next_page_token=None, skip_archival=False,
+ )
+ assert trace["status"] == "available"
+ assert trace["events"] == [{"event_id": 7, "type": "activity_task_started", "category": "activity"}]
+ assert trace["next_cursor"] is not None
+
+
+async def test_trace_scope_rejection_happens_before_temporal_access():
+ team_service = MagicMock()
+ team_service.get_durable_task = AsyncMock(side_effect=ValueError("Task not found in this lead execution"))
+ client = MagicMock()
+
+ with pytest.raises(ValueError, match="lead execution"):
+ await TeamTaskTraceService(team_service, SimpleNamespace(client=client)).get_trace(
+ workflow_id="wf-1", team_lead_node_id="forged-lead", execution_id="exec-1",
+ task_id="task-foreign",
+ )
+ client.get_workflow_handle.assert_not_called()
+
+
+def _event(event_id, name):
+ return SimpleNamespace(
+ event_id=event_id, event_type=EventType.Value(f"EVENT_TYPE_{name}"),
+ event_time=None, WhichOneof=lambda _name: None,
+ )
+
+
+async def test_search_scans_pages_and_returns_bounded_context_with_match_markers():
+ iterator = MagicMock()
+ pages = [
+ ([_event(1, "WORKFLOW_EXECUTION_STARTED"), _event(2, "ACTIVITY_TASK_STARTED")], b"page-2"),
+ ([_event(3, "ACTIVITY_TASK_FAILED"), _event(4, "TIMER_STARTED")], b"page-3"),
+ ]
+
+ async def fetch_next_page(*, page_size):
+ page, token = pages.pop(0)
+ iterator.current_page = page
+ iterator.next_page_token = token
+
+ iterator.fetch_next_page = AsyncMock(side_effect=fetch_next_page)
+ handle = MagicMock()
+ handle.fetch_history_events.return_value = iterator
+ client = MagicMock()
+ client.get_workflow_handle.return_value = handle
+
+ trace = await TeamTaskTraceService(
+ _team_service(_task(child_workflow_id="child-1", child_run_id="run-1")),
+ SimpleNamespace(client=client),
+ ).get_trace(
+ workflow_id="wf-1", team_lead_node_id="lead-1", execution_id="exec-1",
+ task_id="task-1", detail="search", query="activity_task_failed",
+ limit=1, scan_limit=4, context_lines=1,
+ )
+
+ assert trace["search"] == {
+ "mode": "literal", "case_sensitive": False, "context_lines": 1,
+ "scanned_events": 4, "matched_events": 1,
+ }
+ assert [(event["event_id"], event["match"]) for event in trace["events"]] == [
+ (2, False), (3, True), (4, False),
+ ]
+ assert trace["next_cursor"] is not None
+ assert iterator.fetch_next_page.await_count == 2
+
+
+async def test_search_supports_term_modes_and_category_filters():
+ event = _event(9, "ACTIVITY_TASK_TIMED_OUT")
+ iterator = MagicMock(current_page=[event], next_page_token=None)
+ iterator.fetch_next_page = AsyncMock()
+ handle = MagicMock()
+ handle.fetch_history_events.return_value = iterator
+ client = MagicMock()
+ client.get_workflow_handle.return_value = handle
+
+ trace = await TeamTaskTraceService(
+ _team_service(_task(runner_workflow_id="runner-1", runner_run_id="run-1")),
+ SimpleNamespace(client=client),
+ ).get_trace(
+ workflow_id="wf-1", team_lead_node_id="lead-1", execution_id="exec-1",
+ task_id="task-1", detail="search", query="activity timed_out",
+ search_mode="all_terms", categories=["failure"], context_lines=0,
+ )
+
+ assert trace["search"]["matched_events"] == 1
+ assert trace["events"][0]["match"] is True
+
+
+async def test_search_requires_bounded_valid_query():
+ service = TeamTaskTraceService(_team_service(_task()), SimpleNamespace(client=MagicMock()))
+ common = dict(workflow_id="wf-1", team_lead_node_id="lead-1", execution_id="exec-1", task_id="task-1")
+
+ with pytest.raises(ValueError, match="non-empty query"):
+ await service.get_trace(**common, detail="search", query=" ")
+ with pytest.raises(ValueError, match="200 characters"):
+ await service.get_trace(**common, detail="search", query="x" * 201)
+ with pytest.raises(ValueError, match="category"):
+ await service.get_trace(**common, detail="search", query="failed", categories=["payload"])
diff --git a/server/tests/services/test_workflow_control.py b/server/tests/services/test_workflow_control.py
new file mode 100644
index 00000000..6a4f8e39
--- /dev/null
+++ b/server/tests/services/test_workflow_control.py
@@ -0,0 +1,83 @@
+from unittest.mock import AsyncMock
+
+import pytest
+from sqlalchemy.exc import IntegrityError
+
+from services.deployment.control import WorkflowControlService, serialize_control
+
+
+async def test_begin_generation_is_idempotent():
+ db = AsyncMock()
+ db.get_workflow_control_by_idempotency_key.side_effect = [None, None]
+ db.get_latest_workflow_control.return_value = None
+ db.create_workflow_control.side_effect = lambda control: control
+ service = WorkflowControlService(db)
+
+ control, created = await service.begin_generation(
+ workflow_id="wf", nodes=[], edges=[], session_id="s", idempotency_key="request-1"
+ )
+
+ assert created is True
+ assert control.generation == 1
+ assert control.status == "starting"
+ assert serialize_control(control)["can_pause"] is False
+
+
+async def test_concurrent_insert_rereads_idempotent_winner():
+ db = AsyncMock()
+ winner = AsyncMock()
+ winner.generation = 1
+ db.get_workflow_control_by_idempotency_key.side_effect = [None, winner]
+ db.get_latest_workflow_control.return_value = None
+ db.create_workflow_control.side_effect = IntegrityError("insert", {}, Exception("unique"))
+ service = WorkflowControlService(db)
+
+ control, created = await service.begin_generation(
+ workflow_id="wf", nodes=[], edges=[], session_id="s", idempotency_key="same-key"
+ )
+
+ assert control is winner
+ assert created is False
+
+
+async def test_transition_rejects_stale_revision():
+ db = AsyncMock()
+ db.transition_workflow_control.return_value = None
+ service = WorkflowControlService(db)
+ control = AsyncMock(id="control", revision=2)
+
+ with pytest.raises(ValueError, match="control_revision_conflict"):
+ await service.transition(
+ control, expected_revision=1, from_statuses={"running"}, status="pausing"
+ )
+
+
+async def test_reset_generation_is_ready_for_explicit_start():
+ reset_control = AsyncMock(status="reset", generation=3)
+ reset_control.workflow_id = "wf"
+ reset_control.revision = 9
+ reset_control.execution_id = "old-execution"
+ reset_control.root_execution_id = "old-execution"
+ reset_control.controller_workflow_id = "old-controller"
+ reset_control.controller_run_id = "old-run"
+ reset_control.created_at = None
+ reset_control.updated_at = None
+ reset_control.terminal_reason = "workflow_reset"
+ status = serialize_control(reset_control)
+ assert status["state"] == "ready"
+ assert status["can_start"] is True
+ assert status["can_reset"] is False
+
+ db = AsyncMock()
+ db.get_workflow_control_by_idempotency_key.return_value = None
+ db.get_latest_workflow_control.return_value = reset_control
+ db.create_workflow_control.side_effect = lambda control: control
+ control, created = await WorkflowControlService(db).begin_generation(
+ workflow_id="wf", nodes=[], edges=[], session_id="s", idempotency_key="start-after-reset"
+ )
+ assert created is True
+ assert control.generation == 4
diff --git a/server/tests/temporal/test_agent_workflow.py b/server/tests/temporal/test_agent_workflow.py
index 35feb64a..5473bbb0 100644
--- a/server/tests/temporal/test_agent_workflow.py
+++ b/server/tests/temporal/test_agent_workflow.py
@@ -279,6 +279,7 @@ class TestAgentActivities:
"agent.prepare_payload.v1",
"agent.queue_delegation.v1",
"agent.refresh_tools.v1",
+ "agent.register_task_execution.v1",
"agent.release_subagent_permit.v1",
"agent.store_output.v1",
]
diff --git a/server/tests/temporal/test_client_readiness.py b/server/tests/temporal/test_client_readiness.py
index 1b7d0d3f..c113d897 100644
--- a/server/tests/temporal/test_client_readiness.py
+++ b/server/tests/temporal/test_client_readiness.py
@@ -74,6 +74,9 @@ async def test_connect_returns_client_when_serving_first(monkeypatch):
assert result is fc
assert fc.service_client.check_health.await_count == 1
+ connect_kwargs = client_mod.Client.connect.await_args.kwargs
+ assert len(connect_kwargs["interceptors"]) == 1
+ assert connect_kwargs["interceptors"][0].__class__.__name__ == "TracingInterceptor"
async def test_connect_polls_until_serving(monkeypatch):
diff --git a/server/tests/temporal/test_cooperative_pause.py b/server/tests/temporal/test_cooperative_pause.py
new file mode 100644
index 00000000..6dd71735
--- /dev/null
+++ b/server/tests/temporal/test_cooperative_pause.py
@@ -0,0 +1,52 @@
+"""Cooperative pause flags shared by Temporal orchestration workflows."""
+
+import pytest
+
+from services.temporal.agent_workflow import AgentWorkflow, DelegatedTaskWorkflow
+from services.temporal.polling_trigger_workflow import PollingTriggerWorkflow
+from services.temporal.trigger_listener_workflow import TriggerListenerWorkflow
+from services.temporal.workflow import MachinaWorkflow
+from services.temporal.workflow_control_workflow import WorkflowControlWorkflow
+
+
+ "workflow_type",
+ [
+ MachinaWorkflow,
+ AgentWorkflow,
+ DelegatedTaskWorkflow,
+ TriggerListenerWorkflow,
+ PollingTriggerWorkflow,
+ ],
+)
+async def test_pause_and_resume_mutate_durable_workflow_state(workflow_type):
+ instance = workflow_type()
+
+ assert instance._control_paused is False
+ await instance.pause()
+ assert instance._control_paused is True
+ await instance.resume()
+ assert instance._control_paused is False
+
+
+async def test_controller_routes_push_events_without_listener_workflow():
+ controller = WorkflowControlWorkflow()
+ await controller.register_trigger({
+ "listener_id": "wf-chat", "workflow_type": "TriggerListenerWorkflow",
+ "trigger_node_id": "chat-1", "event_type": "com.opencompany.chat.message.received",
+ "event_types": ["com.opencompany.chat.message.received"], "listener_args": {},
+ })
+
+ await controller.on_event({
+ "id": "event-1", "type": "com.opencompany.chat.message.received", "data": {},
+ })
+ await controller.on_event({
+ "id": "event-1", "type": "com.opencompany.chat.message.received", "data": {},
+ })
+ await controller.on_event({"id": "event-2", "type": "unrelated", "data": {}})
+
+ assert len(controller._events) == 1
+ assert controller._events[0][0] == "wf-chat"
+ assert controller.status()["triggers"] == {"wf-chat": "chat-1"}
diff --git a/server/tests/test_cron_canary.py b/server/tests/test_cron_canary.py
index 51a9e393..78c8e6b0 100644
--- a/server/tests/test_cron_canary.py
+++ b/server/tests/test_cron_canary.py
@@ -522,3 +522,32 @@ class TestCronPluginSelfRegisters:
assert canary_registry.is_canary_trigger_type("cronScheduler"), (
"Importing nodes.scheduler.cron_scheduler should call " "register_canary_trigger_type('cronScheduler')."
)
[email protected]("paused", [True, False])
+async def test_schedule_pause_state_follows_workflow_control(paused):
+ from services.temporal.schedules import set_cron_schedules_paused
+
+ handles = {}
+
+ async def fake_list(query, **kwargs):
+ assert "EventWorkflowId='wf-1'" in query
+ return _FakeScheduleIterator(["schedule-a", "schedule-b"])
+
+ def fake_get_handle(schedule_id):
+ handle = MagicMock()
+ handle.pause = AsyncMock()
+ handle.unpause = AsyncMock()
+ handles[schedule_id] = handle
+ return handle
+
+ client = MagicMock(list_schedules=fake_list, get_schedule_handle=fake_get_handle)
+ changed = await set_cron_schedules_paused(client, "wf-1", paused=paused)
+
+ assert changed == 2
+ for handle in handles.values():
+ if paused:
+ handle.pause.assert_awaited_once()
+ handle.unpause.assert_not_awaited()
+ else:
+ handle.unpause.assert_awaited_once()
+ handle.pause.assert_not_awaited()
diff --git a/server/tests/test_deployment_canary_listener.py b/server/tests/test_deployment_canary_listener.py
index 242184ab..88874168 100644
--- a/server/tests/test_deployment_canary_listener.py
+++ b/server/tests/test_deployment_canary_listener.py
@@ -266,6 +266,44 @@ class TestStartCanaryListener:
assert call["id_conflict_policy"] == WorkflowIDConflictPolicy.USE_EXISTING
assert call["task_queue"] == "machina-tasks"
+ @pytest.mark.asyncio
+ async def test_controlled_listener_is_registered_under_controller(self, monkeypatch):
+ mgr, _ = _build_manager_with_state(
+ "wf-controlled", nodes=[_node("wh-1", "webhookTrigger")], edges=[],
+ )
+ controller_handle = MagicMock()
+ controller_handle.signal = AsyncMock()
+ client = MagicMock()
+ client.get_workflow_handle.return_value = controller_handle
+ client.start_workflow = AsyncMock()
+ wrapper = MagicMock(client=client)
+ control = MagicMock(
+ status="running", controller_workflow_id="workflow-control-wf-controlled-g1",
+ controller_run_id="controller-run-1",
+ )
+ database = MagicMock()
+ database.get_latest_workflow_control = AsyncMock(return_value=control)
+
+ from core import container as container_mod
+
+ monkeypatch.setattr(container_mod.container, "temporal_client", lambda: wrapper)
+ monkeypatch.setattr(container_mod.container, "database", lambda: database)
+
+ listener_id = await mgr._start_canary_listener(
+ _node("wh-1", "webhookTrigger"), "wf-controlled", params={},
+ )
+
+ assert listener_id == "wf-controlled-webhookTrigger"
+ client.start_workflow.assert_not_awaited()
+ client.get_workflow_handle.assert_called_once_with(
+ "workflow-control-wf-controlled-g1", run_id="controller-run-1",
+ )
+ signal_name, spec = controller_handle.signal.await_args.args
+ assert signal_name == "register_trigger"
+ assert spec["listener_id"] == listener_id
+ assert spec["workflow_type"] == "TriggerListenerWorkflow"
+ assert spec["workflow_id"] == "wf-controlled"
+
@pytest.mark.asyncio
async def test_search_attributes_include_event_workflow_id(self, monkeypatch):
"""Cancel path queries by EventWorkflowId — start must set it."""
@@ -693,3 +731,20 @@ class TestNoInstanceStateForCanaryListeners:
# Both helpers can be called on the class without an instance.
assert DeploymentManager._listener_workflow_id("wf", "node") == "wf-node"
+async def test_pause_status_disarms_trigger_nodes_without_touching_agents():
+ mgr, broadcaster = _build_manager_with_state(
+ "wf-pause",
+ nodes=[
+ _node("start-1", "start"), _node("chat-1", "chatTrigger"),
+ _node("agent-1", "aiAgent"),
+ ],
+ edges=[],
+ )
+
+ changed = await mgr.update_trigger_pause_status("wf-pause", paused=True)
+
+ assert changed == 1
+ broadcaster.update_node_status.assert_awaited_once_with(
+ "chat-1", "idle", {"paused": True, "message": "Paused"}, workflow_id="wf-pause",
+ )