Self-improving AI Employees for everyone
Python ai-agentsai-employeeai-employeesassistantautonomous-agentsco-employeescompanygrok-botinstinctloop-agentsmuseown-your-datapersonalself-hosted
fix(temporal): drop workflow lifetime caps so paused runs survive months
trohitg2065a67d0cd7
diff --git a/server/nodes/scheduler/cron_scheduler/_workflow.py b/server/nodes/scheduler/cron_scheduler/_workflow.pyindex 605e250e..0dd39e01 100644--- a/server/nodes/scheduler/cron_scheduler/_workflow.py+++ b/server/nodes/scheduler/cron_scheduler/_workflow.py@@ -41,6 +41,11 @@ CHILD_SEARCH_ATTRIBUTES_PATCH = "cron-child-search-attributes-v1"COOPERATIVE_PAUSE_SCHEDULING_PATCH = ("cron-cooperative-pause-scheduling-v1")+# Same contract as trigger_listener_workflow.UNBOUNDED_CHILD_RUNS_PATCH:+# a spawned run may execute or stay paused for months, so new firings+# start children without lifetime caps; the patch keeps replay+# compatibility for histories recorded with the old 1h caps.+UNBOUNDED_CHILD_RUNS_PATCH = "cron-unbounded-child-runs-v1"@workflow.defn(name="CronTriggerWorkflow", sandboxed=False)@@ -136,9 +141,11 @@ class CronTriggerWorkflow:use_cooperative_pause_schedule_gate = workflow.patched(COOPERATIVE_PAUSE_SCHEDULING_PATCH)+ unbounded_child_runs = workflow.patched(UNBOUNDED_CHILD_RUNS_PATCH)except RuntimeError: # direct unit invocation outside Temporal runtimeuse_child_search_attributes = Falseuse_cooperative_pause_schedule_gate = False+ unbounded_child_runs = Truefrom services.temporal.trigger_listener_workflow import (event_workflow_search_attributes,)@@ -149,9 +156,12 @@ class CronTriggerWorkflow:"id_reuse_policy": (WorkflowIDReusePolicy.ALLOW_DUPLICATE_FAILED_ONLY),- "execution_timeout": timedelta(hours=1),- "run_timeout": timedelta(hours=1),}+ if not unbounded_child_runs:+ # Replay-only: pre-patch histories recorded 1h caps on the+ # child-start command. See UNBOUNDED_CHILD_RUNS_PATCH.+ child_options["execution_timeout"] = timedelta(hours=1)+ child_options["run_timeout"] = timedelta(hours=1)if use_child_search_attributes:child_options["search_attributes"] = (event_workflow_search_attributes(deployment_workflow_id)diff --git a/server/services/temporal/_retry_policies.py b/server/services/temporal/_retry_policies.pyindex 00d0a47d..31d34760 100644--- a/server/services/temporal/_retry_policies.py+++ b/server/services/temporal/_retry_policies.py@@ -108,10 +108,27 @@ DELEGATION_CLEANUP_RETRY: RetryPolicy = RetryPolicy()+# Indefinite wait for a subagent-admission permit. The acquire activity+# is a heartbeating poll-until-admitted loop whose single attempt is+# bounded by its start_to_close timeout; with the previous 3-attempt+# default a delegation queued behind long-running siblings (or a paused+# deployment) for more than ~3h failed permanently instead of waiting.+# ``maximum_attempts=0`` is Temporal's "unlimited" — liveness stays+# guaranteed by the activity's heartbeat, and cancellation still+# propagates normally when the parent is cancelled.+PERMIT_WAIT_RETRY: RetryPolicy = RetryPolicy(+ initial_interval=timedelta(seconds=1),+ maximum_interval=timedelta(seconds=30),+ maximum_attempts=0,+ non_retryable_error_types=list(NON_RETRYABLE_ERROR_TYPES),+)++__all__ = ["NON_RETRYABLE_ERROR_TYPES","DEFAULT_ACTIVITY_RETRY","QUICK_ACTIVITY_RETRY","LLM_STEP_RETRY","DELEGATION_CLEANUP_RETRY",+ "PERMIT_WAIT_RETRY",]diff --git a/server/services/temporal/agent_workflow.py b/server/services/temporal/agent_workflow.pyindex 3f0a4738..e2257411 100644--- a/server/services/temporal/agent_workflow.py+++ b/server/services/temporal/agent_workflow.py@@ -60,6 +60,7 @@ from ._retry_policies import (DEFAULT_ACTIVITY_RETRY,DELEGATION_CLEANUP_RETRY,LLM_STEP_RETRY,+ PERMIT_WAIT_RETRY,)from .workflow import AGENT_WORKFLOW_TYPES@@ -131,6 +132,15 @@ DELEGATED_TASK_CANCELLATION_TERMINAL_PATCH = (DELEGATED_TASK_SEARCH_ATTRIBUTES_PATCH = ("agent-delegated-task-search-attributes-v1")+# Agent/delegated children used to carry 1-2h execution/run timeouts and+# the permit wait a ~3h total retry budget. Runs may execute — or stay+# cooperatively paused — for months, and Temporal's timeout timers keep+# ticking through a pause, so the caps silently terminated long/paused+# work (skipping the compensation blocks: leaked permits, stuck task+# rows). New executions start children unbounded and wait for permits+# indefinitely; the patch keeps replay compatibility for histories+# recorded with the old caps.+UNBOUNDED_LIFETIMES_PATCH = "agent-unbounded-lifetimes-v1"DUPLICATE_TOOL_NAME_ERROR_TYPE = "DuplicateToolNameError"@@ -327,6 +337,7 @@ class DelegatedTaskWorkflow:use_cancellation_terminal = workflow.patched(DELEGATED_TASK_CANCELLATION_TERMINAL_PATCH)+ use_unbounded_lifetimes = workflow.patched(UNBOUNDED_LIFETIMES_PATCH)acquire_cancellation_options = ({"cancellation_type": (@@ -367,7 +378,11 @@ class DelegatedTaskWorkflow:args=[acquire_payload],start_to_close_timeout=timedelta(hours=1),heartbeat_timeout=timedelta(seconds=10),- retry_policy=AGENT_ACTIVITY_RETRY,+ # Unlimited attempts on the patched path — a queued+ # delegation waits as long as admission takes.+ retry_policy=(+ PERMIT_WAIT_RETRY if use_unbounded_lifetimes else AGENT_ACTIVITY_RETRY+ ),**acquire_cancellation_options,)acquired_permit_id = str(@@ -388,10 +403,14 @@ class DelegatedTaskWorkflow:request["child_context"]["team_permit_id"] = (acquired_permit_id)+ runner_child_kwargs: Dict[str, Any] = {"id": request["child_workflow_id"]}+ if not use_unbounded_lifetimes:+ # Replay-only: pre-patch histories recorded 1h caps.+ runner_child_kwargs["execution_timeout"] = timedelta(hours=1)+ runner_child_kwargs["run_timeout"] = timedelta(hours=1)child_handle = await workflow.start_child_workflow("AgentWorkflow", args=[request["child_context"]],- id=request["child_workflow_id"],- execution_timeout=timedelta(hours=1), run_timeout=timedelta(hours=1),+ **runner_child_kwargs,)await workflow.execute_activity("agent.register_task_execution.v1",@@ -629,6 +648,7 @@ class AgentWorkflow:use_delegation_permit_cleanup = workflow.patched(DELEGATION_PERMIT_CLEANUP_PATCH)+ use_unbounded_lifetimes = workflow.patched(UNBOUNDED_LIFETIMES_PATCH)use_acquire_cancellation_completion = workflow.patched(DELEGATION_ACQUIRE_CANCELLATION_PATCH)@@ -1294,7 +1314,11 @@ class AgentWorkflow:),start_to_close_timeout=timedelta(hours=1),heartbeat_timeout=timedelta(seconds=10),- retry_policy=AGENT_ACTIVITY_RETRY,+ # Unlimited attempts on the patched path — a queued+ # delegation waits as long as admission takes.+ retry_policy=(+ PERMIT_WAIT_RETRY if use_unbounded_lifetimes else AGENT_ACTIVITY_RETRY+ ),**acquire_cancellation_options,)delegation_permits[call_index] = str(@@ -1341,12 +1365,15 @@ class AgentWorkflow:try:if use_cooperative_pause_schedule_gate:await self._wait_until_resumed()+ delegation_child_kwargs: Dict[str, Any] = {"id": child_id}+ if not use_unbounded_lifetimes:+ # Replay-only: pre-patch histories recorded 1h caps.+ delegation_child_kwargs["execution_timeout"] = timedelta(hours=1)+ delegation_child_kwargs["run_timeout"] = timedelta(hours=1)delegation_handles[call_index] = await workflow.start_child_workflow("AgentWorkflow",args=[child_context],- id=child_id,- execution_timeout=timedelta(hours=1),- run_timeout=timedelta(hours=1),+ **delegation_child_kwargs,)if team_id:child_handle = delegation_handles[call_index]@@ -1479,9 +1506,13 @@ class AgentWorkflow:delegated_task_options: Dict[str, Any] = {"id": runner_id,"parent_close_policy": ParentClosePolicy.ABANDON,- "execution_timeout": timedelta(hours=2),- "run_timeout": timedelta(hours=2),}+ if not use_unbounded_lifetimes:+ # Replay-only: pre-patch histories recorded 2h caps.+ # A capped runner that timed out skipped its finally+ # compensation (leaked permit + stuck task row).+ delegated_task_options["execution_timeout"] = timedelta(hours=2)+ delegated_task_options["run_timeout"] = timedelta(hours=2)use_delegated_task_search_attributes = workflow.patched(DELEGATED_TASK_SEARCH_ATTRIBUTES_PATCH)diff --git a/server/services/temporal/workflow.py b/server/services/temporal/workflow.pyindex e695a301..874adeb8 100644--- a/server/services/temporal/workflow.py+++ b/server/services/temporal/workflow.py@@ -95,6 +95,21 @@ AGENT_CHILD_ID_V2_PATCH = "machina-agent-child-id-v2"COOPERATIVE_PAUSE_SCHEDULING_PATCH = ("machina-cooperative-pause-scheduling-v1")+# Agent child workflows used to carry 1h execution/run timeouts, and node+# activities a 10-minute start_to_close. A run may legitimately execute —+# or stay cooperatively paused — for months, and Temporal's timeout+# timers keep ticking through a pause, so the caps silently terminated+# long/paused runs. New executions start agent children unbounded and+# give node activities a generous start_to_close; liveness is enforced+# by the 2-minute heartbeat timeout (activities self-heartbeat every+# 30s), not by lifetime caps. Patch keeps replay compatibility for+# histories recorded with the old caps.+UNBOUNDED_LIFETIMES_PATCH = "machina-unbounded-lifetimes-v1"+# start_to_close for node activities on the patched path. Generous by+# design: a single node step (agent turn batch, browser op, long shell+# command) may run for hours; worker-death detection is the heartbeat+# timeout's job, so a long ceiling costs nothing in liveness.+_NODE_ACTIVITY_START_TO_CLOSE = timedelta(hours=24)def _agent_child_workflow_id_v2(root_workflow_id: str, agent_node_id: str) -> str:@@ -259,6 +274,7 @@ class MachinaWorkflow:use_cooperative_pause_schedule_gate = workflow.patched(COOPERATIVE_PAUSE_SCHEDULING_PATCH)+ use_unbounded_lifetimes = workflow.patched(UNBOUNDED_LIFETIMES_PATCH)workflow.logger.info(f"Starting workflow orchestration: {len(nodes)} nodes, {len(edges)} edges")@@ -423,14 +439,19 @@ class MachinaWorkflow:)else:child_workflow_id = f"{workflow_slug}-{node_label_slug(node)}"- handle = await workflow.start_child_workflow(- dispatch["name"],+ child_start_kwargs: Dict[str, Any] = dict(args=[context],id=child_workflow_id,- # Child workflow execution timeout: agent loops can run- # 10+ minutes with multiple LLM turns. Set generously.- execution_timeout=timedelta(hours=1),- run_timeout=timedelta(hours=1),+ )+ if not use_unbounded_lifetimes:+ # Replay-only: pre-patch histories recorded 1h caps+ # on the child-start command. New runs are unbounded+ # (see UNBOUNDED_LIFETIMES_PATCH).+ child_start_kwargs["execution_timeout"] = timedelta(hours=1)+ child_start_kwargs["run_timeout"] = timedelta(hours=1)+ handle = await workflow.start_child_workflow(+ dispatch["name"],+ **child_start_kwargs,)running[node_id] = handleworkflow.logger.info(f"Scheduled child workflow for node: {node_id} " f"(workflow={dispatch['name']})")@@ -443,7 +464,15 @@ class MachinaWorkflow:start_kwargs: Dict[str, Any] = dict(args=[context],activity_id=node_id,- start_to_close_timeout=timedelta(minutes=10),+ # Heartbeat is the liveness mechanism (activities+ # self-heartbeat every 30s); start_to_close only+ # bounds a single legitimate step. Pre-patch+ # histories replay against the old 10min cap.+ start_to_close_timeout=(+ _NODE_ACTIVITY_START_TO_CLOSE+ if use_unbounded_lifetimes+ else timedelta(minutes=10)+ ),heartbeat_timeout=timedelta(minutes=2),retry_policy=retry_policy,)