Skip to content

Self-improving AI Employees for everyone

fix(temporal): drop workflow lifetime caps so paused runs survive months

trohitg2065a67d0cd7
diff --git a/server/nodes/scheduler/cron_scheduler/_workflow.py b/server/nodes/scheduler/cron_scheduler/_workflow.py
index 605e250e..0dd39e01 100644
--- a/server/nodes/scheduler/cron_scheduler/_workflow.py
+++ b/server/nodes/scheduler/cron_scheduler/_workflow.py
@@ -41,6 +41,11 @@ CHILD_SEARCH_ATTRIBUTES_PATCH = "cron-child-search-attributes-v1"
COOPERATIVE_PAUSE_SCHEDULING_PATCH = (
"cron-cooperative-pause-scheduling-v1"
)
+# Same contract as trigger_listener_workflow.UNBOUNDED_CHILD_RUNS_PATCH:
+# a spawned run may execute or stay paused for months, so new firings
+# start children without lifetime caps; the patch keeps replay
+# compatibility for histories recorded with the old 1h caps.
+UNBOUNDED_CHILD_RUNS_PATCH = "cron-unbounded-child-runs-v1"
@workflow.defn(name="CronTriggerWorkflow", sandboxed=False)
@@ -136,9 +141,11 @@ class CronTriggerWorkflow:
use_cooperative_pause_schedule_gate = workflow.patched(
COOPERATIVE_PAUSE_SCHEDULING_PATCH
)
+ unbounded_child_runs = workflow.patched(UNBOUNDED_CHILD_RUNS_PATCH)
except RuntimeError: # direct unit invocation outside Temporal runtime
use_child_search_attributes = False
use_cooperative_pause_schedule_gate = False
+ unbounded_child_runs = True
from services.temporal.trigger_listener_workflow import (
event_workflow_search_attributes,
)
@@ -149,9 +156,12 @@ class CronTriggerWorkflow:
"id_reuse_policy": (
WorkflowIDReusePolicy.ALLOW_DUPLICATE_FAILED_ONLY
),
- "execution_timeout": timedelta(hours=1),
- "run_timeout": timedelta(hours=1),
}
+ if not unbounded_child_runs:
+ # Replay-only: pre-patch histories recorded 1h caps on the
+ # child-start command. See UNBOUNDED_CHILD_RUNS_PATCH.
+ child_options["execution_timeout"] = timedelta(hours=1)
+ child_options["run_timeout"] = timedelta(hours=1)
if use_child_search_attributes:
child_options["search_attributes"] = (
event_workflow_search_attributes(deployment_workflow_id)
diff --git a/server/services/temporal/_retry_policies.py b/server/services/temporal/_retry_policies.py
index 00d0a47d..31d34760 100644
--- a/server/services/temporal/_retry_policies.py
+++ b/server/services/temporal/_retry_policies.py
@@ -108,10 +108,27 @@ DELEGATION_CLEANUP_RETRY: RetryPolicy = RetryPolicy(
)
+# Indefinite wait for a subagent-admission permit. The acquire activity
+# is a heartbeating poll-until-admitted loop whose single attempt is
+# bounded by its start_to_close timeout; with the previous 3-attempt
+# default a delegation queued behind long-running siblings (or a paused
+# deployment) for more than ~3h failed permanently instead of waiting.
+# ``maximum_attempts=0`` is Temporal's "unlimited" — liveness stays
+# guaranteed by the activity's heartbeat, and cancellation still
+# propagates normally when the parent is cancelled.
+PERMIT_WAIT_RETRY: RetryPolicy = RetryPolicy(
+ initial_interval=timedelta(seconds=1),
+ maximum_interval=timedelta(seconds=30),
+ maximum_attempts=0,
+ non_retryable_error_types=list(NON_RETRYABLE_ERROR_TYPES),
+)
+
+
__all__ = [
"NON_RETRYABLE_ERROR_TYPES",
"DEFAULT_ACTIVITY_RETRY",
"QUICK_ACTIVITY_RETRY",
"LLM_STEP_RETRY",
"DELEGATION_CLEANUP_RETRY",
+ "PERMIT_WAIT_RETRY",
]
diff --git a/server/services/temporal/agent_workflow.py b/server/services/temporal/agent_workflow.py
index 3f0a4738..e2257411 100644
--- a/server/services/temporal/agent_workflow.py
+++ b/server/services/temporal/agent_workflow.py
@@ -60,6 +60,7 @@ from ._retry_policies import (
DEFAULT_ACTIVITY_RETRY,
DELEGATION_CLEANUP_RETRY,
LLM_STEP_RETRY,
+ PERMIT_WAIT_RETRY,
)
from .workflow import AGENT_WORKFLOW_TYPES
@@ -131,6 +132,15 @@ DELEGATED_TASK_CANCELLATION_TERMINAL_PATCH = (
DELEGATED_TASK_SEARCH_ATTRIBUTES_PATCH = (
"agent-delegated-task-search-attributes-v1"
)
+# Agent/delegated children used to carry 1-2h execution/run timeouts and
+# the permit wait a ~3h total retry budget. Runs may execute — or stay
+# cooperatively paused — for months, and Temporal's timeout timers keep
+# ticking through a pause, so the caps silently terminated long/paused
+# work (skipping the compensation blocks: leaked permits, stuck task
+# rows). New executions start children unbounded and wait for permits
+# indefinitely; the patch keeps replay compatibility for histories
+# recorded with the old caps.
+UNBOUNDED_LIFETIMES_PATCH = "agent-unbounded-lifetimes-v1"
DUPLICATE_TOOL_NAME_ERROR_TYPE = "DuplicateToolNameError"
@@ -327,6 +337,7 @@ class DelegatedTaskWorkflow:
use_cancellation_terminal = workflow.patched(
DELEGATED_TASK_CANCELLATION_TERMINAL_PATCH
)
+ use_unbounded_lifetimes = workflow.patched(UNBOUNDED_LIFETIMES_PATCH)
acquire_cancellation_options = (
{
"cancellation_type": (
@@ -367,7 +378,11 @@ class DelegatedTaskWorkflow:
args=[acquire_payload],
start_to_close_timeout=timedelta(hours=1),
heartbeat_timeout=timedelta(seconds=10),
- retry_policy=AGENT_ACTIVITY_RETRY,
+ # Unlimited attempts on the patched path — a queued
+ # delegation waits as long as admission takes.
+ retry_policy=(
+ PERMIT_WAIT_RETRY if use_unbounded_lifetimes else AGENT_ACTIVITY_RETRY
+ ),
**acquire_cancellation_options,
)
acquired_permit_id = str(
@@ -388,10 +403,14 @@ class DelegatedTaskWorkflow:
request["child_context"]["team_permit_id"] = (
acquired_permit_id
)
+ runner_child_kwargs: Dict[str, Any] = {"id": request["child_workflow_id"]}
+ if not use_unbounded_lifetimes:
+ # Replay-only: pre-patch histories recorded 1h caps.
+ runner_child_kwargs["execution_timeout"] = timedelta(hours=1)
+ runner_child_kwargs["run_timeout"] = timedelta(hours=1)
child_handle = await workflow.start_child_workflow(
"AgentWorkflow", args=[request["child_context"]],
- id=request["child_workflow_id"],
- execution_timeout=timedelta(hours=1), run_timeout=timedelta(hours=1),
+ **runner_child_kwargs,
)
await workflow.execute_activity(
"agent.register_task_execution.v1",
@@ -629,6 +648,7 @@ class AgentWorkflow:
use_delegation_permit_cleanup = workflow.patched(
DELEGATION_PERMIT_CLEANUP_PATCH
)
+ use_unbounded_lifetimes = workflow.patched(UNBOUNDED_LIFETIMES_PATCH)
use_acquire_cancellation_completion = workflow.patched(
DELEGATION_ACQUIRE_CANCELLATION_PATCH
)
@@ -1294,7 +1314,11 @@ class AgentWorkflow:
),
start_to_close_timeout=timedelta(hours=1),
heartbeat_timeout=timedelta(seconds=10),
- retry_policy=AGENT_ACTIVITY_RETRY,
+ # Unlimited attempts on the patched path — a queued
+ # delegation waits as long as admission takes.
+ retry_policy=(
+ PERMIT_WAIT_RETRY if use_unbounded_lifetimes else AGENT_ACTIVITY_RETRY
+ ),
**acquire_cancellation_options,
)
delegation_permits[call_index] = str(
@@ -1341,12 +1365,15 @@ class AgentWorkflow:
try:
if use_cooperative_pause_schedule_gate:
await self._wait_until_resumed()
+ delegation_child_kwargs: Dict[str, Any] = {"id": child_id}
+ if not use_unbounded_lifetimes:
+ # Replay-only: pre-patch histories recorded 1h caps.
+ delegation_child_kwargs["execution_timeout"] = timedelta(hours=1)
+ delegation_child_kwargs["run_timeout"] = timedelta(hours=1)
delegation_handles[call_index] = await workflow.start_child_workflow(
"AgentWorkflow",
args=[child_context],
- id=child_id,
- execution_timeout=timedelta(hours=1),
- run_timeout=timedelta(hours=1),
+ **delegation_child_kwargs,
)
if team_id:
child_handle = delegation_handles[call_index]
@@ -1479,9 +1506,13 @@ class AgentWorkflow:
delegated_task_options: Dict[str, Any] = {
"id": runner_id,
"parent_close_policy": ParentClosePolicy.ABANDON,
- "execution_timeout": timedelta(hours=2),
- "run_timeout": timedelta(hours=2),
}
+ if not use_unbounded_lifetimes:
+ # Replay-only: pre-patch histories recorded 2h caps.
+ # A capped runner that timed out skipped its finally
+ # compensation (leaked permit + stuck task row).
+ delegated_task_options["execution_timeout"] = timedelta(hours=2)
+ delegated_task_options["run_timeout"] = timedelta(hours=2)
use_delegated_task_search_attributes = workflow.patched(
DELEGATED_TASK_SEARCH_ATTRIBUTES_PATCH
)
diff --git a/server/services/temporal/workflow.py b/server/services/temporal/workflow.py
index e695a301..874adeb8 100644
--- a/server/services/temporal/workflow.py
+++ b/server/services/temporal/workflow.py
@@ -95,6 +95,21 @@ AGENT_CHILD_ID_V2_PATCH = "machina-agent-child-id-v2"
COOPERATIVE_PAUSE_SCHEDULING_PATCH = (
"machina-cooperative-pause-scheduling-v1"
)
+# Agent child workflows used to carry 1h execution/run timeouts, and node
+# activities a 10-minute start_to_close. A run may legitimately execute —
+# or stay cooperatively paused — for months, and Temporal's timeout
+# timers keep ticking through a pause, so the caps silently terminated
+# long/paused runs. New executions start agent children unbounded and
+# give node activities a generous start_to_close; liveness is enforced
+# by the 2-minute heartbeat timeout (activities self-heartbeat every
+# 30s), not by lifetime caps. Patch keeps replay compatibility for
+# histories recorded with the old caps.
+UNBOUNDED_LIFETIMES_PATCH = "machina-unbounded-lifetimes-v1"
+# start_to_close for node activities on the patched path. Generous by
+# design: a single node step (agent turn batch, browser op, long shell
+# command) may run for hours; worker-death detection is the heartbeat
+# timeout's job, so a long ceiling costs nothing in liveness.
+_NODE_ACTIVITY_START_TO_CLOSE = timedelta(hours=24)
def _agent_child_workflow_id_v2(root_workflow_id: str, agent_node_id: str) -> str:
@@ -259,6 +274,7 @@ class MachinaWorkflow:
use_cooperative_pause_schedule_gate = workflow.patched(
COOPERATIVE_PAUSE_SCHEDULING_PATCH
)
+ use_unbounded_lifetimes = workflow.patched(UNBOUNDED_LIFETIMES_PATCH)
workflow.logger.info(f"Starting workflow orchestration: {len(nodes)} nodes, {len(edges)} edges")
@@ -423,14 +439,19 @@ class MachinaWorkflow:
)
else:
child_workflow_id = f"{workflow_slug}-{node_label_slug(node)}"
- handle = await workflow.start_child_workflow(
- dispatch["name"],
+ child_start_kwargs: Dict[str, Any] = dict(
args=[context],
id=child_workflow_id,
- # Child workflow execution timeout: agent loops can run
- # 10+ minutes with multiple LLM turns. Set generously.
- execution_timeout=timedelta(hours=1),
- run_timeout=timedelta(hours=1),
+ )
+ if not use_unbounded_lifetimes:
+ # Replay-only: pre-patch histories recorded 1h caps
+ # on the child-start command. New runs are unbounded
+ # (see UNBOUNDED_LIFETIMES_PATCH).
+ child_start_kwargs["execution_timeout"] = timedelta(hours=1)
+ child_start_kwargs["run_timeout"] = timedelta(hours=1)
+ handle = await workflow.start_child_workflow(
+ dispatch["name"],
+ **child_start_kwargs,
)
running[node_id] = handle
workflow.logger.info(f"Scheduled child workflow for node: {node_id} " f"(workflow={dispatch['name']})")
@@ -443,7 +464,15 @@ class MachinaWorkflow:
start_kwargs: Dict[str, Any] = dict(
args=[context],
activity_id=node_id,
- start_to_close_timeout=timedelta(minutes=10),
+ # Heartbeat is the liveness mechanism (activities
+ # self-heartbeat every 30s); start_to_close only
+ # bounds a single legitimate step. Pre-patch
+ # histories replay against the old 10min cap.
+ start_to_close_timeout=(
+ _NODE_ACTIVITY_START_TO_CLOSE
+ if use_unbounded_lifetimes
+ else timedelta(minutes=10)
+ ),
heartbeat_timeout=timedelta(minutes=2),
retry_policy=retry_policy,
)