Skip to content

Self-improving AI Employees for everyone

fix(employees): reviewed delivery re-attaches to its lease (1b.9)

trohitg0c582e649ea7
diff --git a/docs-internal/node-logic-flows/workflow/employeeJob.md b/docs-internal/node-logic-flows/workflow/employeeJob.md
index bd29180a..ade35dde 100644
--- a/docs-internal/node-logic-flows/workflow/employeeJob.md
+++ b/docs-internal/node-logic-flows/workflow/employeeJob.md
@@ -14,6 +14,14 @@ Operations:
- `deliver`: resolve the originating job by its execution or team identity.
Assignment receipts and unaccepted task submissions cannot publish.
Accepted work uses the original delivery configuration and approval gates.
+ On the team runtime (`EmployeeJobWorkflow`, the lead's own delivery and the
+ owner's resolution of an uncertain send) the delivery activity retries
+ without limit (`REVIEWED_DELIVERY_RETRY`): a worker restart during the
+ approval wait re-attaches to the same delivery under its lease, a sink
+ that refuses cleanly ends the job failed, and a failed job is never
+ delivered again. In a v1 graph `deliver` runs as the node's own activity
+ under the plugin's default policy, 3 attempts of up to 24 h each, so it
+ re-attaches only within those.
`operation`, `lead_node_id` and `delivery_node_ids` are server-controlled
configuration. Models can supply only the mission when this node is a tool.
diff --git a/server/services/employees/delivery_resolution.py b/server/services/employees/delivery_resolution.py
index 2f39c64e..af15b150 100644
--- a/server/services/employees/delivery_resolution.py
+++ b/server/services/employees/delivery_resolution.py
@@ -126,6 +126,7 @@ async def continue_delivery(database: Any, job_id: str, resolution_id: str) -> d
return {"delivered": False, "state": "not_found"}
if resolution.get("state") == "complete":
return resolution.get("result") or {"delivered": job.state == "delivered", "state": job.state}
+ from services.plugin.base import NodeWaitInterrupted
from services.temporal.employee_job_workflow import deliver_employee_job
# Only the stored reviewed answer is reconstructed; the owner request
# cannot supply a response, a recipient, a graph, or approval parameters.
@@ -136,6 +137,10 @@ async def continue_delivery(database: Any, job_id: str, resolution_id: str) -> d
{"response": job.result, "team_id": job.team_id, "execution_id": job.origin_execution_id}}}
try:
result = await deliver_employee_job(context)
+ except NodeWaitInterrupted:
+ # The worker stopped in the middle of a wait: this activity's retry
+ # re-attaches; the resolution is not finished.
+ raise
except Exception:
# A failed external activity may already have sent. Its durable
# inflight intent is retained for another explicit owner decision.
diff --git a/server/services/employees/jobs.py b/server/services/employees/jobs.py
index ed8157ff..722c97b5 100644
--- a/server/services/employees/jobs.py
+++ b/server/services/employees/jobs.py
@@ -259,6 +259,12 @@ async def deliver_job(database: Any, ctx: Any, *, lead_node_id: str) -> dict:
team_id = str(lead_output.get("team_id") or ctx.raw.get("team_id") or "")
execution_id = str(lead_output.get("execution_id") or ctx.execution_id or "")
claim = uuid.uuid4().hex
+ # The activity that claims the delivery holds its lease, so its own
+ # retry (a restart in the middle of an approval wait) re-attaches
+ # instead of answering "delivering" and leaving the draft stranded.
+ from temporalio import activity
+ info = activity.info() if activity.in_activity() else None
+ holder = f"{info.workflow_id}:{info.activity_id}" if info else None
async with database.reserved_session() as session:
criteria = EmployeeJob.team_id == team_id if team_id else EmployeeJob.origin_execution_id == execution_id
found = await session.execute(select(EmployeeJob).where(EmployeeJob.workflow_id == ctx.workflow_id,
@@ -270,8 +276,11 @@ async def deliver_job(database: Any, ctx: Any, *, lead_node_id: str) -> dict:
return {"delivered": False, "state": "failed", "job_id": job.id, "failure": job.delivery["failure"]}
if job.state == "delivered":
return {"delivered": True, "state": "delivered", "job_id": job.id}
+ if job.state == "failed":
+ # Never deliver a failed job again (a refused send ends it so).
+ return {"delivered": False, "state": "failed", "job_id": job.id}
lease = job.delivery.get("lease_until")
- if lease and datetime.fromisoformat(lease) > _now():
+ if lease and datetime.fromisoformat(lease) > _now() and not (holder and job.delivery.get("lease_holder") == holder):
return {"delivered": False, "state": "delivering", "job_id": job.id}
inflight = job.delivery.get("inflight")
node_types = {node["id"]: node["type"] for node in job.source["nodes"]}
@@ -316,7 +325,7 @@ async def deliver_job(database: Any, ctx: Any, *, lead_node_id: str) -> dict:
# reattach after a crashed worker rather than treating a waiting gate
# as an uncertain external send.
job.result = job.result or response
- job.delivery = {**job.delivery, "inflight": None, "claim_token": claim,
+ job.delivery = {**job.delivery, "inflight": None, "claim_token": claim, "lease_holder": holder,
"lease_until": (_now() + timedelta(minutes=2)).isoformat()}
job.state = "delivering"
job.updated_at = _now()
@@ -383,7 +392,25 @@ async def deliver_job(database: Any, ctx: Any, *, lead_node_id: str) -> dict:
renewal.cancel()
await asyncio.gather(renewal, return_exceptions=True)
if not result.get("success"):
- raise RuntimeError("The reviewed result could not be delivered; check its saved delivery request")
+ from services.plugin.base import NODE_WAIT_INTERRUPTED, NodeWaitInterrupted
+
+ if result.get("error_type") == NODE_WAIT_INTERRUPTED:
+ # The worker stopped in the middle of a wait: the retry
+ # re-attaches under this lease, and nothing is recorded.
+ raise NodeWaitInterrupted(str(result.get("error") or "The delivery's wait was interrupted"))
+ # A clean refusal ends the job failed, unless a Reset or an Apply
+ # ended it first.
+ async with database.reserved_session() as session:
+ current = await session.get(EmployeeJob, job.id)
+ if current.delivery.get("claim_token") == claim and current.state == "delivering":
+ current.delivery = {**current.delivery, "inflight": None, "claim_token": None, "lease_until": None}
+ current.state = "failed"
+ current.updated_at = _now()
+ await session.commit()
+ state = current.state
+ from services.employees.events import employee_changed_now
+ employee_changed_now(job.workflow_id)
+ return {"delivered": False, "state": state, "job_id": job.id}
restored[node_id] = result.get("result", {})
completed.append(node_id)
async with database.reserved_session() as session:
diff --git a/server/services/temporal/_retry_policies.py b/server/services/temporal/_retry_policies.py
index a137d493..7760f321 100644
--- a/server/services/temporal/_retry_policies.py
+++ b/server/services/temporal/_retry_policies.py
@@ -135,6 +135,22 @@ PERMIT_WAIT_RETRY: RetryPolicy = RetryPolicy(
)
+# Reviewed delivery (``employee.job.deliver`` from EmployeeJobWorkflow and
+# from the lead itself, and ``employee.job.resolve_delivery``) waits inside
+# its activity for the owner's approval, possibly for days, and a worker
+# restart ends that attempt. The next attempt re-attaches to the same
+# delivery under its lease (``services/employees/jobs.deliver_job``), and the
+# write-ahead ``inflight`` mark keeps an external send at most once, so the
+# retries are unlimited; one that keeps failing shows as an attention item
+# (the ``activity_retry`` interceptor, limit 10).
+REVIEWED_DELIVERY_RETRY: RetryPolicy = RetryPolicy(
+ initial_interval=timedelta(seconds=2),
+ maximum_interval=timedelta(minutes=1),
+ maximum_attempts=0,
+ non_retryable_error_types=list(NON_RETRYABLE_ERROR_TYPES),
+)
+
+
# Keep the existing Browser task policies unchanged for Temporal replay.
# A busy claim fails immediately; cleanup must wait for its owner to return.
BROWSER_TASK_CLAIM_RETRY: RetryPolicy = RetryPolicy(maximum_attempts=1)
@@ -166,6 +182,7 @@ __all__ = [
"LLM_STEP_RETRY",
"DELEGATION_CLEANUP_RETRY",
"PERMIT_WAIT_RETRY",
+ "REVIEWED_DELIVERY_RETRY",
"BROWSER_TASK_CLAIM_RETRY",
"BROWSER_TASK_CLEANUP_RETRY",
"frozen_retry_policy",
diff --git a/server/services/temporal/agent_workflow.py b/server/services/temporal/agent_workflow.py
index dbe4fd70..8bef45a0 100644
--- a/server/services/temporal/agent_workflow.py
+++ b/server/services/temporal/agent_workflow.py
@@ -69,6 +69,7 @@ from ._retry_policies import (
DELEGATION_CLEANUP_RETRY,
LLM_STEP_RETRY,
PERMIT_WAIT_RETRY,
+ REVIEWED_DELIVERY_RETRY,
frozen_retry_policy,
)
from .agent_context_pressure import (
@@ -3283,7 +3284,7 @@ class AgentWorkflow:
await workflow.execute_activity("employee.job.deliver", {**context, "team_id": result_payload.get("team_id"),
"outputs": {**(context.get("outputs") or {}), agent_node_id: result_payload}},
activity_id="employee-reviewed-delivery", start_to_close_timeout=timedelta(days=31),
- heartbeat_timeout=timedelta(minutes=2), retry_policy=AGENT_ACTIVITY_RETRY)
+ heartbeat_timeout=timedelta(minutes=2), retry_policy=REVIEWED_DELIVERY_RETRY)
# Final lifecycle broadcast — canvas glow goes green + FE
# consumers of com.opencompany.agent.progress see phase="completed".
diff --git a/server/services/temporal/employee_job_workflow.py b/server/services/temporal/employee_job_workflow.py
index dc62b281..48e061a7 100644
--- a/server/services/temporal/employee_job_workflow.py
+++ b/server/services/temporal/employee_job_workflow.py
@@ -5,6 +5,7 @@ from typing import Any
from temporalio import workflow, activity
from temporalio.common import RetryPolicy
from temporalio.exceptions import is_cancelled_exception
+from services.temporal._retry_policies import REVIEWED_DELIVERY_RETRY
from services.temporal.execution_control import ExecutionControl
@@ -133,10 +134,10 @@ class EmployeeJobWorkflow:
async with self._execution_control.action():
return await workflow.execute_activity("employee.job.deliver", delivery,
start_to_close_timeout=timedelta(days=31), heartbeat_timeout=timedelta(minutes=2),
- retry_policy=RetryPolicy(maximum_attempts=3))
+ retry_policy=REVIEWED_DELIVERY_RETRY)
return await workflow.execute_activity("employee.job.deliver", delivery,
start_to_close_timeout=timedelta(days=31), heartbeat_timeout=timedelta(minutes=2),
- retry_policy=RetryPolicy(maximum_attempts=3))
+ retry_policy=REVIEWED_DELIVERY_RETRY)
@activity.defn(name="employee.job.resolve_delivery")
@@ -154,4 +155,4 @@ class EmployeeJobDeliveryWorkflow:
async def run(self, request: dict[str, Any]) -> dict:
return await workflow.execute_activity("employee.job.resolve_delivery", request,
start_to_close_timeout=timedelta(days=31), heartbeat_timeout=timedelta(minutes=2),
- retry_policy=RetryPolicy(maximum_attempts=3))
+ retry_policy=REVIEWED_DELIVERY_RETRY)
diff --git a/server/tests/services/employees/test_delivery_resolution.py b/server/tests/services/employees/test_delivery_resolution.py
index a64e1a16..cc00d550 100644
--- a/server/tests/services/employees/test_delivery_resolution.py
+++ b/server/tests/services/employees/test_delivery_resolution.py
@@ -99,6 +99,18 @@ async def test_retry_uses_saved_reviewed_response_recipient_and_approval(real_da
assert tail.kwargs["node_id"] == "tail"
+async def test_an_interrupted_wait_is_retried_not_settled(real_database, delivery_runtime):
+ # 1b.9: the activity's retry re-attaches; the resolution stays open.
+ from services.plugin.base import NodeWaitInterrupted
+
+ await install(real_database)
+ review = await decide(real_database)
+ delivery_runtime.execute.return_value = {"success": False, "error": "The worker is stopping", "error_type": "NodeWaitInterrupted"}
+ with pytest.raises(NodeWaitInterrupted):
+ await continue_delivery(real_database, "job", review["request_id"])
+ assert (await saved(real_database)).delivery["resolutions"][review["request_id"]].get("state") != "complete"
+
+
async def test_retry_preserves_declined_approval_and_does_not_send(real_database, delivery_runtime):
await install(real_database, tail=False, approval=False)
review = await decide(real_database)
diff --git a/server/tests/services/employees/test_jobs.py b/server/tests/services/employees/test_jobs.py
index 19785f6b..55e7607a 100644
--- a/server/tests/services/employees/test_jobs.py
+++ b/server/tests/services/employees/test_jobs.py
@@ -164,3 +164,67 @@ async def test_expired_gate_claim_reattaches_and_chat_uid_is_stable(database, ru
assert (await deliver_job(database, context("changed"), lead_node_id="lead"))["delivered"]
assert len(ids) == 2 and ids[0] == ids[1]
assert (await saved(database)).result == "Reviewed result"
+
+
+# 1b.9: reviewed delivery re-attaches to its own lease.
+WAIT_INTERRUPTED = {"success": False, "error": "The worker is stopping", "error_type": "NodeWaitInterrupted"}
+
+
+def attempt(number, *, workflow_id="job", activity_id="deliver"):
+ from dataclasses import replace
+ from temporalio.testing import ActivityEnvironment
+
+ env = ActivityEnvironment()
+ env.info = replace(env.info, workflow_id=workflow_id, activity_id=activity_id, attempt=number)
+ return env
+
+
+async def test_a_retry_of_the_same_delivery_reattaches_within_its_lease(database, runtime):
+ from services.plugin.base import NodeWaitInterrupted
+
+ await install(database)
+ runtime.execute.return_value = WAIT_INTERRUPTED
+ with pytest.raises(NodeWaitInterrupted):
+ await attempt(1).run(deliver_job, database, context(), lead_node_id="lead")
+ row = await saved(database)
+ # The interrupted wait records nothing.
+ assert row.state == "delivering"
+ assert row.delivery["inflight"] == "gate" and row.delivery["completed"] == []
+ assert row.delivery["lease_holder"] == "job:deliver"
+
+ # Another delivery within the lease still waits its turn.
+ other = await attempt(1, workflow_id="job:lead", activity_id="employee-reviewed-delivery").run(
+ deliver_job, database, context(), lead_node_id="lead")
+ assert other["state"] == "delivering"
+
+ runtime.execute.return_value = {"success": True, "result": {"approved": True}}
+ assert (await attempt(2).run(deliver_job, database, context(), lead_node_id="lead"))["delivered"]
+ assert (await saved(database)).state == "delivered"
+
+
+async def test_a_refused_send_fails_the_job_and_a_failed_job_is_never_delivered(database, runtime):
+ await install(database)
+ runtime.execute.return_value = {"success": False, "error": "Telegram refused the message"}
+ assert await attempt(1).run(deliver_job, database, context(), lead_node_id="lead") == {
+ "delivered": False, "state": "failed", "job_id": "job"}
+ row = await saved(database)
+ assert row.state == "failed"
+ assert (row.delivery["inflight"], row.delivery["claim_token"], row.delivery["lease_until"]) == (None, None, None)
+ sends = runtime.execute.await_count
+
+ assert (await attempt(2).run(deliver_job, database, context(), lead_node_id="lead"))["state"] == "failed"
+ assert runtime.execute.await_count == sends
+
+
+async def test_a_refusal_keeps_a_resets_cancelled(database, runtime):
+ await install(database)
+
+ async def refuse_after_reset(**_kwargs):
+ async with database.reserved_session() as session:
+ (await session.get(EmployeeJob, "job")).state = "cancelled"
+ await session.commit()
+ return {"success": False, "error": "refused"}
+
+ runtime.execute.side_effect = refuse_after_reset
+ assert (await deliver_job(database, context(), lead_node_id="lead"))["state"] == "cancelled"
+ assert (await saved(database)).state == "cancelled"
diff --git a/server/tests/test_retry_policies.py b/server/tests/test_retry_policies.py
index 061e7dc4..821dc6c1 100644
--- a/server/tests/test_retry_policies.py
+++ b/server/tests/test_retry_policies.py
@@ -80,6 +80,40 @@ class TestRetryPolicyConstants:
assert BROWSER_TASK_CLEANUP_RETRY == RetryPolicy(maximum_attempts=0)
+class TestReviewedDeliveryRetry:
+ """1b.9: reviewed delivery retries without limit, re-attaching to its lease."""
+
+ def test_the_policy_is_unlimited_and_keeps_the_non_retryable_list(self):
+ from datetime import timedelta
+
+ from services.temporal._retry_policies import NON_RETRYABLE_ERROR_TYPES, REVIEWED_DELIVERY_RETRY
+
+ assert REVIEWED_DELIVERY_RETRY.maximum_attempts == 0
+ assert REVIEWED_DELIVERY_RETRY.initial_interval == timedelta(seconds=2)
+ assert REVIEWED_DELIVERY_RETRY.maximum_interval == timedelta(minutes=1)
+ assert list(REVIEWED_DELIVERY_RETRY.non_retryable_error_types) == list(NON_RETRYABLE_ERROR_TYPES)
+
+ @pytest.mark.parametrize(
+ ("rel_path", "count"),
+ [("services/temporal/employee_job_workflow.py", 3), ("services/temporal/agent_workflow.py", 1)],
+ )
+ def test_every_delivery_call_uses_it(self, rel_path, count):
+ import ast
+ import pathlib
+
+ tree = ast.parse((pathlib.Path(__file__).resolve().parent.parent / rel_path).read_text(encoding="utf-8"))
+ policies = [
+ keyword.value.id
+ for node in ast.walk(tree)
+ if isinstance(node, ast.Call) and getattr(node.func, "attr", None) == "execute_activity"
+ and node.args and isinstance(node.args[0], ast.Constant)
+ and node.args[0].value in {"employee.job.deliver", "employee.job.resolve_delivery"}
+ for keyword in node.keywords
+ if keyword.arg == "retry_policy" and isinstance(keyword.value, ast.Name)
+ ]
+ assert policies == ["REVIEWED_DELIVERY_RETRY"] * count
+
+
class TestNoDuplicatedInlineRetryPolicy:
"""Workflow files must import the shared constant — not re-type
``RetryPolicy(initial_interval=..., maximum_interval=..., maximum_attempts=N)``