Self-improving AI Employees for everyone
Python ai-agentsai-employeeai-employeesassistantautonomous-agentsco-employeescompanygrok-botinstinctloop-agentsmuseown-your-datapersonalself-hosted
fix(employees): reviewed delivery re-attaches to its lease (1b.9)
trohitg0c582e649ea7
diff --git a/docs-internal/node-logic-flows/workflow/employeeJob.md b/docs-internal/node-logic-flows/workflow/employeeJob.mdindex bd29180a..ade35dde 100644--- a/docs-internal/node-logic-flows/workflow/employeeJob.md+++ b/docs-internal/node-logic-flows/workflow/employeeJob.md@@ -14,6 +14,14 @@ Operations:- `deliver`: resolve the originating job by its execution or team identity.Assignment receipts and unaccepted task submissions cannot publish.Accepted work uses the original delivery configuration and approval gates.+ On the team runtime (`EmployeeJobWorkflow`, the lead's own delivery and the+ owner's resolution of an uncertain send) the delivery activity retries+ without limit (`REVIEWED_DELIVERY_RETRY`): a worker restart during the+ approval wait re-attaches to the same delivery under its lease, a sink+ that refuses cleanly ends the job failed, and a failed job is never+ delivered again. In a v1 graph `deliver` runs as the node's own activity+ under the plugin's default policy, 3 attempts of up to 24 h each, so it+ re-attaches only within those.`operation`, `lead_node_id` and `delivery_node_ids` are server-controlledconfiguration. Models can supply only the mission when this node is a tool.diff --git a/server/services/employees/delivery_resolution.py b/server/services/employees/delivery_resolution.pyindex 2f39c64e..af15b150 100644--- a/server/services/employees/delivery_resolution.py+++ b/server/services/employees/delivery_resolution.py@@ -126,6 +126,7 @@ async def continue_delivery(database: Any, job_id: str, resolution_id: str) -> dreturn {"delivered": False, "state": "not_found"}if resolution.get("state") == "complete":return resolution.get("result") or {"delivered": job.state == "delivered", "state": job.state}+ from services.plugin.base import NodeWaitInterruptedfrom services.temporal.employee_job_workflow import deliver_employee_job# Only the stored reviewed answer is reconstructed; the owner request# cannot supply a response, a recipient, a graph, or approval parameters.@@ -136,6 +137,10 @@ async def continue_delivery(database: Any, job_id: str, resolution_id: str) -> d{"response": job.result, "team_id": job.team_id, "execution_id": job.origin_execution_id}}}try:result = await deliver_employee_job(context)+ except NodeWaitInterrupted:+ # The worker stopped in the middle of a wait: this activity's retry+ # re-attaches; the resolution is not finished.+ raiseexcept Exception:# A failed external activity may already have sent. Its durable# inflight intent is retained for another explicit owner decision.diff --git a/server/services/employees/jobs.py b/server/services/employees/jobs.pyindex ed8157ff..722c97b5 100644--- a/server/services/employees/jobs.py+++ b/server/services/employees/jobs.py@@ -259,6 +259,12 @@ async def deliver_job(database: Any, ctx: Any, *, lead_node_id: str) -> dict:team_id = str(lead_output.get("team_id") or ctx.raw.get("team_id") or "")execution_id = str(lead_output.get("execution_id") or ctx.execution_id or "")claim = uuid.uuid4().hex+ # The activity that claims the delivery holds its lease, so its own+ # retry (a restart in the middle of an approval wait) re-attaches+ # instead of answering "delivering" and leaving the draft stranded.+ from temporalio import activity+ info = activity.info() if activity.in_activity() else None+ holder = f"{info.workflow_id}:{info.activity_id}" if info else Noneasync with database.reserved_session() as session:criteria = EmployeeJob.team_id == team_id if team_id else EmployeeJob.origin_execution_id == execution_idfound = await session.execute(select(EmployeeJob).where(EmployeeJob.workflow_id == ctx.workflow_id,@@ -270,8 +276,11 @@ async def deliver_job(database: Any, ctx: Any, *, lead_node_id: str) -> dict:return {"delivered": False, "state": "failed", "job_id": job.id, "failure": job.delivery["failure"]}if job.state == "delivered":return {"delivered": True, "state": "delivered", "job_id": job.id}+ if job.state == "failed":+ # Never deliver a failed job again (a refused send ends it so).+ return {"delivered": False, "state": "failed", "job_id": job.id}lease = job.delivery.get("lease_until")- if lease and datetime.fromisoformat(lease) > _now():+ if lease and datetime.fromisoformat(lease) > _now() and not (holder and job.delivery.get("lease_holder") == holder):return {"delivered": False, "state": "delivering", "job_id": job.id}inflight = job.delivery.get("inflight")node_types = {node["id"]: node["type"] for node in job.source["nodes"]}@@ -316,7 +325,7 @@ async def deliver_job(database: Any, ctx: Any, *, lead_node_id: str) -> dict:# reattach after a crashed worker rather than treating a waiting gate# as an uncertain external send.job.result = job.result or response- job.delivery = {**job.delivery, "inflight": None, "claim_token": claim,+ job.delivery = {**job.delivery, "inflight": None, "claim_token": claim, "lease_holder": holder,"lease_until": (_now() + timedelta(minutes=2)).isoformat()}job.state = "delivering"job.updated_at = _now()@@ -383,7 +392,25 @@ async def deliver_job(database: Any, ctx: Any, *, lead_node_id: str) -> dict:renewal.cancel()await asyncio.gather(renewal, return_exceptions=True)if not result.get("success"):- raise RuntimeError("The reviewed result could not be delivered; check its saved delivery request")+ from services.plugin.base import NODE_WAIT_INTERRUPTED, NodeWaitInterrupted++ if result.get("error_type") == NODE_WAIT_INTERRUPTED:+ # The worker stopped in the middle of a wait: the retry+ # re-attaches under this lease, and nothing is recorded.+ raise NodeWaitInterrupted(str(result.get("error") or "The delivery's wait was interrupted"))+ # A clean refusal ends the job failed, unless a Reset or an Apply+ # ended it first.+ async with database.reserved_session() as session:+ current = await session.get(EmployeeJob, job.id)+ if current.delivery.get("claim_token") == claim and current.state == "delivering":+ current.delivery = {**current.delivery, "inflight": None, "claim_token": None, "lease_until": None}+ current.state = "failed"+ current.updated_at = _now()+ await session.commit()+ state = current.state+ from services.employees.events import employee_changed_now+ employee_changed_now(job.workflow_id)+ return {"delivered": False, "state": state, "job_id": job.id}restored[node_id] = result.get("result", {})completed.append(node_id)async with database.reserved_session() as session:diff --git a/server/services/temporal/_retry_policies.py b/server/services/temporal/_retry_policies.pyindex a137d493..7760f321 100644--- a/server/services/temporal/_retry_policies.py+++ b/server/services/temporal/_retry_policies.py@@ -135,6 +135,22 @@ PERMIT_WAIT_RETRY: RetryPolicy = RetryPolicy()+# Reviewed delivery (``employee.job.deliver`` from EmployeeJobWorkflow and+# from the lead itself, and ``employee.job.resolve_delivery``) waits inside+# its activity for the owner's approval, possibly for days, and a worker+# restart ends that attempt. The next attempt re-attaches to the same+# delivery under its lease (``services/employees/jobs.deliver_job``), and the+# write-ahead ``inflight`` mark keeps an external send at most once, so the+# retries are unlimited; one that keeps failing shows as an attention item+# (the ``activity_retry`` interceptor, limit 10).+REVIEWED_DELIVERY_RETRY: RetryPolicy = RetryPolicy(+ initial_interval=timedelta(seconds=2),+ maximum_interval=timedelta(minutes=1),+ maximum_attempts=0,+ non_retryable_error_types=list(NON_RETRYABLE_ERROR_TYPES),+)++# Keep the existing Browser task policies unchanged for Temporal replay.# A busy claim fails immediately; cleanup must wait for its owner to return.BROWSER_TASK_CLAIM_RETRY: RetryPolicy = RetryPolicy(maximum_attempts=1)@@ -166,6 +182,7 @@ __all__ = ["LLM_STEP_RETRY","DELEGATION_CLEANUP_RETRY","PERMIT_WAIT_RETRY",+ "REVIEWED_DELIVERY_RETRY","BROWSER_TASK_CLAIM_RETRY","BROWSER_TASK_CLEANUP_RETRY","frozen_retry_policy",diff --git a/server/services/temporal/agent_workflow.py b/server/services/temporal/agent_workflow.pyindex dbe4fd70..8bef45a0 100644--- a/server/services/temporal/agent_workflow.py+++ b/server/services/temporal/agent_workflow.py@@ -69,6 +69,7 @@ from ._retry_policies import (DELEGATION_CLEANUP_RETRY,LLM_STEP_RETRY,PERMIT_WAIT_RETRY,+ REVIEWED_DELIVERY_RETRY,frozen_retry_policy,)from .agent_context_pressure import (@@ -3283,7 +3284,7 @@ class AgentWorkflow:await workflow.execute_activity("employee.job.deliver", {**context, "team_id": result_payload.get("team_id"),"outputs": {**(context.get("outputs") or {}), agent_node_id: result_payload}},activity_id="employee-reviewed-delivery", start_to_close_timeout=timedelta(days=31),- heartbeat_timeout=timedelta(minutes=2), retry_policy=AGENT_ACTIVITY_RETRY)+ heartbeat_timeout=timedelta(minutes=2), retry_policy=REVIEWED_DELIVERY_RETRY)# Final lifecycle broadcast — canvas glow goes green + FE# consumers of com.opencompany.agent.progress see phase="completed".diff --git a/server/services/temporal/employee_job_workflow.py b/server/services/temporal/employee_job_workflow.pyindex dc62b281..48e061a7 100644--- a/server/services/temporal/employee_job_workflow.py+++ b/server/services/temporal/employee_job_workflow.py@@ -5,6 +5,7 @@ from typing import Anyfrom temporalio import workflow, activityfrom temporalio.common import RetryPolicyfrom temporalio.exceptions import is_cancelled_exception+from services.temporal._retry_policies import REVIEWED_DELIVERY_RETRYfrom services.temporal.execution_control import ExecutionControl@@ -133,10 +134,10 @@ class EmployeeJobWorkflow:async with self._execution_control.action():return await workflow.execute_activity("employee.job.deliver", delivery,start_to_close_timeout=timedelta(days=31), heartbeat_timeout=timedelta(minutes=2),- retry_policy=RetryPolicy(maximum_attempts=3))+ retry_policy=REVIEWED_DELIVERY_RETRY)return await workflow.execute_activity("employee.job.deliver", delivery,start_to_close_timeout=timedelta(days=31), heartbeat_timeout=timedelta(minutes=2),- retry_policy=RetryPolicy(maximum_attempts=3))+ retry_policy=REVIEWED_DELIVERY_RETRY)@activity.defn(name="employee.job.resolve_delivery")@@ -154,4 +155,4 @@ class EmployeeJobDeliveryWorkflow:async def run(self, request: dict[str, Any]) -> dict:return await workflow.execute_activity("employee.job.resolve_delivery", request,start_to_close_timeout=timedelta(days=31), heartbeat_timeout=timedelta(minutes=2),- retry_policy=RetryPolicy(maximum_attempts=3))+ retry_policy=REVIEWED_DELIVERY_RETRY)diff --git a/server/tests/services/employees/test_delivery_resolution.py b/server/tests/services/employees/test_delivery_resolution.pyindex a64e1a16..cc00d550 100644--- a/server/tests/services/employees/test_delivery_resolution.py+++ b/server/tests/services/employees/test_delivery_resolution.py@@ -99,6 +99,18 @@ async def test_retry_uses_saved_reviewed_response_recipient_and_approval(real_daassert tail.kwargs["node_id"] == "tail"+async def test_an_interrupted_wait_is_retried_not_settled(real_database, delivery_runtime):+ # 1b.9: the activity's retry re-attaches; the resolution stays open.+ from services.plugin.base import NodeWaitInterrupted++ await install(real_database)+ review = await decide(real_database)+ delivery_runtime.execute.return_value = {"success": False, "error": "The worker is stopping", "error_type": "NodeWaitInterrupted"}+ with pytest.raises(NodeWaitInterrupted):+ await continue_delivery(real_database, "job", review["request_id"])+ assert (await saved(real_database)).delivery["resolutions"][review["request_id"]].get("state") != "complete"++async def test_retry_preserves_declined_approval_and_does_not_send(real_database, delivery_runtime):await install(real_database, tail=False, approval=False)review = await decide(real_database)diff --git a/server/tests/services/employees/test_jobs.py b/server/tests/services/employees/test_jobs.pyindex 19785f6b..55e7607a 100644--- a/server/tests/services/employees/test_jobs.py+++ b/server/tests/services/employees/test_jobs.py@@ -164,3 +164,67 @@ async def test_expired_gate_claim_reattaches_and_chat_uid_is_stable(database, ruassert (await deliver_job(database, context("changed"), lead_node_id="lead"))["delivered"]assert len(ids) == 2 and ids[0] == ids[1]assert (await saved(database)).result == "Reviewed result"+++# 1b.9: reviewed delivery re-attaches to its own lease.+WAIT_INTERRUPTED = {"success": False, "error": "The worker is stopping", "error_type": "NodeWaitInterrupted"}+++def attempt(number, *, workflow_id="job", activity_id="deliver"):+ from dataclasses import replace+ from temporalio.testing import ActivityEnvironment++ env = ActivityEnvironment()+ env.info = replace(env.info, workflow_id=workflow_id, activity_id=activity_id, attempt=number)+ return env+++async def test_a_retry_of_the_same_delivery_reattaches_within_its_lease(database, runtime):+ from services.plugin.base import NodeWaitInterrupted++ await install(database)+ runtime.execute.return_value = WAIT_INTERRUPTED+ with pytest.raises(NodeWaitInterrupted):+ await attempt(1).run(deliver_job, database, context(), lead_node_id="lead")+ row = await saved(database)+ # The interrupted wait records nothing.+ assert row.state == "delivering"+ assert row.delivery["inflight"] == "gate" and row.delivery["completed"] == []+ assert row.delivery["lease_holder"] == "job:deliver"++ # Another delivery within the lease still waits its turn.+ other = await attempt(1, workflow_id="job:lead", activity_id="employee-reviewed-delivery").run(+ deliver_job, database, context(), lead_node_id="lead")+ assert other["state"] == "delivering"++ runtime.execute.return_value = {"success": True, "result": {"approved": True}}+ assert (await attempt(2).run(deliver_job, database, context(), lead_node_id="lead"))["delivered"]+ assert (await saved(database)).state == "delivered"+++async def test_a_refused_send_fails_the_job_and_a_failed_job_is_never_delivered(database, runtime):+ await install(database)+ runtime.execute.return_value = {"success": False, "error": "Telegram refused the message"}+ assert await attempt(1).run(deliver_job, database, context(), lead_node_id="lead") == {+ "delivered": False, "state": "failed", "job_id": "job"}+ row = await saved(database)+ assert row.state == "failed"+ assert (row.delivery["inflight"], row.delivery["claim_token"], row.delivery["lease_until"]) == (None, None, None)+ sends = runtime.execute.await_count++ assert (await attempt(2).run(deliver_job, database, context(), lead_node_id="lead"))["state"] == "failed"+ assert runtime.execute.await_count == sends+++async def test_a_refusal_keeps_a_resets_cancelled(database, runtime):+ await install(database)++ async def refuse_after_reset(**_kwargs):+ async with database.reserved_session() as session:+ (await session.get(EmployeeJob, "job")).state = "cancelled"+ await session.commit()+ return {"success": False, "error": "refused"}++ runtime.execute.side_effect = refuse_after_reset+ assert (await deliver_job(database, context(), lead_node_id="lead"))["state"] == "cancelled"+ assert (await saved(database)).state == "cancelled"diff --git a/server/tests/test_retry_policies.py b/server/tests/test_retry_policies.pyindex 061e7dc4..821dc6c1 100644--- a/server/tests/test_retry_policies.py+++ b/server/tests/test_retry_policies.py@@ -80,6 +80,40 @@ class TestRetryPolicyConstants:assert BROWSER_TASK_CLEANUP_RETRY == RetryPolicy(maximum_attempts=0)+class TestReviewedDeliveryRetry:+ """1b.9: reviewed delivery retries without limit, re-attaching to its lease."""++ def test_the_policy_is_unlimited_and_keeps_the_non_retryable_list(self):+ from datetime import timedelta++ from services.temporal._retry_policies import NON_RETRYABLE_ERROR_TYPES, REVIEWED_DELIVERY_RETRY++ assert REVIEWED_DELIVERY_RETRY.maximum_attempts == 0+ assert REVIEWED_DELIVERY_RETRY.initial_interval == timedelta(seconds=2)+ assert REVIEWED_DELIVERY_RETRY.maximum_interval == timedelta(minutes=1)+ assert list(REVIEWED_DELIVERY_RETRY.non_retryable_error_types) == list(NON_RETRYABLE_ERROR_TYPES)++ @pytest.mark.parametrize(+ ("rel_path", "count"),+ [("services/temporal/employee_job_workflow.py", 3), ("services/temporal/agent_workflow.py", 1)],+ )+ def test_every_delivery_call_uses_it(self, rel_path, count):+ import ast+ import pathlib++ tree = ast.parse((pathlib.Path(__file__).resolve().parent.parent / rel_path).read_text(encoding="utf-8"))+ policies = [+ keyword.value.id+ for node in ast.walk(tree)+ if isinstance(node, ast.Call) and getattr(node.func, "attr", None) == "execute_activity"+ and node.args and isinstance(node.args[0], ast.Constant)+ and node.args[0].value in {"employee.job.deliver", "employee.job.resolve_delivery"}+ for keyword in node.keywords+ if keyword.arg == "retry_policy" and isinstance(keyword.value, ast.Name)+ ]+ assert policies == ["REVIEWED_DELIVERY_RETRY"] * count++class TestNoDuplicatedInlineRetryPolicy:"""Workflow files must import the shared constant — not re-type``RetryPolicy(initial_interval=..., maximum_interval=..., maximum_attempts=N)``