Skip to content

Self-improving AI Employees for everyone

fix(approvals): an agent's send goes out at most once

trohitg93e7f63bbd00
diff --git a/server/services/approvals/tool_calls.py b/server/services/approvals/tool_calls.py
index f81d5a1c..a825fdee 100644
--- a/server/services/approvals/tool_calls.py
+++ b/server/services/approvals/tool_calls.py
@@ -17,6 +17,15 @@ A call that comes back to run because the owner pressed Send carries
``approval_execution`` (services/approvals/execution.py): it runs only when
its row is the one being sent, and is never held again.
+**At most once.** Temporal runs a tool activity again only when an attempt
+broke off (the worker stopped, a timeout): a failure the node reports comes
+back as a result and is not retried. The attempt that broke off may have
+sent the message, so a later attempt of a call that sends does not send it
+again (:func:`resend_refusal`): the model reads that it may have gone out,
+and with Ask first off the chat's record says so (``outcome: unknown``). The
+owner's approved sends already run with no retry
+(services/temporal/approved_tool_call_workflow.py).
+
What a node sends as (its locked ``server_controlled_fields``: the account,
the mailbox) never comes from the model, so the row keeps the call's
arguments without them; the node's own settings supply them when it runs.
@@ -46,6 +55,10 @@ REFUSED_MESSAGE = (
"Not run: {channel} cannot wait for the owner's OK, and they ask to approve everything first. "
"Tell them what you would do; they can turn Ask first off to let you."
)
+RESEND_MESSAGE = (
+ "Not sent again: the first try at this broke off, and it may already have gone out. "
+ "Do not send it again unless the owner asks; tell them it may or may not have gone out."
+)
@dataclass
@@ -257,9 +270,11 @@ async def _show_in_chat(database: Any, context: Mapping[str, Any], row: Any) ->
def _auto_recorder(database: Any, context: Mapping[str, Any], node_cls: Any, spec: Any, node_data: Mapping[str, Any]):
- async def record(result: Dict[str, Any]) -> None:
+ async def record(result: Dict[str, Any], *, unknown: bool = False) -> None:
+ """Record how the call went; ``unknown``: it may have gone out
+ (an attempt broke off, see :func:`resend_refusal`)."""
try:
- success = bool(isinstance(result, dict) and result.get("success", True) is not False)
+ success = not unknown and bool(isinstance(result, dict) and result.get("success", True) is not False)
now = _utcnow()
fields = _row_fields(context, node_cls, spec, node_data)
fields.update(
@@ -267,7 +282,7 @@ def _auto_recorder(database: Any, context: Mapping[str, Any], node_cls: Any, spe
approved_by="auto",
decided_at=now,
consumed_at=now,
- outcome="sent" if success else "not_sent",
+ outcome="sent" if success else ("unknown" if unknown else "not_sent"),
outcome_error=None if success else str((result or {}).get("error") or "It did not go out.")[:1000],
outcome_at=now,
final_text=fields.get("draft_text"),
@@ -282,8 +297,44 @@ def _auto_recorder(database: Any, context: Mapping[str, Any], node_cls: Any, spe
return record
+def _attempt() -> int:
+ """This activity attempt's number (1 outside an activity)."""
+ try:
+ from temporalio import activity
+
+ return int(activity.info().attempt)
+ except Exception: # noqa: BLE001 - not in an activity
+ return 1
+
+
+def resend_refusal(context: Mapping[str, Any], node_cls: Any, node_data: Mapping[str, Any]) -> Optional[Dict[str, Any]]:
+ """What a later attempt of an agent's call that sends answers instead
+ of sending (see the module docstring), or None to run it."""
+ from services.plugin.approval import approval_spec
+
+ spec = approval_spec(node_cls)
+ if spec is None or _attempt() <= 1:
+ return None
+ if context.get("approval_execution") is None and not is_agent_tool_call(context):
+ return None
+ if not spec.sends(node_data):
+ return None
+ return {"success": False, "error": RESEND_MESSAGE, "error_type": "SendOutcomeUnknown"}
+
+
def claim_token() -> str:
return uuid.uuid4().hex
-__all__ = ["Checked", "HELD_MESSAGE", "RUN", "call_key", "check", "claim_token", "hold", "is_agent_tool_call"]
+__all__ = [
+ "Checked",
+ "HELD_MESSAGE",
+ "RESEND_MESSAGE",
+ "RUN",
+ "call_key",
+ "check",
+ "claim_token",
+ "hold",
+ "is_agent_tool_call",
+ "resend_refusal",
+]
diff --git a/server/services/plugin/base.py b/server/services/plugin/base.py
index 577610bf..30c74956 100644
--- a/server/services/plugin/base.py
+++ b/server/services/plugin/base.py
@@ -1159,6 +1159,27 @@ class BaseNode:
node_data = checked.node_data
if checked.tool_args is not None:
context = {**context, "tool_args": checked.tool_args}
+ # A send runs at most once: a later attempt (the one before broke
+ # off) answers that it may have gone out instead of sending again.
+ resend = approval_calls.resend_refusal(context, cls, node_data)
+ if resend is not None:
+ result = {
+ "node_id": node_id,
+ "node_type": cls.type,
+ "execution_id": execution_id,
+ "timestamp": datetime.now().isoformat(),
+ **resend,
+ }
+ if checked.record is not None:
+ await checked.record(result, unknown=True)
+ await chat_steps.end(chat_step, state="failed", detail="It may already have gone out")
+ await broadcaster.update_node_status(
+ node_id,
+ "error",
+ {"error": resend["error"], "execution_id": execution_id},
+ workflow_id=workflow_id,
+ )
+ return result
# Broadcast executing — UI cyan-glow.
await broadcaster.update_node_status(
diff --git a/server/tests/services/approvals/test_held_activity.py b/server/tests/services/approvals/test_held_activity.py
index 6c1f7a72..b08f5bb8 100644
--- a/server/tests/services/approvals/test_held_activity.py
+++ b/server/tests/services/approvals/test_held_activity.py
@@ -1,10 +1,12 @@
"""A sending tool call through the node's own Temporal activity
(``BaseNode.as_activity``): held, it never reaches the node and the model
reads that it waits; with Ask first off it runs and is recorded; a send
-the owner approved runs under its claim."""
+the owner approved runs under its claim; a later attempt (the one before
+broke off) never sends again."""
from __future__ import annotations
+import dataclasses
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock
@@ -89,3 +91,53 @@ async def test_an_approved_send_runs_only_under_its_claim(harness, service):
stale = await ActivityEnvironment().run(activity_fn, node_context(sending, "someone-else"))
assert stale["success"] is False and stale["error_type"] == "ApprovalNotClaimed"
service.execute_node.assert_awaited_once()
+
+
+def attempt(number: int) -> ActivityEnvironment:
+ env = ActivityEnvironment()
+ env.info = dataclasses.replace(env.info, attempt=number)
+ return env
+
+
+async def test_a_later_attempt_does_not_send_again(harness, service):
+ await rules.set_ask_first(harness.database, "wf", False)
+ result = await attempt(2).run(get_node_class("whatsappSend").as_activity(), context())
+ service.execute_node.assert_not_awaited()
+ assert result["success"] is False and result["error_type"] == "SendOutcomeUnknown"
+ assert "may already have gone out" in result["error"]
+ assert ("wf:whatsappSend:1", "error") in [(node_id, status) for node_id, status, *_ in harness.broadcaster.statuses]
+
+
+async def test_a_later_attempt_in_the_chat_is_recorded_as_unknown(harness, service, monkeypatch):
+ from services.chat import parts
+
+ async def show_approval(database, stream, **_):
+ return None
+
+ monkeypatch.setattr(parts, "show_approval", show_approval)
+ await rules.set_ask_first(harness.database, "wf", False)
+ stream = {"run_id": "r_1", "session_id": "wf", "workflow_id": "wf"}
+ await attempt(3).run(get_node_class("whatsappSend").as_activity(), context(chat_stream=stream))
+ service.execute_node.assert_not_awaited()
+ (row,) = await store.list_approvals(harness.database, status="failed")
+ assert (row.approved_by, row.outcome) == ("auto", "unknown")
+
+
+async def test_a_later_attempt_still_holds_or_runs_what_does_not_send(harness, service):
+ # A draft is made once per call, so holding it again is safe.
+ await rules.set_ask_first(harness.database, "wf", True)
+ held = await attempt(2).run(get_node_class("whatsappSend").as_activity(), context())
+ assert held["result"]["status"] == "waiting_for_owner"
+ # A call that does not send (a search) retries as before.
+ search = {"operation": "search", "query": "invoice"}
+ reading = context(node_id="wf:googleGmail:1", node_data=dict(search), tool_args=dict(search))
+ result = await attempt(2).run(get_node_class("googleGmail").as_activity(), reading)
+ service.execute_node.assert_awaited_once()
+ assert result["success"] is True
+
+
+async def test_the_first_attempt_sends_as_before(harness, service):
+ await rules.set_ask_first(harness.database, "wf", False)
+ result = await attempt(1).run(get_node_class("whatsappSend").as_activity(), context())
+ service.execute_node.assert_awaited_once()
+ assert result["success"] is True