Self-improving AI Employees for everyone
Python ai-agentsai-employeeai-employeesassistantautonomous-agentsco-employeescompanygrok-botinstinctloop-agentsmuseown-your-datapersonalself-hosted
feat(approvals): an approved send waits while its employee is stopped (1b.5, with 1b.6)
trohitg32c082de1732
diff --git a/client/src/features/chat/__tests__/approvals.test.tsx b/client/src/features/chat/__tests__/approvals.test.tsxindex 1ff01bd2..8d5e7ee0 100644--- a/client/src/features/chat/__tests__/approvals.test.tsx+++ b/client/src/features/chat/__tests__/approvals.test.tsx@@ -79,7 +79,15 @@ describe('the card says what happened', () => {it('a gate’s draft for a paused employee goes on Resume', () => {const paused = view({ kind: 'gate', status: 'approved', undo_until: '2026-10-04T08:59:59Z', deployment_state: 'paused' });- expect(paused.footer).toEqual({ kind: 'waiting', text: 'Sends when you resume Maya.' });+ expect(paused.footer).toEqual({ kind: 'waiting', text: 'Sends when you resume Maya.', undo: false });+ });++ it('a held draft waits for Resume too, and can still be undone', () => {+ for (const state of ['paused', 'pausing', 'resuming']) {+ const paused = view({ status: 'approved', undo_until: '2026-10-04T08:59:59Z', deployment_state: state });+ expect(paused.footer).toEqual({ kind: 'waiting', text: 'Sends when you resume Maya.', undo: true });+ }+ expect(view({ status: 'approved', undo_until: '2026-10-04T08:59:59Z', deployment_state: 'running' }).footer).toEqual({ kind: 'sending' });});it('says it went only once it went', () => {@@ -144,7 +152,7 @@ describe('the card says what happened', () => {// A retry makes it a draft again.expect(optimistic(approval({ waits: true, status: 'failed' }), 'retry', NOW, 0)).toMatchObject({ waits: false, undoUntil: '2026-10-04T09:00:05.000Z' });const paused = view({ waits: true, status: 'approved', deployment_state: 'paused' });- expect(paused.footer).toEqual({ kind: 'waiting', text: 'Sends when you resume Maya.' });+ expect(paused.footer).toEqual({ kind: 'waiting', text: 'Sends when you resume Maya.', undo: false });});it('reads the list with the server’s clock, and moves at once on a decision', () => {@@ -297,6 +305,15 @@ describe('a draft in the chat', () => {expect(server.decisions[0].decision_key).not.toBe(server.decisions[1].decision_key);});+ it('a draft waiting for Resume can be undone', async () => {+ server.approvals = [summary({ status: 'approved', undo_until: '2026-01-01T00:00:00Z', deployment_state: 'paused', revision: 1 })];+ renderPane();+ const card = (await screen.findByText('Sends when you resume Maya.')).closest('[data-approval]') as HTMLElement;+ fireEvent.click(within(card).getByRole('button', { name: /^Undo$/ }));+ await waitFor(() => expect(server.decisions[0]).toMatchObject({ approval_id: 'ap1', decision: 'undo' }));+ expect(await within(card).findByRole('button', { name: /^Send$/ })).toBeInTheDocument();+ });+it('allows a call that has no message as it is, showing what it runs with', async () => {server.approvals = [summary({diff --git a/client/src/features/chat/approval/ApprovalCard.tsx b/client/src/features/chat/approval/ApprovalCard.tsxindex e87536b5..bccbb95e 100644--- a/client/src/features/chat/approval/ApprovalCard.tsx+++ b/client/src/features/chat/approval/ApprovalCard.tsx@@ -305,7 +305,13 @@ function Footer({return (<div className={cn(bar, 'bg-bg-app text-xs text-fg-muted')} role="status"><Pause aria-hidden className="size-3.5" />- {footer.text}+ <span className="flex-1">{footer.text}</span>+ {footer.undo && (+ <Button variant="outline" size="sm" disabled={busy} onClick={() => decide({ approval, decision: 'undo' })} className="gap-1.5 font-semibold">+ <Undo2 aria-hidden className="size-3.25" />+ Undo+ </Button>+ )}</div>);}diff --git a/client/src/features/chat/approval/view.ts b/client/src/features/chat/approval/view.tsindex c0cbc8bc..32b4e3a6 100644--- a/client/src/features/chat/approval/view.ts+++ b/client/src/features/chat/approval/view.ts@@ -32,8 +32,8 @@ export type CardFooter =| { kind: 'decide' }| { kind: 'undo'; seconds: number }| { kind: 'sending' }- /** Approved, and its run waits for Resume before it goes. */- | { kind: 'waiting'; text: string }+ /** Approved, and it waits for Resume before it goes; a draft can still be undone. */+ | { kind: 'waiting'; text: string; undo: boolean }| { kind: 'sent'; text: string }/** Declined, with what the owner said, if anything. */| { kind: 'discarded'; text: string; note: string | null; restoreSeconds: number | null; canRestore: boolean }@@ -96,12 +96,14 @@ export function approvalView(approval: ChatApproval, { name, nowMs, offsetMs, edcase 'approved': {if (approval.consumedAt === null) {const seconds = secondsLeft(approval.undoUntil, offsetMs, nowMs);- // Its run waits for Resume: a gate's draft, or a call its turn waits on.+ // Nothing goes while its employee is stopped: it waits for Resume. A+ // draft can be undone until its send starts; a call its turn waits+ // on cannot.const paused =- (approval.kind === 'gate' || approval.waits) && (approval.deploymentState === 'paused' || approval.deploymentState === 'pausing');+ approval.deploymentState === 'paused' || approval.deploymentState === 'pausing' || approval.deploymentState === 'resuming';const footer: CardFooter =seconds > 0 ? { kind: 'undo', seconds }- : paused ? { kind: 'waiting', text: `${message ? 'Sends' : 'Runs'} when you resume ${name}.` }+ : paused ? { kind: 'waiting', text: `${message ? 'Sends' : 'Runs'} when you resume ${name}.`, undo: approval.kind === 'tool_call' && !approval.waits }: { kind: 'sending' };return {title: `${name} is ${message ? 'sending' : 'doing'} it`,diff --git a/docs-internal/chat_protocol.md b/docs-internal/chat_protocol.mdindex 9e4ee234..c97a6a55 100644--- a/docs-internal/chat_protocol.md+++ b/docs-internal/chat_protocol.md@@ -173,7 +173,8 @@ and workflows answer; any other id reads as `not_found`. `decide_approval` errorbroke off), `approval_conflict`, `send_unavailable` (nothing could start the send; the draft is back as it was),`not_found`, `invalid_request` (also for `always` on a card that offers no rule, and a `note` over 50,000characters). The same `decision_key` again answers `idempotent` with the row as it is. `note` goes with `discard`;-`always` with `send` saves the rule the card offers (`always` in its summary).+`always` with `send` saves the rule the card offers (`always` in its summary). `will_send_on_resume` is true for an+approved row whose employee is stopped (generation above 0): it goes after Resume.**ApprovalSummary**: `{approval_id, workflow_id, node_id, kind: "gate" | "tool_call" | "effect", status, channel,channel_label, action?, recipient, recipient_label, body, subject?, context_excerpt?, details: [{label, value}],@@ -564,8 +565,9 @@ What an employee sends to someone waits for the owner's OK while its workflow as`tool_calls.check_in_process` applies the same rule without drafts: with Ask first on, a call that sends is not runand the model reads why, the browser runs read-only, and nothing is recorded.- **Decisions** (`services/approvals/decisions.py`), each a compare-and-swap on the row's revision recorded in- `approval_decisions`: `send` (pending -> approved; it goes after `UNDO_SECONDS`, 5, and Undo works until then),- `undo` (approved -> pending), `discard` (pending -> discarded; Restore works for `RESTORE_SECONDS` on a gate's draft,+ `approval_decisions`: `send` (pending -> approved; it goes after `UNDO_SECONDS`, 5, and Undo works until then; a+ held call's Undo also works past that while its send is not claimed, as while it waits for Resume), `undo`+ (approved -> pending), `discard` (pending -> discarded; Restore works for `RESTORE_SECONDS` on a gate's draft,whose run waits that long, and until expiry for a held call), `restore`, `retry` (failed -> approved; when thesend broke off it may have gone, so the owner confirms) and `dismiss` (an `effect` row: failed -> cancelled). A gate waits through the Undo and Restore windows beforeit lets the draft through or not.@@ -573,8 +575,14 @@ What an employee sends to someone waits for the owner's OK while its workflow asthe Undo window closes, claims the row at that revision (`approvals.claim_send`: approved -> sending; an Undo, anewer Send or a cancel moved it, and nothing runs), runs the node's own activity once with no retry, and recordsthe outcome (`approvals.record_outcome`): `sent`, `not_sent`, or `unknown` when the activity broke off. The node's- activity runs only under that claim. The first time the drafts are listed after a start, sends that never started- are started again and sends that never reported end `failed` with an unknown outcome.+ activity runs only under that claim.+ - **Nothing goes while its employee is stopped** (`execution.send_state`). The claim refuses (`stopped`) while the+ generation is pausing, paused or resuming, and the row stays approved.+ - **The chat watchdog's round** (`reconcile.recover_sends`, every round) starts such a send again once its+ employee runs, and so a send that never started.+ - **A send whose generation ended** (a Reset, a failure, a newer generation) is cancelled, and the employee told.+ A manual run's send (generation 0) is not held.+ - **A send that never reported** ends `failed` with an unknown outcome.- **At most once.** Temporal runs an activity again only when an attempt broke off (the worker stopped, a timeout);a failure the node reports comes back as a result and is not retried. A later attempt runs only a call that issafe to repeat (`services/approvals/repeats.py`: reads, waits, keyed calls, operations that do the same thing@@ -599,8 +607,8 @@ What an employee sends to someone waits for the owner's OK while its workflow as`ledger.ui_part_of_run`), and the card says "Linked to the form above". Drafts no reply in view made (agate's, from the employee's own work) show after the conversation. The card (`features/chat/approval/`): Discard /Edit / Send while it waits, "Sends in Ns" with Undo, Sending, Sent (and when, and whether Ask first was off),- Restore while it can, Try again (asking first when it may have gone), and "Sends when you resume" for a gate's- draft whose employee is paused. An `effect` card has no recipient and no message: "{name} tried to …", May have+ Restore while it can, Try again (asking first when it may have gone), and "Sends when you resume" for an+ approved draft whose employee is stopped (with Undo on a held call's). An `effect` card has no recipient and no message: "{name} tried to …", May havehappened, and Dismiss only (Dismissed afterwards). Countdowns use the server's clock (`server_time`).Ctrl/Cmd+Enter sends the newest waiting draft; in the edit box it sends that one.- **Ends.** Clearing or resetting the chat cancels the drafts its runs made that still wait, and their failed sends,diff --git a/docs-internal/normal_mode.md b/docs-internal/normal_mode.mdindex 03f770fb..01259653 100644--- a/docs-internal/normal_mode.md+++ b/docs-internal/normal_mode.md@@ -913,6 +913,9 @@ of running; both show as cards in the employee's chat, on the reply that madethem or after the conversation. Send goes after a 5-second Undo window, adiscarded draft can be restored, and a held call is sent once by its ownTemporal workflow, which tells the employee how it went on its next turn.+Nothing goes while the employee is stopped: an approved draft waits for+Resume (a held call can be undone until then), and one whose generation+ended is cancelled.With it off, replies and calls go at once and the chat still shows what went.The gate stores the draft in `approval_requests`, wakes the moment it isdecided, survives restarts (its idempotency key finds the same row on everydiff --git a/server/services/approvals/decisions.py b/server/services/approvals/decisions.pyindex 7d30ec91..f272e0ca 100644--- a/server/services/approvals/decisions.py+++ b/server/services/approvals/decisions.py@@ -174,7 +174,12 @@ async def decide(await _save_rule(database, moved, scope, actor)elif decision == "undo":grace = store.aware(row.grace_until)- if row.status != "approved" or row.consumed_at is not None or row.claim_token or grace is None or grace <= now:+ # A draft can be undone until its send is claimed, as while it waits+ # for Resume; a gate's draft only within its grace, and a call its+ # turn waits on (no grace) never. The claim's compare-and-swap settles+ # a race with Undo.+ draft = kind == "tool_call" and not turn+ if row.status != "approved" or row.consumed_at is not None or row.claim_token or grace is None or (grace <= now and not draft):raise DecisionRefused("too_late", "It has already gone.")values = {"grace_until": None, "decided_at": None, "approved_by": None, "final_text": None, "final_subject": None, "edited": False}moved = await _move(database, row, ("approved",), "pending", values, record)diff --git a/server/services/approvals/execution.py b/server/services/approvals/execution.pyindex cb73a703..337214bc 100644--- a/server/services/approvals/execution.py+++ b/server/services/approvals/execution.py@@ -11,6 +11,13 @@ when the call broke off and may have gone out. The employee learns it at thestart of its next turn in the chat (an ``[update]`` note), the card through``approval_lifecycle``.+Nothing goes while its employee is stopped (:func:`send_state`): the claim+refuses, the row stays approved (Undo still works), and the chat watchdog's+round starts the send again after Resume (``reconcile.recover_sends``). A+send whose generation ended is cancelled, and the employee told+(:func:`cancel_ended`). A call its turn waits on is its turn's: the turn+waited for Resume before it claims.+The node runs with the call's arguments over its own settings, as theemployee made it (with the owner's edits); what it sends as (its lockedfields) comes from its settings. Its activity checks that the row it@@ -31,6 +38,10 @@ logger = get_logger(__name__)WORKFLOW_NAME = "ApprovedToolCallWorkflow"PAYLOAD_VERSION = 1+#: A generation's control states while it lives, and those in which its+#: employee is stopped.+LIVE_STATES = frozenset({"starting", "running", "pausing", "paused", "resuming"})+STOPPED_STATES = frozenset({"pausing", "paused", "resuming"})class SendUnavailable(RuntimeError):@@ -113,6 +124,13 @@ async def claim_send(database: Any, approval_id: str, revision: int, token: str)return {"claimed": True, "activity": activity_name, "context": node_context(row, token)}if (row.kind or "gate") != "tool_call" or row.status != "approved" or row.revision != revision:return {"claimed": False, "reason": "changed"}+ if row.runtime != "turn":+ state = send_state(row, await database.get_latest_workflow_control(row.workflow_id))+ if state == "wait":+ return {"claimed": False, "reason": "stopped"}+ if state == "ended":+ await cancel_ended(database, row)+ return {"claimed": False, "reason": "cancelled"}activity_name = node_activity_name(row.node_type or "")if activity_name is None:failed = await _fail_unclaimed(database, row, "The tool it uses is no longer installed.")@@ -132,6 +150,38 @@ async def claim_send(database: Any, approval_id: str, revision: int, token: str)return {"claimed": True, "activity": activity_name, "context": node_context(claimed, token)}+def send_state(row: Any, control: Any) -> str:+ """Whether an approved draft may go now, given its workflow's latest+ control row: ``send``; ``wait`` while its employee is stopped; ``ended``+ when its generation is over (no control, a newer generation, or one+ that no longer lives). A manual run's draft (generation 0) always goes."""+ if not row.generation:+ return "send"+ if control is None or control.generation != row.generation or control.status not in LIVE_STATES:+ return "ended"+ return "wait" if control.status in STOPPED_STATES else "send"+++async def cancel_ended(database: Any, row: Any) -> Optional[Any]:+ """Cancel an approved draft whose generation ended: it never goes, and+ the employee hears so. None when the row moved meanwhile."""+ try:+ cancelled = await store.transition(+ database,+ row.id,+ expected_revision=row.revision,+ from_statuses=("approved",),+ status="cancelled",+ values={"decided_at": _utcnow(), "grace_until": None},+ )+ except store.ApprovalConflict:+ return None+ logger.info("Cancelled a send whose generation ended", approval_id=row.id, workflow_id=row.workflow_id)+ await notify_approval_changed(change_of(cancelled, "cancelled"))+ await tell_employee(database, cancelled)+ return cancelled++async def _fail_unclaimed(database: Any, row: Any, error: str) -> Any:try:failed = await store.transition(@@ -228,15 +278,19 @@ async def tell_employee(database: Any, row: Any) -> None:__all__ = [+ "LIVE_STATES","PAYLOAD_VERSION",+ "STOPPED_STATES","SendUnavailable","WORKFLOW_NAME",+ "cancel_ended","claim_send","node_activity_name","node_context","outcome_of","record_outcome","send_payload",+ "send_state","send_workflow_id","start_send","tell_employee",diff --git a/server/services/approvals/handlers.py b/server/services/approvals/handlers.pyindex 28054ae1..de6d888b 100644--- a/server/services/approvals/handlers.py+++ b/server/services/approvals/handlers.py@@ -227,10 +227,9 @@ async def handle_decide_approval(data: Dict[str, Any], websocket: WebSocket) ->return {"success": True,"approval": store.summary(decided, deployment_state=state),- # A paused employee's run holds the approved draft until Resume.- "will_send_on_resume": ((decided.kind or "gate") == "gate" or decided.runtime == "turn")- and decided.status == "approved"- and state in ("paused", "pausing"),+ # Nothing a stopped employee approved goes before Resume+ # (execution.send_state); a manual run's draft is not held.+ "will_send_on_resume": decided.status == "approved" and (decided.generation or 0) > 0 and state in execution.STOPPED_STATES,}diff --git a/server/services/approvals/reconcile.py b/server/services/approvals/reconcile.pyindex 1b47de8f..3e84a7fa 100644--- a/server/services/approvals/reconcile.py+++ b/server/services/approvals/reconcile.py@@ -11,14 +11,17 @@ so a Temporal gate row is kept while its generation is live. A held toolcall waits on nothing (its run already went on), so only expiry or a cancelends it.-Sends: an approved held call whose send never started (the server stopped-between the decision and the workflow) is started again; the claim at its-revision makes a second start harmless. A send claimed long ago that never-reported broke off somewhere: it ends ``failed`` with an unknown outcome,-since it may have gone out.--Runs once per process, the first time the drafts are listed; local rows-created by this process are never touched.+Sends, every round of the chat watchdog (:func:`recover_sends`): an+approved held call whose send never started (the server stopped between the+decision and the workflow), or whose claim refused because its employee was+stopped, is started again once its employee runs; the claim at its revision+makes a second start harmless. One whose generation ended is cancelled, and+the employee told (``execution.send_state``). A send claimed long ago that+never reported broke off somewhere: it ends ``failed`` with an unknown+outcome, since it may have gone out.++Orphaned gate rows are ended once per process, the first time the drafts+are listed; local rows created by this process are never touched.Expiry: a held call waits on nothing, so the chat watchdog ends one pastits expiry every round (:func:`expire_due`). Rows are kept until their@@ -32,16 +35,15 @@ the decision sent was lost (services/approvals/waiting.py).from __future__ import annotationsfrom datetime import datetime, timedelta, timezone-from typing import Any, List+from typing import Any, Dict, Listfrom core.logging import get_logger-from services.approvals import store+from services.approvals import execution, storefrom services.approvals.listeners import change_of, notify_approval_changedlogger = get_logger(__name__)PROCESS_STARTED = datetime.now(timezone.utc)-_LIVE_STATUSES = frozenset({"starting", "running", "pausing", "paused", "resuming"})#: How long after its grace an approved send may still be on its way._START_GRACE = timedelta(seconds=60)#: How long a claimed send may run before it counts as broken off.@@ -67,7 +69,7 @@ async def cancel_orphaned_pending(database: Any, *, started_at: datetime = PROCEorphaned = _aware(row.created_at) < started_atelse:control = await database.get_latest_workflow_control(row.workflow_id)- orphaned = control is None or control.generation != row.generation or control.status not in _LIVE_STATUSES+ orphaned = control is None or control.generation != row.generation or control.status not in execution.LIVE_STATESif not orphaned:continuetry:@@ -82,19 +84,31 @@ async def cancel_orphaned_pending(database: Any, *, started_at: datetime = PROCEasync def recover_sends(database: Any, *, now: datetime | None = None) -> List[str]:- """Restart sends that never started; end sends that never reported."""- from services.approvals import execution-+ """Start sends that never started, or that waited for Resume, once their+ employee runs; cancel those of a generation that ended; end sends that+ never reported."""now = now or datetime.now(timezone.utc)touched: List[str] = []+ controls: Dict[str, Any] = {}for row in await store.list_unfinished_sends(database):if row.status == "approved" and row.consumed_at is None:grace = store.aware(row.grace_until)if grace is None or now - grace < _START_GRACE:continue+ if row.workflow_id not in controls:+ controls[row.workflow_id] = await database.get_latest_workflow_control(row.workflow_id)+ state = execution.send_state(row, controls[row.workflow_id])+ if state == "wait":+ continue+ if state == "ended":+ if await execution.cancel_ended(database, row) is not None:+ touched.append(row.id)+ continuetry:await execution.start_send(row)touched.append(row.id)+ except execution.SendUnavailable:+ pass # no workflow engine: the next round tries againexcept Exception:logger.warning("Could not restart a send", approval_id=row.id, exc_info=True)elif row.status == "sending":@@ -143,7 +157,6 @@ async def reconcile_once(database: Any) -> None:_done = Truetry:await cancel_orphaned_pending(database)- await recover_sends(database)except Exception:_done = Falselogger.warning("Could not reconcile waiting drafts", exc_info=True)diff --git a/server/services/chat/watchdog.py b/server/services/chat/watchdog.pyindex 13c4be6f..a7a719c2 100644--- a/server/services/chat/watchdog.py+++ b/server/services/chat/watchdog.py@@ -113,18 +113,19 @@ class ChatRunWatchdog:raiseexcept Exception: # noqa: BLE001 - the next sweep tries againlogger.warning("Chat run sweep failed", exc_info=True)- try:- # Drafts held for the owner end here once past their expiry,- # and a turn's wait whose nudge was lost is nudged again- # (services/approvals/reconcile.py).- from services.approvals.reconcile import expire_due, renudge_turns+ # Drafts held for the owner end here once past their expiry, an+ # approved send that never started (or waited for Resume) starts+ # once its employee runs, and a turn's wait whose nudge was lost+ # is nudged again (services/approvals/reconcile.py).+ from services.approvals.reconcile import expire_due, recover_sends, renudge_turns- await expire_due(self.database)- await renudge_turns(self.database)- except asyncio.CancelledError:- raise- except Exception: # noqa: BLE001 - the next round tries again- logger.warning("Held draft expiry or nudge failed", exc_info=True)+ for step in (expire_due, recover_sends, renudge_turns):+ try:+ await step(self.database)+ except asyncio.CancelledError:+ raise+ except Exception: # noqa: BLE001 - the next round tries again+ logger.warning("Approvals housekeeping failed", step=step.__name__, exc_info=True)await asyncio.sleep(self.interval)diff --git a/server/tests/services/approvals/test_stopped_sends.py b/server/tests/services/approvals/test_stopped_sends.pynew file mode 100644index 00000000..f3ecef20--- /dev/null+++ b/server/tests/services/approvals/test_stopped_sends.py@@ -0,0 +1,135 @@+"""An approved send and its employee's state (1b.5): it never goes while+its generation is stopped and starts within one watchdog round of Resume; a+send whose generation ended is cancelled and its employee told; Undo works+until the send is claimed; a manual run's send is not held."""++from __future__ import annotations++import asyncio+from datetime import datetime, timedelta, timezone+from types import SimpleNamespace++import pytest++from services.approvals import execution, handlers, reconcile, rules, store, tool_calls+from services.chat import notes+from services.node_registry import get_node_class++pytestmark = pytest.mark.asyncio++SOCKET = SimpleNamespace(scope={"path": "/ws/status"}, state=SimpleNamespace(user_id="owner"))+WHATSAPP = {"recipient_type": "phone", "phone": "447700900123", "message": "Running late, there at 10."}+LATER = datetime.now(timezone.utc) + timedelta(minutes=5)+++def control(harness, monkeypatch):+ """The workflow's latest control row, as the test sets it."""+ import nodes # noqa: F401 - the plugin registry++ current = SimpleNamespace(generation=1, status="paused")++ async def latest(workflow_id):+ return current++ monkeypatch.setattr(harness.database, "get_latest_workflow_control", latest)+ return current+++def starts(monkeypatch):+ started = []++ async def start_send(row):+ started.append(row.id)++ monkeypatch.setattr(execution, "start_send", start_send)+ return started+++async def decide(row_id, decision, key):+ return await handlers.handle_decide_approval({"approval_id": row_id, "decision": decision, "decision_key": key}, SOCKET)+++async def approved(harness, *, generation=1):+ """A draft held for the owner, then sent."""+ await rules.set_ask_first(harness.database, "wf", True)+ context = {+ "node_id": "wf:whatsappSend:1",+ "workflow_id": "wf",+ "execution_id": "wf:execution:1",+ "user_id": "owner",+ "parent_node_id": "wf:aiAgent:1",+ "tool_call_id": "call_1",+ "node_data": {"message_type": "text", **WHATSAPP},+ "tool_args": WHATSAPP,+ **({"generation": generation} if generation else {}),+ }+ await tool_calls.check(context, get_node_class("whatsappSend"))+ (row,) = await store.list_approvals(harness.database, status="pending")+ answer = await decide(row.id, "send", "s1")+ return answer, await store.get(harness.database, row.id)+++async def test_a_send_waits_while_its_employee_is_stopped(harness, control, starts, monkeypatch):+ async def paused(database, workflow_id):+ return "paused"++ monkeypatch.setattr(handlers, "_deployment_state", paused)+ answer, row = await approved(harness)+ assert answer["will_send_on_resume"] is True and starts == [row.id]+ # The send the decision started refuses, and the row keeps waiting.+ assert await execution.claim_send(harness.database, row.id, row.revision, "t-1") == {"claimed": False, "reason": "stopped"}+ assert (await store.get(harness.database, row.id)).status == "approved"+ for state in ("pausing", "paused", "resuming"):+ control.status = state+ assert await reconcile.recover_sends(harness.database, now=LATER) == []+ # Once its employee runs, the round starts it, and the claim goes.+ control.status = "running"+ assert await reconcile.recover_sends(harness.database, now=LATER) == [row.id]+ assert starts == [row.id, row.id]+ assert (await execution.claim_send(harness.database, row.id, row.revision, "t-2"))["claimed"] is True+++ "ended",+ [+ SimpleNamespace(generation=1, status="reset"),+ SimpleNamespace(generation=1, status="failed"),+ SimpleNamespace(generation=2, status="running"),+ ],+ ids=["reset", "failed", "newer generation"],+)+async def test_a_send_whose_generation_ended_is_cancelled_and_the_employee_told(harness, control, starts, ended):+ _, row = await approved(harness)+ control.generation, control.status = ended.generation, ended.status+ assert await reconcile.recover_sends(harness.database, now=LATER) == [row.id]+ assert (await store.get(harness.database, row.id)).status == "cancelled"+ [note] = await notes.claim_notes(harness.database, session_id="wf", run_id="r_next")+ assert '"result": "cancelled unsent"' in note.text+ # Nothing starts it again.+ assert await reconcile.recover_sends(harness.database, now=LATER) == []+++async def test_a_claim_for_an_ended_generation_cancels_the_send(harness, control, starts):+ _, row = await approved(harness)+ control.status = "reset"+ assert await execution.claim_send(harness.database, row.id, row.revision, "t-1") == {"claimed": False, "reason": "cancelled"}+ assert (await store.get(harness.database, row.id)).status == "cancelled"+++async def test_a_waiting_send_can_be_undone_until_it_is_claimed(harness, control, starts):+ _, row = await approved(harness)+ await asyncio.sleep(0.25) # past its Undo window+ undone = await decide(row.id, "undo", "u1")+ assert undone["success"] is True and undone["approval"]["status"] == "pending"+ await decide(row.id, "send", "s2")+ control.status = "running"+ resent = await store.get(harness.database, row.id)+ assert (await execution.claim_send(harness.database, row.id, resent.revision, "t-1"))["claimed"] is True+ assert (await decide(row.id, "undo", "u2"))["error"] == "too_late"+++async def test_a_manual_runs_send_goes_while_the_deployment_is_paused(harness, control, starts):+ _, row = await approved(harness, generation=0)+ assert (await execution.claim_send(harness.database, row.id, row.revision, "t-1"))["claimed"] is Truediff --git a/server/tests/services/approvals/test_turn_rows.py b/server/tests/services/approvals/test_turn_rows.pyindex e94f6f2d..1aba44c2 100644--- a/server/tests/services/approvals/test_turn_rows.py+++ b/server/tests/services/approvals/test_turn_rows.py@@ -23,7 +23,7 @@ SOCKET = SimpleNamespace(scope={"path": "/ws/status"}, state=SimpleNamespace(useWHATSAPP = {"recipient_type": "phone", "phone": "447700900123", "message": "On my way."}-async def held(harness, *, waits=True):+async def held(harness, *, waits=True, generation=0):await rules.set_mode(harness.database, "wf", "outside")context = {"node_id": "wf:whatsappSend:1",@@ -36,6 +36,7 @@ async def held(harness, *, waits=True):"tool_args": WHATSAPP,"approval_wait_version": 1,**({"approval_wait": True} if waits else {}),+ **({"generation": generation} if generation else {}),}await tool_calls.check(context, get_node_class("whatsappSend"))(row,) = await store.list_approvals(harness.database, status="pending")@@ -75,7 +76,7 @@ async def test_a_turn_row_approved_while_paused_runs_on_resume(harness, sends, mreturn "paused"monkeypatch.setattr(handlers, "_deployment_state", paused)- row = await held(harness)+ row = await held(harness, generation=1)assert (await decide(row, "send", "s1"))["will_send_on_resume"] is True