Skip to content

Self-improving AI Employees for everyone

feat(approvals): an approved send waits while its employee is stopped (1b.5, with 1b.6)

trohitg32c082de1732
diff --git a/client/src/features/chat/__tests__/approvals.test.tsx b/client/src/features/chat/__tests__/approvals.test.tsx
index 1ff01bd2..8d5e7ee0 100644
--- a/client/src/features/chat/__tests__/approvals.test.tsx
+++ b/client/src/features/chat/__tests__/approvals.test.tsx
@@ -79,7 +79,15 @@ describe('the card says what happened', () => {
it('a gate’s draft for a paused employee goes on Resume', () => {
const paused = view({ kind: 'gate', status: 'approved', undo_until: '2026-10-04T08:59:59Z', deployment_state: 'paused' });
- expect(paused.footer).toEqual({ kind: 'waiting', text: 'Sends when you resume Maya.' });
+ expect(paused.footer).toEqual({ kind: 'waiting', text: 'Sends when you resume Maya.', undo: false });
+ });
+
+ it('a held draft waits for Resume too, and can still be undone', () => {
+ for (const state of ['paused', 'pausing', 'resuming']) {
+ const paused = view({ status: 'approved', undo_until: '2026-10-04T08:59:59Z', deployment_state: state });
+ expect(paused.footer).toEqual({ kind: 'waiting', text: 'Sends when you resume Maya.', undo: true });
+ }
+ expect(view({ status: 'approved', undo_until: '2026-10-04T08:59:59Z', deployment_state: 'running' }).footer).toEqual({ kind: 'sending' });
});
it('says it went only once it went', () => {
@@ -144,7 +152,7 @@ describe('the card says what happened', () => {
// A retry makes it a draft again.
expect(optimistic(approval({ waits: true, status: 'failed' }), 'retry', NOW, 0)).toMatchObject({ waits: false, undoUntil: '2026-10-04T09:00:05.000Z' });
const paused = view({ waits: true, status: 'approved', deployment_state: 'paused' });
- expect(paused.footer).toEqual({ kind: 'waiting', text: 'Sends when you resume Maya.' });
+ expect(paused.footer).toEqual({ kind: 'waiting', text: 'Sends when you resume Maya.', undo: false });
});
it('reads the list with the server’s clock, and moves at once on a decision', () => {
@@ -297,6 +305,15 @@ describe('a draft in the chat', () => {
expect(server.decisions[0].decision_key).not.toBe(server.decisions[1].decision_key);
});
+ it('a draft waiting for Resume can be undone', async () => {
+ server.approvals = [summary({ status: 'approved', undo_until: '2026-01-01T00:00:00Z', deployment_state: 'paused', revision: 1 })];
+ renderPane();
+ const card = (await screen.findByText('Sends when you resume Maya.')).closest('[data-approval]') as HTMLElement;
+ fireEvent.click(within(card).getByRole('button', { name: /^Undo$/ }));
+ await waitFor(() => expect(server.decisions[0]).toMatchObject({ approval_id: 'ap1', decision: 'undo' }));
+ expect(await within(card).findByRole('button', { name: /^Send$/ })).toBeInTheDocument();
+ });
+
it('allows a call that has no message as it is, showing what it runs with', async () => {
server.approvals = [
summary({
diff --git a/client/src/features/chat/approval/ApprovalCard.tsx b/client/src/features/chat/approval/ApprovalCard.tsx
index e87536b5..bccbb95e 100644
--- a/client/src/features/chat/approval/ApprovalCard.tsx
+++ b/client/src/features/chat/approval/ApprovalCard.tsx
@@ -305,7 +305,13 @@ function Footer({
return (
<div className={cn(bar, 'bg-bg-app text-xs text-fg-muted')} role="status">
<Pause aria-hidden className="size-3.5" />
- {footer.text}
+ <span className="flex-1">{footer.text}</span>
+ {footer.undo && (
+ <Button variant="outline" size="sm" disabled={busy} onClick={() => decide({ approval, decision: 'undo' })} className="gap-1.5 font-semibold">
+ <Undo2 aria-hidden className="size-3.25" />
+ Undo
+ </Button>
+ )}
</div>
);
}
diff --git a/client/src/features/chat/approval/view.ts b/client/src/features/chat/approval/view.ts
index c0cbc8bc..32b4e3a6 100644
--- a/client/src/features/chat/approval/view.ts
+++ b/client/src/features/chat/approval/view.ts
@@ -32,8 +32,8 @@ export type CardFooter =
| { kind: 'decide' }
| { kind: 'undo'; seconds: number }
| { kind: 'sending' }
- /** Approved, and its run waits for Resume before it goes. */
- | { kind: 'waiting'; text: string }
+ /** Approved, and it waits for Resume before it goes; a draft can still be undone. */
+ | { kind: 'waiting'; text: string; undo: boolean }
| { kind: 'sent'; text: string }
/** Declined, with what the owner said, if anything. */
| { kind: 'discarded'; text: string; note: string | null; restoreSeconds: number | null; canRestore: boolean }
@@ -96,12 +96,14 @@ export function approvalView(approval: ChatApproval, { name, nowMs, offsetMs, ed
case 'approved': {
if (approval.consumedAt === null) {
const seconds = secondsLeft(approval.undoUntil, offsetMs, nowMs);
- // Its run waits for Resume: a gate's draft, or a call its turn waits on.
+ // Nothing goes while its employee is stopped: it waits for Resume. A
+ // draft can be undone until its send starts; a call its turn waits
+ // on cannot.
const paused =
- (approval.kind === 'gate' || approval.waits) && (approval.deploymentState === 'paused' || approval.deploymentState === 'pausing');
+ approval.deploymentState === 'paused' || approval.deploymentState === 'pausing' || approval.deploymentState === 'resuming';
const footer: CardFooter =
seconds > 0 ? { kind: 'undo', seconds }
- : paused ? { kind: 'waiting', text: `${message ? 'Sends' : 'Runs'} when you resume ${name}.` }
+ : paused ? { kind: 'waiting', text: `${message ? 'Sends' : 'Runs'} when you resume ${name}.`, undo: approval.kind === 'tool_call' && !approval.waits }
: { kind: 'sending' };
return {
title: `${name} is ${message ? 'sending' : 'doing'} it`,
diff --git a/docs-internal/chat_protocol.md b/docs-internal/chat_protocol.md
index 9e4ee234..c97a6a55 100644
--- a/docs-internal/chat_protocol.md
+++ b/docs-internal/chat_protocol.md
@@ -173,7 +173,8 @@ and workflows answer; any other id reads as `not_found`. `decide_approval` error
broke off), `approval_conflict`, `send_unavailable` (nothing could start the send; the draft is back as it was),
`not_found`, `invalid_request` (also for `always` on a card that offers no rule, and a `note` over 50,000
characters). The same `decision_key` again answers `idempotent` with the row as it is. `note` goes with `discard`;
-`always` with `send` saves the rule the card offers (`always` in its summary).
+`always` with `send` saves the rule the card offers (`always` in its summary). `will_send_on_resume` is true for an
+approved row whose employee is stopped (generation above 0): it goes after Resume.
**ApprovalSummary**: `{approval_id, workflow_id, node_id, kind: "gate" | "tool_call" | "effect", status, channel,
channel_label, action?, recipient, recipient_label, body, subject?, context_excerpt?, details: [{label, value}],
@@ -564,8 +565,9 @@ What an employee sends to someone waits for the owner's OK while its workflow as
`tool_calls.check_in_process` applies the same rule without drafts: with Ask first on, a call that sends is not run
and the model reads why, the browser runs read-only, and nothing is recorded.
- **Decisions** (`services/approvals/decisions.py`), each a compare-and-swap on the row's revision recorded in
- `approval_decisions`: `send` (pending -> approved; it goes after `UNDO_SECONDS`, 5, and Undo works until then),
- `undo` (approved -> pending), `discard` (pending -> discarded; Restore works for `RESTORE_SECONDS` on a gate's draft,
+ `approval_decisions`: `send` (pending -> approved; it goes after `UNDO_SECONDS`, 5, and Undo works until then; a
+ held call's Undo also works past that while its send is not claimed, as while it waits for Resume), `undo`
+ (approved -> pending), `discard` (pending -> discarded; Restore works for `RESTORE_SECONDS` on a gate's draft,
whose run waits that long, and until expiry for a held call), `restore`, `retry` (failed -> approved; when the
send broke off it may have gone, so the owner confirms) and `dismiss` (an `effect` row: failed -> cancelled). A gate waits through the Undo and Restore windows before
it lets the draft through or not.
@@ -573,8 +575,14 @@ What an employee sends to someone waits for the owner's OK while its workflow as
the Undo window closes, claims the row at that revision (`approvals.claim_send`: approved -> sending; an Undo, a
newer Send or a cancel moved it, and nothing runs), runs the node's own activity once with no retry, and records
the outcome (`approvals.record_outcome`): `sent`, `not_sent`, or `unknown` when the activity broke off. The node's
- activity runs only under that claim. The first time the drafts are listed after a start, sends that never started
- are started again and sends that never reported end `failed` with an unknown outcome.
+ activity runs only under that claim.
+ - **Nothing goes while its employee is stopped** (`execution.send_state`). The claim refuses (`stopped`) while the
+ generation is pausing, paused or resuming, and the row stays approved.
+ - **The chat watchdog's round** (`reconcile.recover_sends`, every round) starts such a send again once its
+ employee runs, and so a send that never started.
+ - **A send whose generation ended** (a Reset, a failure, a newer generation) is cancelled, and the employee told.
+ A manual run's send (generation 0) is not held.
+ - **A send that never reported** ends `failed` with an unknown outcome.
- **At most once.** Temporal runs an activity again only when an attempt broke off (the worker stopped, a timeout);
a failure the node reports comes back as a result and is not retried. A later attempt runs only a call that is
safe to repeat (`services/approvals/repeats.py`: reads, waits, keyed calls, operations that do the same thing
@@ -599,8 +607,8 @@ What an employee sends to someone waits for the owner's OK while its workflow as
`ledger.ui_part_of_run`), and the card says "Linked to the form above". Drafts no reply in view made (a
gate's, from the employee's own work) show after the conversation. The card (`features/chat/approval/`): Discard /
Edit / Send while it waits, "Sends in Ns" with Undo, Sending, Sent (and when, and whether Ask first was off),
- Restore while it can, Try again (asking first when it may have gone), and "Sends when you resume" for a gate's
- draft whose employee is paused. An `effect` card has no recipient and no message: "{name} tried to …", May have
+ Restore while it can, Try again (asking first when it may have gone), and "Sends when you resume" for an
+ approved draft whose employee is stopped (with Undo on a held call's). An `effect` card has no recipient and no message: "{name} tried to …", May have
happened, and Dismiss only (Dismissed afterwards). Countdowns use the server's clock (`server_time`).
Ctrl/Cmd+Enter sends the newest waiting draft; in the edit box it sends that one.
- **Ends.** Clearing or resetting the chat cancels the drafts its runs made that still wait, and their failed sends,
diff --git a/docs-internal/normal_mode.md b/docs-internal/normal_mode.md
index 03f770fb..01259653 100644
--- a/docs-internal/normal_mode.md
+++ b/docs-internal/normal_mode.md
@@ -913,6 +913,9 @@ of running; both show as cards in the employee's chat, on the reply that made
them or after the conversation. Send goes after a 5-second Undo window, a
discarded draft can be restored, and a held call is sent once by its own
Temporal workflow, which tells the employee how it went on its next turn.
+Nothing goes while the employee is stopped: an approved draft waits for
+Resume (a held call can be undone until then), and one whose generation
+ended is cancelled.
With it off, replies and calls go at once and the chat still shows what went.
The gate stores the draft in `approval_requests`, wakes the moment it is
decided, survives restarts (its idempotency key finds the same row on every
diff --git a/server/services/approvals/decisions.py b/server/services/approvals/decisions.py
index 7d30ec91..f272e0ca 100644
--- a/server/services/approvals/decisions.py
+++ b/server/services/approvals/decisions.py
@@ -174,7 +174,12 @@ async def decide(
await _save_rule(database, moved, scope, actor)
elif decision == "undo":
grace = store.aware(row.grace_until)
- if row.status != "approved" or row.consumed_at is not None or row.claim_token or grace is None or grace <= now:
+ # A draft can be undone until its send is claimed, as while it waits
+ # for Resume; a gate's draft only within its grace, and a call its
+ # turn waits on (no grace) never. The claim's compare-and-swap settles
+ # a race with Undo.
+ draft = kind == "tool_call" and not turn
+ if row.status != "approved" or row.consumed_at is not None or row.claim_token or grace is None or (grace <= now and not draft):
raise DecisionRefused("too_late", "It has already gone.")
values = {"grace_until": None, "decided_at": None, "approved_by": None, "final_text": None, "final_subject": None, "edited": False}
moved = await _move(database, row, ("approved",), "pending", values, record)
diff --git a/server/services/approvals/execution.py b/server/services/approvals/execution.py
index cb73a703..337214bc 100644
--- a/server/services/approvals/execution.py
+++ b/server/services/approvals/execution.py
@@ -11,6 +11,13 @@ when the call broke off and may have gone out. The employee learns it at the
start of its next turn in the chat (an ``[update]`` note), the card through
``approval_lifecycle``.
+Nothing goes while its employee is stopped (:func:`send_state`): the claim
+refuses, the row stays approved (Undo still works), and the chat watchdog's
+round starts the send again after Resume (``reconcile.recover_sends``). A
+send whose generation ended is cancelled, and the employee told
+(:func:`cancel_ended`). A call its turn waits on is its turn's: the turn
+waited for Resume before it claims.
+
The node runs with the call's arguments over its own settings, as the
employee made it (with the owner's edits); what it sends as (its locked
fields) comes from its settings. Its activity checks that the row it
@@ -31,6 +38,10 @@ logger = get_logger(__name__)
WORKFLOW_NAME = "ApprovedToolCallWorkflow"
PAYLOAD_VERSION = 1
+#: A generation's control states while it lives, and those in which its
+#: employee is stopped.
+LIVE_STATES = frozenset({"starting", "running", "pausing", "paused", "resuming"})
+STOPPED_STATES = frozenset({"pausing", "paused", "resuming"})
class SendUnavailable(RuntimeError):
@@ -113,6 +124,13 @@ async def claim_send(database: Any, approval_id: str, revision: int, token: str)
return {"claimed": True, "activity": activity_name, "context": node_context(row, token)}
if (row.kind or "gate") != "tool_call" or row.status != "approved" or row.revision != revision:
return {"claimed": False, "reason": "changed"}
+ if row.runtime != "turn":
+ state = send_state(row, await database.get_latest_workflow_control(row.workflow_id))
+ if state == "wait":
+ return {"claimed": False, "reason": "stopped"}
+ if state == "ended":
+ await cancel_ended(database, row)
+ return {"claimed": False, "reason": "cancelled"}
activity_name = node_activity_name(row.node_type or "")
if activity_name is None:
failed = await _fail_unclaimed(database, row, "The tool it uses is no longer installed.")
@@ -132,6 +150,38 @@ async def claim_send(database: Any, approval_id: str, revision: int, token: str)
return {"claimed": True, "activity": activity_name, "context": node_context(claimed, token)}
+def send_state(row: Any, control: Any) -> str:
+ """Whether an approved draft may go now, given its workflow's latest
+ control row: ``send``; ``wait`` while its employee is stopped; ``ended``
+ when its generation is over (no control, a newer generation, or one
+ that no longer lives). A manual run's draft (generation 0) always goes."""
+ if not row.generation:
+ return "send"
+ if control is None or control.generation != row.generation or control.status not in LIVE_STATES:
+ return "ended"
+ return "wait" if control.status in STOPPED_STATES else "send"
+
+
+async def cancel_ended(database: Any, row: Any) -> Optional[Any]:
+ """Cancel an approved draft whose generation ended: it never goes, and
+ the employee hears so. None when the row moved meanwhile."""
+ try:
+ cancelled = await store.transition(
+ database,
+ row.id,
+ expected_revision=row.revision,
+ from_statuses=("approved",),
+ status="cancelled",
+ values={"decided_at": _utcnow(), "grace_until": None},
+ )
+ except store.ApprovalConflict:
+ return None
+ logger.info("Cancelled a send whose generation ended", approval_id=row.id, workflow_id=row.workflow_id)
+ await notify_approval_changed(change_of(cancelled, "cancelled"))
+ await tell_employee(database, cancelled)
+ return cancelled
+
+
async def _fail_unclaimed(database: Any, row: Any, error: str) -> Any:
try:
failed = await store.transition(
@@ -228,15 +278,19 @@ async def tell_employee(database: Any, row: Any) -> None:
__all__ = [
+ "LIVE_STATES",
"PAYLOAD_VERSION",
+ "STOPPED_STATES",
"SendUnavailable",
"WORKFLOW_NAME",
+ "cancel_ended",
"claim_send",
"node_activity_name",
"node_context",
"outcome_of",
"record_outcome",
"send_payload",
+ "send_state",
"send_workflow_id",
"start_send",
"tell_employee",
diff --git a/server/services/approvals/handlers.py b/server/services/approvals/handlers.py
index 28054ae1..de6d888b 100644
--- a/server/services/approvals/handlers.py
+++ b/server/services/approvals/handlers.py
@@ -227,10 +227,9 @@ async def handle_decide_approval(data: Dict[str, Any], websocket: WebSocket) ->
return {
"success": True,
"approval": store.summary(decided, deployment_state=state),
- # A paused employee's run holds the approved draft until Resume.
- "will_send_on_resume": ((decided.kind or "gate") == "gate" or decided.runtime == "turn")
- and decided.status == "approved"
- and state in ("paused", "pausing"),
+ # Nothing a stopped employee approved goes before Resume
+ # (execution.send_state); a manual run's draft is not held.
+ "will_send_on_resume": decided.status == "approved" and (decided.generation or 0) > 0 and state in execution.STOPPED_STATES,
}
diff --git a/server/services/approvals/reconcile.py b/server/services/approvals/reconcile.py
index 1b47de8f..3e84a7fa 100644
--- a/server/services/approvals/reconcile.py
+++ b/server/services/approvals/reconcile.py
@@ -11,14 +11,17 @@ so a Temporal gate row is kept while its generation is live. A held tool
call waits on nothing (its run already went on), so only expiry or a cancel
ends it.
-Sends: an approved held call whose send never started (the server stopped
-between the decision and the workflow) is started again; the claim at its
-revision makes a second start harmless. A send claimed long ago that never
-reported broke off somewhere: it ends ``failed`` with an unknown outcome,
-since it may have gone out.
-
-Runs once per process, the first time the drafts are listed; local rows
-created by this process are never touched.
+Sends, every round of the chat watchdog (:func:`recover_sends`): an
+approved held call whose send never started (the server stopped between the
+decision and the workflow), or whose claim refused because its employee was
+stopped, is started again once its employee runs; the claim at its revision
+makes a second start harmless. One whose generation ended is cancelled, and
+the employee told (``execution.send_state``). A send claimed long ago that
+never reported broke off somewhere: it ends ``failed`` with an unknown
+outcome, since it may have gone out.
+
+Orphaned gate rows are ended once per process, the first time the drafts
+are listed; local rows created by this process are never touched.
Expiry: a held call waits on nothing, so the chat watchdog ends one past
its expiry every round (:func:`expire_due`). Rows are kept until their
@@ -32,16 +35,15 @@ the decision sent was lost (services/approvals/waiting.py).
from __future__ import annotations
from datetime import datetime, timedelta, timezone
-from typing import Any, List
+from typing import Any, Dict, List
from core.logging import get_logger
-from services.approvals import store
+from services.approvals import execution, store
from services.approvals.listeners import change_of, notify_approval_changed
logger = get_logger(__name__)
PROCESS_STARTED = datetime.now(timezone.utc)
-_LIVE_STATUSES = frozenset({"starting", "running", "pausing", "paused", "resuming"})
#: How long after its grace an approved send may still be on its way.
_START_GRACE = timedelta(seconds=60)
#: How long a claimed send may run before it counts as broken off.
@@ -67,7 +69,7 @@ async def cancel_orphaned_pending(database: Any, *, started_at: datetime = PROCE
orphaned = _aware(row.created_at) < started_at
else:
control = await database.get_latest_workflow_control(row.workflow_id)
- orphaned = control is None or control.generation != row.generation or control.status not in _LIVE_STATUSES
+ orphaned = control is None or control.generation != row.generation or control.status not in execution.LIVE_STATES
if not orphaned:
continue
try:
@@ -82,19 +84,31 @@ async def cancel_orphaned_pending(database: Any, *, started_at: datetime = PROCE
async def recover_sends(database: Any, *, now: datetime | None = None) -> List[str]:
- """Restart sends that never started; end sends that never reported."""
- from services.approvals import execution
-
+ """Start sends that never started, or that waited for Resume, once their
+ employee runs; cancel those of a generation that ended; end sends that
+ never reported."""
now = now or datetime.now(timezone.utc)
touched: List[str] = []
+ controls: Dict[str, Any] = {}
for row in await store.list_unfinished_sends(database):
if row.status == "approved" and row.consumed_at is None:
grace = store.aware(row.grace_until)
if grace is None or now - grace < _START_GRACE:
continue
+ if row.workflow_id not in controls:
+ controls[row.workflow_id] = await database.get_latest_workflow_control(row.workflow_id)
+ state = execution.send_state(row, controls[row.workflow_id])
+ if state == "wait":
+ continue
+ if state == "ended":
+ if await execution.cancel_ended(database, row) is not None:
+ touched.append(row.id)
+ continue
try:
await execution.start_send(row)
touched.append(row.id)
+ except execution.SendUnavailable:
+ pass # no workflow engine: the next round tries again
except Exception:
logger.warning("Could not restart a send", approval_id=row.id, exc_info=True)
elif row.status == "sending":
@@ -143,7 +157,6 @@ async def reconcile_once(database: Any) -> None:
_done = True
try:
await cancel_orphaned_pending(database)
- await recover_sends(database)
except Exception:
_done = False
logger.warning("Could not reconcile waiting drafts", exc_info=True)
diff --git a/server/services/chat/watchdog.py b/server/services/chat/watchdog.py
index 13c4be6f..a7a719c2 100644
--- a/server/services/chat/watchdog.py
+++ b/server/services/chat/watchdog.py
@@ -113,18 +113,19 @@ class ChatRunWatchdog:
raise
except Exception: # noqa: BLE001 - the next sweep tries again
logger.warning("Chat run sweep failed", exc_info=True)
- try:
- # Drafts held for the owner end here once past their expiry,
- # and a turn's wait whose nudge was lost is nudged again
- # (services/approvals/reconcile.py).
- from services.approvals.reconcile import expire_due, renudge_turns
+ # Drafts held for the owner end here once past their expiry, an
+ # approved send that never started (or waited for Resume) starts
+ # once its employee runs, and a turn's wait whose nudge was lost
+ # is nudged again (services/approvals/reconcile.py).
+ from services.approvals.reconcile import expire_due, recover_sends, renudge_turns
- await expire_due(self.database)
- await renudge_turns(self.database)
- except asyncio.CancelledError:
- raise
- except Exception: # noqa: BLE001 - the next round tries again
- logger.warning("Held draft expiry or nudge failed", exc_info=True)
+ for step in (expire_due, recover_sends, renudge_turns):
+ try:
+ await step(self.database)
+ except asyncio.CancelledError:
+ raise
+ except Exception: # noqa: BLE001 - the next round tries again
+ logger.warning("Approvals housekeeping failed", step=step.__name__, exc_info=True)
await asyncio.sleep(self.interval)
diff --git a/server/tests/services/approvals/test_stopped_sends.py b/server/tests/services/approvals/test_stopped_sends.py
new file mode 100644
index 00000000..f3ecef20
--- /dev/null
+++ b/server/tests/services/approvals/test_stopped_sends.py
@@ -0,0 +1,135 @@
+"""An approved send and its employee's state (1b.5): it never goes while
+its generation is stopped and starts within one watchdog round of Resume; a
+send whose generation ended is cancelled and its employee told; Undo works
+until the send is claimed; a manual run's send is not held."""
+
+from __future__ import annotations
+
+import asyncio
+from datetime import datetime, timedelta, timezone
+from types import SimpleNamespace
+
+import pytest
+
+from services.approvals import execution, handlers, reconcile, rules, store, tool_calls
+from services.chat import notes
+from services.node_registry import get_node_class
+
+pytestmark = pytest.mark.asyncio
+
+SOCKET = SimpleNamespace(scope={"path": "/ws/status"}, state=SimpleNamespace(user_id="owner"))
+WHATSAPP = {"recipient_type": "phone", "phone": "447700900123", "message": "Running late, there at 10."}
+LATER = datetime.now(timezone.utc) + timedelta(minutes=5)
+
+
+def control(harness, monkeypatch):
+ """The workflow's latest control row, as the test sets it."""
+ import nodes # noqa: F401 - the plugin registry
+
+ current = SimpleNamespace(generation=1, status="paused")
+
+ async def latest(workflow_id):
+ return current
+
+ monkeypatch.setattr(harness.database, "get_latest_workflow_control", latest)
+ return current
+
+
+def starts(monkeypatch):
+ started = []
+
+ async def start_send(row):
+ started.append(row.id)
+
+ monkeypatch.setattr(execution, "start_send", start_send)
+ return started
+
+
+async def decide(row_id, decision, key):
+ return await handlers.handle_decide_approval({"approval_id": row_id, "decision": decision, "decision_key": key}, SOCKET)
+
+
+async def approved(harness, *, generation=1):
+ """A draft held for the owner, then sent."""
+ await rules.set_ask_first(harness.database, "wf", True)
+ context = {
+ "node_id": "wf:whatsappSend:1",
+ "workflow_id": "wf",
+ "execution_id": "wf:execution:1",
+ "user_id": "owner",
+ "parent_node_id": "wf:aiAgent:1",
+ "tool_call_id": "call_1",
+ "node_data": {"message_type": "text", **WHATSAPP},
+ "tool_args": WHATSAPP,
+ **({"generation": generation} if generation else {}),
+ }
+ await tool_calls.check(context, get_node_class("whatsappSend"))
+ (row,) = await store.list_approvals(harness.database, status="pending")
+ answer = await decide(row.id, "send", "s1")
+ return answer, await store.get(harness.database, row.id)
+
+
+async def test_a_send_waits_while_its_employee_is_stopped(harness, control, starts, monkeypatch):
+ async def paused(database, workflow_id):
+ return "paused"
+
+ monkeypatch.setattr(handlers, "_deployment_state", paused)
+ answer, row = await approved(harness)
+ assert answer["will_send_on_resume"] is True and starts == [row.id]
+ # The send the decision started refuses, and the row keeps waiting.
+ assert await execution.claim_send(harness.database, row.id, row.revision, "t-1") == {"claimed": False, "reason": "stopped"}
+ assert (await store.get(harness.database, row.id)).status == "approved"
+ for state in ("pausing", "paused", "resuming"):
+ control.status = state
+ assert await reconcile.recover_sends(harness.database, now=LATER) == []
+ # Once its employee runs, the round starts it, and the claim goes.
+ control.status = "running"
+ assert await reconcile.recover_sends(harness.database, now=LATER) == [row.id]
+ assert starts == [row.id, row.id]
+ assert (await execution.claim_send(harness.database, row.id, row.revision, "t-2"))["claimed"] is True
+
+
+ "ended",
+ [
+ SimpleNamespace(generation=1, status="reset"),
+ SimpleNamespace(generation=1, status="failed"),
+ SimpleNamespace(generation=2, status="running"),
+ ],
+ ids=["reset", "failed", "newer generation"],
+)
+async def test_a_send_whose_generation_ended_is_cancelled_and_the_employee_told(harness, control, starts, ended):
+ _, row = await approved(harness)
+ control.generation, control.status = ended.generation, ended.status
+ assert await reconcile.recover_sends(harness.database, now=LATER) == [row.id]
+ assert (await store.get(harness.database, row.id)).status == "cancelled"
+ [note] = await notes.claim_notes(harness.database, session_id="wf", run_id="r_next")
+ assert '"result": "cancelled unsent"' in note.text
+ # Nothing starts it again.
+ assert await reconcile.recover_sends(harness.database, now=LATER) == []
+
+
+async def test_a_claim_for_an_ended_generation_cancels_the_send(harness, control, starts):
+ _, row = await approved(harness)
+ control.status = "reset"
+ assert await execution.claim_send(harness.database, row.id, row.revision, "t-1") == {"claimed": False, "reason": "cancelled"}
+ assert (await store.get(harness.database, row.id)).status == "cancelled"
+
+
+async def test_a_waiting_send_can_be_undone_until_it_is_claimed(harness, control, starts):
+ _, row = await approved(harness)
+ await asyncio.sleep(0.25) # past its Undo window
+ undone = await decide(row.id, "undo", "u1")
+ assert undone["success"] is True and undone["approval"]["status"] == "pending"
+ await decide(row.id, "send", "s2")
+ control.status = "running"
+ resent = await store.get(harness.database, row.id)
+ assert (await execution.claim_send(harness.database, row.id, resent.revision, "t-1"))["claimed"] is True
+ assert (await decide(row.id, "undo", "u2"))["error"] == "too_late"
+
+
+async def test_a_manual_runs_send_goes_while_the_deployment_is_paused(harness, control, starts):
+ _, row = await approved(harness, generation=0)
+ assert (await execution.claim_send(harness.database, row.id, row.revision, "t-1"))["claimed"] is True
diff --git a/server/tests/services/approvals/test_turn_rows.py b/server/tests/services/approvals/test_turn_rows.py
index e94f6f2d..1aba44c2 100644
--- a/server/tests/services/approvals/test_turn_rows.py
+++ b/server/tests/services/approvals/test_turn_rows.py
@@ -23,7 +23,7 @@ SOCKET = SimpleNamespace(scope={"path": "/ws/status"}, state=SimpleNamespace(use
WHATSAPP = {"recipient_type": "phone", "phone": "447700900123", "message": "On my way."}
-async def held(harness, *, waits=True):
+async def held(harness, *, waits=True, generation=0):
await rules.set_mode(harness.database, "wf", "outside")
context = {
"node_id": "wf:whatsappSend:1",
@@ -36,6 +36,7 @@ async def held(harness, *, waits=True):
"tool_args": WHATSAPP,
"approval_wait_version": 1,
**({"approval_wait": True} if waits else {}),
+ **({"generation": generation} if generation else {}),
}
await tool_calls.check(context, get_node_class("whatsappSend"))
(row,) = await store.list_approvals(harness.database, status="pending")
@@ -75,7 +76,7 @@ async def test_a_turn_row_approved_while_paused_runs_on_resume(harness, sends, m
return "paused"
monkeypatch.setattr(handlers, "_deployment_state", paused)
- row = await held(harness)
+ row = await held(harness, generation=1)
assert (await decide(row, "send", "s1"))["will_send_on_resume"] is True