Skip to content

Automate browser based workflows with AI

Release the Copilot save hold on an interrupted row the server marks final (SKY-17648) (#8775)

Andrew Neilsonbe9f86894949
diff --git a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.historyRace.test.tsx b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.historyRace.test.tsx
index 4ae1192b4..2b24a158d 100644
--- a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.historyRace.test.tsx
+++ b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.historyRace.test.tsx
@@ -891,7 +891,7 @@ describe("WorkflowCopilotChat — recovery poll after a non-terminal stream clos
});
}
- function interruptedHistory(): HistoryData {
+ function interruptedHistory(final = false): HistoryData {
return historyWithRow({
sender: "ai",
content: interruptedText,
@@ -901,6 +901,7 @@ describe("WorkflowCopilotChat — recovery poll after a non-terminal stream clos
response_kind: "recover",
terminal_reason: "interrupted",
copilot_turn_id: turnId,
+ ...(final ? { interrupted_row_final: true } : {}),
},
});
}
@@ -2790,6 +2791,26 @@ describe("WorkflowCopilotChat — recovery poll after a non-terminal stream clos
expect(historyQueue).toHaveLength(0);
});
+ it("releases the save hold on the first read of an interrupted row the server marked final", async () => {
+ await startTurn();
+ await closeStreamWithoutTerminal();
+ const readsBeforeRow = workflowGets.length;
+
+ await advance(2_000);
+ await resolveNextHistory(interruptedHistory(true));
+ await act(async () => {
+ await Promise.resolve();
+ });
+
+ expect(renderedText()).toContain(interruptedText);
+ expect(renderedText()).not.toContain(
+ "Could not confirm whether Copilot saved changes",
+ );
+ expect(screen.queryByRole("button", { name: "Retry" })).toBeNull();
+ expect(workflowGets.length).toBeGreaterThan(readsBeforeRow);
+ expect(useWorkflowYamlEditorStore.getState().copilotAcceptance).toBeNull();
+ });
+
it("rechecks canonical after the terminal row despite unsaved edits", async () => {
hasLocalChanges.current = true;
await startTurn();
diff --git a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.reviewGate.test.tsx b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.reviewGate.test.tsx
index 445b6f7a4..56ea68b5f 100644
--- a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.reviewGate.test.tsx
+++ b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.reviewGate.test.tsx
@@ -15345,6 +15345,52 @@ describe("editor-state rollback lifecycle", () => {
useWorkflowHasChangesStore.getState().saveGeneration,
).toBeGreaterThan(0);
});
+ async function interruptStagedTurn(canonical: unknown) {
+ const call = await stage();
+ historyGet.mockImplementation((path: string) =>
+ Promise.resolve(
+ path === "/workflows/wpid_1" ? { data: canonical } : historyResponse,
+ ),
+ );
+ vi.useFakeTimers();
+ await act(async () => call.reject(new Error("Connection dropped")));
+ historyResponse.data.request_turn_id = "turn-1";
+ historyResponse.data.chat_history = [
+ {
+ sender: "ai",
+ content: "This turn was interrupted before it could finish.",
+ turn_outcome: {
+ copilot_turn_id: "turn-1",
+ terminal_reason: "interrupted",
+ interrupted_row_final: true,
+ },
+ },
+ ];
+ await act(async () => vi.advanceTimersByTimeAsync(2_000));
+ }
+ it("does not credit Copilot with a canonical change it never made when its turn was interrupted", async () => {
+ await renderChat();
+ await interruptStagedTurn({
+ ...proposedWorkflowPayload(),
+ workflow_id: "wf_saved",
+ version: 2,
+ });
+
+ expect(useWorkflowYamlEditorStore.getState().copilotAcceptance).toBeNull();
+ expect(toast).not.toHaveBeenCalledWith(
+ expect.objectContaining({ title: "The draft could not be restored" }),
+ );
+ });
+ it("restores the pre-turn editor when an interrupted turn left canonical untouched", async () => {
+ const restore = vi
+ .fn<(snapshot: EditorStateSnapshot) => RestoreResult>()
+ .mockImplementation(restoreLive);
+ await renderChat({ onRestore: restore });
+ await interruptStagedTurn(saveData.workflow);
+
+ expect(restore).toHaveBeenCalledOnce();
+ expect(useWorkflowYamlEditorStore.getState().copilotAcceptance).toBeNull();
+ });
it("reports a stale-workflow refusal during rollback and retains the reservation", async () => {
changesState.hasChanges = true;
const restore = vi
diff --git a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx
index 4c1c5e3a7..647a73e20 100644
--- a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx
+++ b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx
@@ -374,6 +374,12 @@ const RECOVERY_POLL_STEADY_MS = 30_000;
const RECOVERY_POLL_BUDGET_MS = 1_500_000;
const CANONICAL_READ_TIMEOUT_MS = 5_000;
const INTERRUPTED_TERMINAL_REASON = "interrupted";
+// Only an older backend writes an interrupted row a late reply may still replace.
+const isReplaceableInterruptedRow = (
+ outcome: WorkflowCopilotChatHistoryMessage["turn_outcome"],
+): boolean =>
+ outcome?.terminal_reason === INTERRUPTED_TERMINAL_REASON &&
+ outcome.interrupted_row_final !== true;
const SEND_FAILED_MESSAGE = "Sorry, I encountered an error. Please try again.";
// A severed stream is usually the client losing the network, so recovery reads
// fail too. Few enough that an offline user gets the plain failure back in
@@ -1492,6 +1498,9 @@ type CanonicalRecovery = {
retryApply?: () => Promise<boolean>;
rollback?: TurnSnapshot;
restoreRollback?: boolean;
+ // An interrupted turn commits canonical only if its process died between commit and reply, so a
+ // canonical change is not credited to Copilot; the editor loads it either way.
+ endedInterrupted?: boolean;
waitingForUnlock: boolean;
yaml: { draft: string; entrySnapshot: string } | null;
};
@@ -3999,7 +4008,8 @@ export function WorkflowCopilotChat({
(recoveredNarrative?.draft || ownsChatProposal),
);
const recoveredStatus: RecordingRefinementStatus = interrupted
- ? holdingReservation
+ ? holdingReservation ||
+ !isReplaceableInterruptedRow(row.turn_outcome)
? "failed"
: "working"
: isCancelledRefinementTurn(
@@ -4046,10 +4056,8 @@ export function WorkflowCopilotChat({
autoAcceptWritesBeforeRead,
);
}
- // An interrupted row can be replaced by the still-running finalizer.
- if (
- row.turn_outcome?.terminal_reason !== INTERRUPTED_TERMINAL_REASON
- ) {
+ // A replaceable interrupted row can be superseded by the still-running finalizer.
+ if (!isReplaceableInterruptedRow(row.turn_outcome)) {
if (!schedulesRefreshed) {
schedulesRefreshed = true;
void queryClient.invalidateQueries({
@@ -4068,6 +4076,8 @@ export function WorkflowCopilotChat({
canonicalRecovery.awaitingTurnId === rowTurnId
) {
canonicalRecovery.terminalConfirmed = true;
+ canonicalRecovery.endedInterrupted =
+ reason === INTERRUPTED_TERMINAL_REASON;
canonicalRecovery.restoreRollback ||=
isCancelledRefinementTurn(reason, narrative) ||
narrative?.terminal === "error" ||
@@ -4076,6 +4086,7 @@ export function WorkflowCopilotChat({
"cancelled",
"error",
"copilot_recoverable_failure",
+ INTERRUPTED_TERMINAL_REASON,
].includes(reason));
canonicalReadAttempted = true;
await reconcileCanonicalWorkflowRef.current?.(
@@ -4385,7 +4396,7 @@ export function WorkflowCopilotChat({
data.chat_history.flatMap((message) =>
message.sender === "ai" &&
message.turn_outcome?.copilot_turn_id &&
- message.turn_outcome.terminal_reason !== INTERRUPTED_TERMINAL_REASON
+ !isReplaceableInterruptedRow(message.turn_outcome)
? [message.turn_outcome.copilot_turn_id]
: [],
),
@@ -5190,6 +5201,7 @@ export function WorkflowCopilotChat({
if (pending) {
if (
!canonicalUnchanged &&
+ !pending.endedInterrupted &&
pending.rollback?.snapshot &&
(pending.rollback.titlePersisted ||
pending.rollback.workflowPersisted)
diff --git a/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.ts b/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.ts
index 65fe39d2e..955c5117b 100644
--- a/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.ts
+++ b/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.ts
@@ -223,6 +223,9 @@ export interface WorkflowCopilotChatHistoryMessage {
// turn_start frame carries, so a client can correlate a row to its own send.
copilot_turn_id?: string | null;
terminal_reason?: string | null;
+ // Set on every interrupted row the server serves, since no late reply replaces
+ // a stored row. Absent only from a backend that still replaced rows.
+ interrupted_row_final?: boolean;
})
| null;
narrative_payload?: Record<string, unknown> | null;
diff --git a/skyvern/forge/sdk/copilot/credential_pause.py b/skyvern/forge/sdk/copilot/credential_pause.py
index 1b42b4a2a..69ec5a54d 100644
--- a/skyvern/forge/sdk/copilot/credential_pause.py
+++ b/skyvern/forge/sdk/copilot/credential_pause.py
@@ -700,6 +700,14 @@ async def _run_credential_pause(
# Can't act on a rescued resolution mid-unwind, only avoid corrupting state.
await _invalidate_active_pause_record()
raise
+ try:
+ await app.DATABASE.workflow_params.record_pending_copilot_turn_credential_resume(
+ organization_id=organization_id,
+ workflow_copilot_chat_id=chat_id,
+ turn_id=turn_id,
+ )
+ except Exception:
+ LOG.warning("Could not record the credential pause resume on the turn marker", exc_info=True)
if resolution is None:
resolution = await _invalidate_active_pause_record()
diff --git a/skyvern/forge/sdk/db/agent_db.py b/skyvern/forge/sdk/db/agent_db.py
index cf2ed680c..0536e9715 100644
--- a/skyvern/forge/sdk/db/agent_db.py
+++ b/skyvern/forge/sdk/db/agent_db.py
@@ -406,7 +406,9 @@ class AgentDB(BaseAlchemyDB):
db_engine=self.engine,
sqlite_workflow_creation_lock=self._sqlite_workflow_creation_lock,
)
- self.workflow_params = WorkflowParametersRepository(self.Session, debug_enabled, self.is_retryable_error)
+ self.workflow_params = WorkflowParametersRepository(
+ self.Session, debug_enabled, self.is_retryable_error, db_engine=self.engine
+ )
self.workflow_run_attempts = WorkflowRunAttemptsRepository(self.Session, debug_enabled, self.is_retryable_error)
self.workflow_run_credential_selections = WorkflowRunCredentialSelectionsRepository(
self.Session, debug_enabled, self.is_retryable_error
diff --git a/skyvern/forge/sdk/db/repositories/workflow_parameters.py b/skyvern/forge/sdk/db/repositories/workflow_parameters.py
index 9101165bc..2714376ca 100644
--- a/skyvern/forge/sdk/db/repositories/workflow_parameters.py
+++ b/skyvern/forge/sdk/db/repositories/workflow_parameters.py
@@ -1,15 +1,17 @@
from __future__ import annotations
import json
-from collections.abc import Mapping
+from collections.abc import AsyncIterator, Callable, Mapping
+from contextlib import asynccontextmanager
from datetime import datetime, timedelta, timezone
-from typing import Any
+from typing import Any, Literal
import structlog
-from sqlalchemy import cast, func, or_, select
+from sqlalchemy import cast, func, or_, select, text
from sqlalchemy.dialects.postgresql import JSONB, JSONPATH
from sqlalchemy.dialects.postgresql import insert as pg_insert
-from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy.exc import SQLAlchemyError
+from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession
from sqlalchemy.orm import defer
from skyvern.config import settings
@@ -24,7 +26,7 @@ from skyvern.forge.sdk.copilot.completion_criteria_store import criteria_from_js
from skyvern.forge.sdk.copilot.context import ProposalDisposition, TurnNarrativePayload
from skyvern.forge.sdk.db._error_handling import db_operation
from skyvern.forge.sdk.db._sentinels import _UNSET
-from skyvern.forge.sdk.db.base_alchemy_db import read_with_disconnect_recovery
+from skyvern.forge.sdk.db.base_alchemy_db import _SessionFactory, read_with_disconnect_recovery
from skyvern.forge.sdk.db.base_repository import BaseRepository
from skyvern.forge.sdk.db.datetime_utils import naive_utc_now, to_naive_utc
from skyvern.forge.sdk.db.exceptions import (
@@ -101,6 +103,19 @@ from skyvern.webeye.actions.actions import Action
LOG = structlog.get_logger()
PENDING_TURN_RETENTION = timedelta(days=30)
+# ponytail: SQLite has no advisory locks and runs in one process, so an in-process set stands in.
+_SQLITE_FINALISING_TURNS: set[str] = set()
+
+
+def _turn_finalisation_lock_key(workflow_copilot_chat_id: str, turn_id: str) -> str:
+ return f"copilot_turn_finalisation:{workflow_copilot_chat_id}:{turn_id}"
+
+
+async def _turn_finalisation_held(session: AsyncSession, key: str) -> bool:
+ if session.bind is not None and session.bind.dialect.name == "sqlite":
+ return key in _SQLITE_FINALISING_TURNS
+ acquired = await session.scalar(text("SELECT pg_try_advisory_xact_lock(hashtext(:key))"), {"key": key})
+ return not acquired
def _pending_turn_id_for_idempotency_digest(
@@ -274,6 +289,16 @@ def _dump_attached_files(attached_files: list[CopilotAttachedFile] | None) -> li
class WorkflowParametersRepository(BaseRepository):
"""Database operations for workflow parameters, copilot chat, task generation, actions, and runs."""
+ def __init__(
+ self,
+ session_factory: _SessionFactory,
+ debug_enabled: bool = False,
+ is_retryable_error_fn: Callable[[SQLAlchemyError], bool] | None = None,
+ db_engine: AsyncEngine | None = None,
+ ) -> None:
+ super().__init__(session_factory, debug_enabled, is_retryable_error_fn)
+ self._db_engine = db_engine
+
@db_operation("create_workflow_parameter")
async def create_workflow_parameter(
self,
@@ -1390,11 +1415,86 @@ class WorkflowParametersRepository(BaseRepository):
recovering_at = _parse_pending_turn_timestamp(entry.get("recovering_at"))
if recovering_at is not None and recovering_at > claim_before:
return False
+ if await _turn_finalisation_held(session, _turn_finalisation_lock_key(workflow_copilot_chat_id, turn_id)):
+ return False
+ credential_resumed_at = _parse_pending_turn_timestamp(entry.get("credential_resumed_at"))
+ if credential_resumed_at is not None and credential_resumed_at > claim_before:
+ return False
pending[turn_id] = {**entry, "recovering_at": datetime.now(timezone.utc).isoformat()}
chat.pending_turns = pending
await session.commit()
return True
+ @db_operation("record_pending_copilot_turn_credential_resume")
+ async def record_pending_copilot_turn_credential_resume(
+ self,
+ organization_id: str,
+ workflow_copilot_chat_id: str,
+ turn_id: str,
+ ) -> None:
+ async with self.Session() as session:
+ chat = (
+ await session.scalars(
+ select(WorkflowCopilotChatModel)
+ .where(WorkflowCopilotChatModel.organization_id == organization_id)
+ .where(WorkflowCopilotChatModel.workflow_copilot_chat_id == workflow_copilot_chat_id)
+ .with_for_update()
+ )
+ ).first()
+ if chat is None:
+ return
+ pending = dict(chat.pending_turns or {})
+ entry = pending.get(turn_id)
+ if not isinstance(entry, dict):
+ return
+ pending[turn_id] = {**entry, "credential_resumed_at": datetime.now(timezone.utc).isoformat()}
+ chat.pending_turns = pending
+ await session.commit()
+
+ @db_operation("claim_pending_copilot_turn_for_finalisation")
+ async def claim_pending_copilot_turn_for_finalisation(
+ self,
+ organization_id: str,
+ workflow_copilot_chat_id: str,
+ turn_id: str,
+ ) -> Literal["claimed", "reconciling", "no_marker"]:
+ """ "claimed" means the live handler owns the turn and "reconciling" that reconcile does. Call it inside
+ ``hold_copilot_turn_finalisation``: reconcile cannot claim a turn while that lock is held."""
+ async with self.Session() as session:
+ chat = (
+ await session.scalars(
+ select(WorkflowCopilotChatModel)
+ .where(WorkflowCopilotChatModel.organization_id == organization_id)
+ .where(WorkflowCopilotChatModel.workflow_copilot_chat_id == workflow_copilot_chat_id)
+ .with_for_update()
+ )
+ ).first()
+ if chat is None:
+ return "no_marker"
+ entry = (chat.pending_turns or {}).get(turn_id)
+ if not isinstance(entry, dict):
+ return "no_marker"
+ return "reconciling" if entry.get("recovering_at") is not None else "claimed"
+
+ @asynccontextmanager
+ async def hold_copilot_turn_finalisation(self, workflow_copilot_chat_id: str, turn_id: str) -> AsyncIterator[None]:
+ """Keep reconcile off this turn while the live handler writes its outcome. The lock lives on this
+ transaction's connection, so a dead process releases it and a stalled live one keeps it."""
+ if self._db_engine is None:
+ raise RuntimeError("Copilot turn finalisation locking requires a database engine")
+ key = _turn_finalisation_lock_key(workflow_copilot_chat_id, turn_id)
+ if self._db_engine.dialect.name == "sqlite":
+ _SQLITE_FINALISING_TURNS.add(key)
+ try:
+ yield
+ finally:
+ _SQLITE_FINALISING_TURNS.discard(key)
+ return
+ # A raw connection, never the ambient session: the handler's own writes commit on their own.
+ async with self._db_engine.connect() as connection, connection.begin():
+ await connection.execute(text("SELECT pg_advisory_xact_lock(hashtext(:key))"), {"key": key})
+ yield
+
@db_operation("clear_pending_copilot_turn")
async def clear_pending_copilot_turn(
self,
diff --git a/skyvern/forge/sdk/routes/workflow_copilot.py b/skyvern/forge/sdk/routes/workflow_copilot.py
index 5c4691700..ae262352c 100644
--- a/skyvern/forge/sdk/routes/workflow_copilot.py
+++ b/skyvern/forge/sdk/routes/workflow_copilot.py
@@ -7,10 +7,10 @@ import json
import re
import time
import uuid
-from collections.abc import Iterator
+from collections.abc import Awaitable, Coroutine, Iterator
from contextlib import contextmanager
from datetime import UTC, datetime, timedelta
-from typing import Any, Literal, cast, get_args
+from typing import Any, Literal, TypeVar, cast, get_args
from urllib.parse import urlparse
import structlog
@@ -190,6 +190,7 @@ ALLOWED_WORKFLOW_COPILOT_AUDIO_CONTENT_TYPES = {
}
LOG = structlog.get_logger()
+_T = TypeVar("_T")
async def _resolve_copilot_agent_handler(
@@ -1175,6 +1176,23 @@ def _is_interrupted_recovery_row(message: WorkflowCopilotChatMessage) -> bool:
return message.turn_outcome is not None and message.turn_outcome.terminal_reason == INTERRUPTED_TERMINAL_REASON
+async def _claim_turn_finalisation(chat: WorkflowCopilotChat, turn_id: str) -> bool:
+ """Whether the live turn still owns its post-agent writes; False once reconcile has claimed it."""
+ claim = await app.DATABASE.workflow_params.claim_pending_copilot_turn_for_finalisation(
+ organization_id=chat.organization_id,
+ workflow_copilot_chat_id=chat.workflow_copilot_chat_id,
+ turn_id=turn_id,
+ )
+ if claim == "reconciling":
+ return False
+ if claim == "claimed":
+ return True
+ # Reconcile drops the marker only after writing its interrupted row, so a missing marker
+ # beside that row means reconcile already finished this turn.
+ existing = await _assistant_row_for_turn(chat, turn_id)
+ return existing is None or not _is_interrupted_recovery_row(existing)
+
+
async def _clear_pending_turn(chat: WorkflowCopilotChat, turn_id: str) -> None:
# A failed clear is self-healing: the reconcile pass sees the turn already
# answered, skips recovery and retries the clear.
@@ -1251,46 +1269,7 @@ async def _persist_turn_messages(
assistant_message: WorkflowCopilotChatMessage | None = None
existing = await _assistant_row_for_turn(chat, turn_id) if turn_id is not None else None
- superseding_recovery = (
- existing is not None
- and _is_interrupted_recovery_row(existing)
- and turn_outcome is not None
- and turn_outcome.terminal_reason != INTERRUPTED_TERMINAL_REASON
- )
- if existing is not None and superseding_recovery and turn_outcome is not None:
- replacement_updates: dict[str, str] = {}
- if (
- turn_outcome.user_message_id is None
- and existing.turn_outcome is not None
- and existing.turn_outcome.user_message_id is not None
- ):
- replacement_updates["user_message_id"] = existing.turn_outcome.user_message_id
- if (
- turn_outcome.request_cancel_token is None
- and existing.turn_outcome is not None
- and existing.turn_outcome.request_cancel_token is not None
- ):
- replacement_updates["request_cancel_token"] = existing.turn_outcome.request_cancel_token
- if replacement_updates:
- turn_outcome = turn_outcome.model_copy(update=replacement_updates)
- # Recovery reached this turn first and wrote an interrupted row. The turn then finished,
- # so its real reply is the truth and replaces that row rather than being dropped.
- LOG.info(
- "Copilot turn finished after being recovered; replacing the interrupted row with its reply",
- workflow_copilot_chat_id=chat.workflow_copilot_chat_id,
- turn_id=turn_id,
- )
- assistant_message = await asyncio.shield(
- app.DATABASE.workflow_params.replace_workflow_copilot_chat_message(
- organization_id=chat.organization_id,
- workflow_copilot_chat_message_id=existing.workflow_copilot_chat_message_id,
- content=assistant_content,
- global_llm_context=global_llm_context,
- turn_outcome=turn_outcome,
- narrative_payload=narrative_payload,
- )
- )
- elif existing is not None:
+ if existing is not None:
LOG.info(
"Copilot turn already has an assistant row; skipping duplicate persist",
workflow_copilot_chat_id=chat.workflow_copilot_chat_id,
@@ -1377,6 +1356,7 @@ def _interrupted_turn_outcome(
),
copilot_effective_mode=effective_mode,
copilot_code_available=code_available or False,
+ interrupted_row_final=True,
)
@@ -2286,7 +2266,7 @@ async def _new_copilot_chat_post(
cancel_watcher: asyncio.Task[None] | None = None
current_code_available = False
turn_index = 0
- effective_mode = _effective_copilot_build_mode(chat_request)
+ effective_mode: PersistedCopilotComposerMode | None = _effective_copilot_build_mode(chat_request)
prior_turn_outcome: TurnOutcome | None = None
def capture_code_mode_opt_out_after_persist() -> None:
@@ -2301,6 +2281,48 @@ async def _new_copilot_chat_post(
turn_id=turn_id,
)
+ # Released in the route's finally, after every post-agent write, so reconcile cannot mark the
+ # row final while this handler can still write.
+ finalisation_fence = contextlib.AsyncExitStack()
+ ownership_claim: list[asyncio.Future[bool]] = []
+ post_agent_writes: list[asyncio.Future[Any]] = []
+
+ async def claim_turn_ownership() -> bool:
+ if chat is None or not turn_started:
+ return True
+ await finalisation_fence.enter_async_context(
+ app.DATABASE.workflow_params.hold_copilot_turn_finalisation(chat.workflow_copilot_chat_id, turn_id)
+ )
+ owned = await _claim_turn_finalisation(chat, turn_id)
+ if not owned:
+ LOG.info(
+ "Copilot turn finished after reconcile claimed it; leaving its interrupted row final",
+ organization_id=organization.organization_id,
+ workflow_copilot_chat_id=chat.workflow_copilot_chat_id,
+ turn_id=turn_id,
+ )
+ return owned
+
+ def turn_still_owned() -> Awaitable[bool]:
+ # One claim per handler: a second concurrent acquire of the lock would wait on this one.
+ if not ownership_claim:
+ ownership_claim.append(asyncio.ensure_future(claim_turn_ownership()))
+ return asyncio.shield(ownership_claim[0])
+
+ async def turn_owned_or_left_to_reconcile() -> bool:
+ # For error arms with nothing else to report: an unclaimed marker stays for reconcile.
+ try:
+ return await turn_still_owned()
+ except Exception:
+ LOG.warning("Could not claim the copilot turn's finalisation", turn_id=turn_id, exc_info=True)
+ return False
+
+ def tracked_write(write: Coroutine[Any, Any, _T]) -> Awaitable[_T]:
+ # Shielded writes outlive a cancelled handler; the finally waits for them before releasing the lock.
+ task = asyncio.ensure_future(write)
+ post_agent_writes.append(task)
+ return asyncio.shield(task)
+
async def _emit_unpersisted_failure(
failure: RecoverableFailure,
*,
@@ -2348,7 +2370,17 @@ async def _new_copilot_chat_post(
nonlocal failure_kind_for_log, route_exit, terminal_frame_emitted, turn_outcome_for_log
route_exit = "error"
failure_kind_for_log = failure_kind
- if any(isinstance(item, DatabaseConnectionUnavailableError) for item in iter_exception_chain(exc)):
+ database_down = any(
+ isinstance(item, DatabaseConnectionUnavailableError) for item in iter_exception_chain(exc)
+ )
+ claim_failure: Exception | None = None
+ if not database_down:
+ try:
+ if not await turn_still_owned():
+ return
+ except Exception as error:
+ claim_failure = error
+ if database_down or claim_failure is not None:
# Rolling the workflow back and writing the reply are both database work, against a
# database that just exhausted a full reconnection budget. Report what is already
# known rather than spending more of the caller's wait on writes that cannot land;
@@ -2372,7 +2404,7 @@ async def _new_copilot_chat_post(
await _emit_unpersisted_failure(
failure,
failure_kind=failure_kind,
- writer_exc=exc,
+ writer_exc=claim_failure or exc,
)
return
restored = chat is not None and _should_restore_persisted_workflow(
@@ -2453,7 +2485,7 @@ async def _new_copilot_chat_post(
else chat_request.model_copy(update={"message": UNSCREENED_MESSAGE_PLACEHOLDER})
)
try:
- await asyncio.shield(
+ await tracked_write(
_finalise_normal_turn(
stream=stream,
chat=chat,
@@ -2860,25 +2892,39 @@ async def _new_copilot_chat_post(
# carrying ``workflow_was_persisted`` so rollback proceeds normally.
# Nobody pressed Stop on a cancellation the user never asked for, so
# that turn is recorded as interrupted rather than as their intent.
- await _persist_cancel_turn(
- stream=stream,
- chat=chat,
- organization_id=organization.organization_id,
- original_workflow=original_workflow,
- user_message=chat_request.message,
- agent_result=agent_result,
- audio_artifact_id=chat_request.audio_artifact_id,
- turn_id=turn_id,
- keep_pending_proposal=chat_request.keep_pending_proposal,
- user_row_already_persisted=turn_started,
- sender=_turn_opener_sender(chat_request),
- record_as_interrupted=not user_cancel_observed[0],
- cancel_source=user_cancel_source[0],
- effective_mode=effective_mode,
- code_available=current_code_available,
- request_cancel_token=chat_request.cancel_token,
- )
- terminal_frame_emitted = True
+ # Set before the shielded write for the same reason as the success path below.
+ finalise_started = True
+ cancelled_agent_result = agent_result
+
+ async def persist_cancel_if_owned() -> bool:
+ nonlocal terminal_frame_emitted
+ if not await turn_still_owned():
+ return False
+ await _persist_cancel_turn(
+ stream=stream,
+ chat=chat,
+ organization_id=organization.organization_id,
+ original_workflow=original_workflow,
+ user_message=chat_request.message,
+ agent_result=cancelled_agent_result,
+ audio_artifact_id=chat_request.audio_artifact_id,
+ turn_id=turn_id,
+ keep_pending_proposal=chat_request.keep_pending_proposal,
+ user_row_already_persisted=turn_started,
+ sender=_turn_opener_sender(chat_request),
+ record_as_interrupted=not user_cancel_observed[0],
+ cancel_source=user_cancel_source[0],
+ effective_mode=effective_mode,
+ code_available=current_code_available,
+ request_cancel_token=chat_request.cancel_token,
+ )
+ # Set inside the shield: a cancelled handler skips the line after its await,
+ # and the finally would send a second, false terminal frame.
+ terminal_frame_emitted = True
+ return True
+
+ if not await tracked_write(persist_cancel_if_owned()):
+ return
capture_code_mode_opt_out_after_persist()
LOG.info(
"Workflow copilot agent turn cancelled",
@@ -2892,21 +2938,30 @@ async def _new_copilot_chat_post(
# Atomic finalisation — a late cancel that fires here cannot tear
# the success-path writes apart mid-way (no half-written turn,
- # no duplicate user/AI rows).
+ # no duplicate user/AI rows). The ownership claim sits inside the
+ # shield for the same reason.
finalise_started = True
- await asyncio.shield(
- _finalise_normal_turn(
+ finalised_agent_result = agent_result
+
+ async def finalise_if_owned() -> bool:
+ nonlocal terminal_frame_emitted
+ if not await turn_still_owned():
+ return False
+ await _finalise_normal_turn(
stream=stream,
chat=chat,
organization_id=organization.organization_id,
original_workflow=original_workflow,
chat_request=chat_request,
- agent_result=agent_result,
+ agent_result=finalised_agent_result,
turn_id=turn_id,
user_row_already_persisted=turn_started,
)
- )
- terminal_frame_emitted = True
+ terminal_frame_emitted = True
+ return True
+
+ if not await tracked_write(finalise_if_owned()):
+ return
capture_code_mode_opt_out_after_persist()
route_exit = "completed"
except HTTPException as exc:
@@ -2924,9 +2979,13 @@ async def _new_copilot_chat_post(
workflow_permanent_id=chat_request.workflow_permanent_id,
exc_info=exc.status_code >= 500,
)
- if chat is not None and _should_restore_persisted_workflow(
- chat.auto_accept,
- agent_result,
+ if (
+ chat is not None
+ and _should_restore_persisted_workflow(
+ chat.auto_accept,
+ agent_result,
+ )
+ and await turn_owned_or_left_to_reconcile()
):
try:
await _restore_workflow_definition(original_workflow, organization.organization_id)
@@ -2958,12 +3017,14 @@ async def _new_copilot_chat_post(
except asyncio.CancelledError:
route_exit = "cancelled"
turn_outcome_for_log = agent_result.turn_outcome if agent_result is not None else None
+ if not await turn_owned_or_left_to_reconcile():
+ raise
if chat is not None and _should_restore_persisted_workflow(
chat.auto_accept,
agent_result,
):
try:
- await asyncio.shield(_restore_workflow_definition(original_workflow, organization.organization_id))
+ await tracked_write(_restore_workflow_definition(original_workflow, organization.organization_id))
except Exception:
LOG.warning(
"Workflow restore failed inside cancel-error handler",
@@ -2975,7 +3036,7 @@ async def _new_copilot_chat_post(
# the agent_result.cancelled branch above couldn't run.
# _persist_cancel_turn skips rollback when agent_result is None.
try:
- await asyncio.shield(
+ await tracked_write(
_persist_cancel_turn(
stream=stream,
chat=chat,
@@ -3037,7 +3098,7 @@ async def _new_copilot_chat_post(
workflow_copilot_chat_id=chat.workflow_copilot_chat_id,
turn_id=turn_id,
):
- await asyncio.shield(
+ await tracked_write(
_persist_interrupted_turn(
chat,
turn_id,
@@ -3067,6 +3128,24 @@ async def _new_copilot_chat_post(
)
finally:
try:
+ # A cancelled handler reaches here while its shielded writes still run. A further cancel
+ # must not cut the wait short, or the lock drops under a live write; it is re-raised below.
+ cancelled_while_waiting = False
+ for pending_write in (*ownership_claim, *post_agent_writes):
+ while not pending_write.done():
+ try:
+ await asyncio.shield(pending_write)
+ except asyncio.CancelledError:
+ cancelled_while_waiting = cancelled_while_waiting or not pending_write.done()
+ except Exception:
+ break
+ try:
+ await finalisation_fence.aclose()
+ except Exception:
+ with contained_effect("copilot turn finalisation lock release failure", turn_id=turn_id):
+ LOG.warning(
+ "Could not release the copilot turn finalisation lock", turn_id=turn_id, exc_info=True
+ )
if cancel_watcher is not None and not cancel_watcher.done():
cancel_watcher.cancel()
with contextlib.suppress(asyncio.CancelledError, Exception):
@@ -3090,6 +3169,8 @@ async def _new_copilot_chat_post(
browser_session_id,
reason=BrowserSessionCloseReason.user_requested,
)
+ if cancelled_while_waiting:
+ raise asyncio.CancelledError
finally:
with contained_effect("copilot turn outcome log"):
LOG.info(
@@ -4431,7 +4512,7 @@ def convert_to_history_messages(
feedback=getattr(message, "feedback", None),
audio_artifact_id=message.audio_artifact_id,
attached_files=message.attached_files,
- turn_outcome=message.turn_outcome,
+ turn_outcome=_served_turn_outcome(message.turn_outcome),
created_at=message.created_at,
modified_at=getattr(message, "modified_at", message.created_at),
narrative_payload=message.narrative_payload,
@@ -4440,6 +4521,14 @@ def convert_to_history_messages(
]
+def _served_turn_outcome(outcome: TurnOutcome | None) -> TurnOutcome | None:
+ # Nothing in this backend replaces a stored assistant row, so an interrupted row written before
+ # the flag existed is final too; serving it unflagged would hold the chat's save lock forever.
+ if outcome is None or outcome.terminal_reason != INTERRUPTED_TERMINAL_REASON or outcome.interrupted_row_final:
+ return outcome
+ return outcome.model_copy(update={"interrupted_row_final": True})
+
+
@base_router.post("/workflow/copilot/suggest-goal", include_in_schema=False)
async def workflow_copilot_suggest_goal(
suggestion_request: WorkflowCopilotGoalSuggestionRequest,
diff --git a/skyvern/forge/sdk/schemas/copilot_turn_outcome.py b/skyvern/forge/sdk/schemas/copilot_turn_outcome.py
index 8003b5dbe..cf3b9cc3e 100644
--- a/skyvern/forge/sdk/schemas/copilot_turn_outcome.py
+++ b/skyvern/forge/sdk/schemas/copilot_turn_outcome.py
@@ -117,6 +117,9 @@ class TurnOutcome(BaseModel):
# Client-generated per-request correlation value. Retained after the pending
# marker is cleared so a disconnected client can recover the exact chat.
request_cancel_token: str | None = None
+ # True on an interrupted row no late finalizer can replace. Chat history serves every stored
+ # interrupted row this way; False only reaches a client from a backend that still replaced rows.
+ interrupted_row_final: bool = False
idempotency_digest: str | None = None
unresolved_runtime_failure: UnresolvedRuntimeFailure | None = None
connected_account_choices: list[ConnectedAccountChoice] | None = None
diff --git a/skyvern/forge/sdk/schemas/workflow_copilot.py b/skyvern/forge/sdk/schemas/workflow_copilot.py
index 6f5e9d6af..62c7dbb69 100644
--- a/skyvern/forge/sdk/schemas/workflow_copilot.py
+++ b/skyvern/forge/sdk/schemas/workflow_copilot.py
@@ -149,6 +149,9 @@ class CopilotPendingTurn(BaseModel):
copilot_code_available: bool = False
user_message_id: str | None = None
recovering_at: datetime | None = None
+ # A credential pause stops the turn's budget clock, so reconcile measures abandonment from here
+ # too, as it does from a resolved question.
+ credential_resumed_at: datetime | None = None
# Fingerprint of the canonical workflow as this turn last left it. None means the turn
# never wrote canonical, so it owns no write to roll back.
canonical_write_fingerprint: str | None = None
diff --git a/tests/unit/copilot_route_test_support.py b/tests/unit/copilot_route_test_support.py
index 016f98869..965ba226f 100644
--- a/tests/unit/copilot_route_test_support.py
+++ b/tests/unit/copilot_route_test_support.py
@@ -6,6 +6,8 @@ DB / LLM / agent mocks; these helpers keep that wiring in one place.
from __future__ import annotations
+from collections.abc import AsyncIterator
+from contextlib import asynccontextmanager
from datetime import datetime, timezone
from types import SimpleNamespace
from typing import Any
@@ -51,6 +53,11 @@ def narrative_payload_with_run(run_id: str | None) -> dict[str, Any]:
}
+@asynccontextmanager
+async def no_finalisation_fence(*_: object) -> AsyncIterator[None]:
+ yield
+
+
def install_fake_create(monkeypatch: pytest.MonkeyPatch) -> dict[str, object]:
"""Capture the stream handler that the route hands to EventSourceStream."""
captured: dict[str, object] = {}
@@ -148,6 +155,8 @@ def setup_new_copilot_mocks(
),
replace_workflow_copilot_chat_message=AsyncMock(),
claim_pending_copilot_turn=AsyncMock(return_value=True),
+ claim_pending_copilot_turn_for_finalisation=AsyncMock(return_value="claimed"),
+ hold_copilot_turn_finalisation=no_finalisation_fence,
clear_pending_copilot_turn=AsyncMock(),
)
app.DATABASE.workflow_params = workflow_params
diff --git a/tests/unit/test_copilot_credential_pause.py b/tests/unit/test_copilot_credential_pause.py
index 01d760a1e..2511621e7 100644
--- a/tests/unit/test_copilot_credential_pause.py
+++ b/tests/unit/test_copilot_credential_pause.py
@@ -394,10 +394,14 @@ async def test_connected_action_mutates_policy_and_resolves(monkeypatch: pytest.
)
monkeypatch.setattr(credential_pause_module.app._inst, "CACHE", cache, raising=False)
credential = _make_credential()
+ record_resume = AsyncMock()
monkeypatch.setattr(
credential_pause_module.app,
"DATABASE",
- SimpleNamespace(credentials=SimpleNamespace(get_credentials_by_ids=AsyncMock(return_value=[credential]))),
+ SimpleNamespace(
+ credentials=SimpleNamespace(get_credentials_by_ids=AsyncMock(return_value=[credential])),
+ workflow_params=SimpleNamespace(record_pending_copilot_turn_credential_resume=record_resume),
+ ),
)
monkeypatch.setattr(credential_pause_module, "CREDENTIAL_RESPONSE_POLL_SECONDS", 0.01)
@@ -407,6 +411,8 @@ async def test_connected_action_mutates_policy_and_resolves(monkeypatch: pytest.
resume_msgs = await maybe_credential_pause(ctx, _fake_result(), stream, config)
assert resume_msgs is not None
+ # Reconcile measures abandonment from this stamp, so a resumed turn is not recovered as dead.
+ record_resume.assert_awaited_with(organization_id="org-1", workflow_copilot_chat_id="chat-1", turn_id="turn-1")
assert ctx.credential_pause_outcome == "connected"
assert ctx.request_policy.resolved_credentials == [credential]
assert ctx.request_policy.allow_run_blocks is True
diff --git a/tests/unit/test_workflow_copilot_cancel.py b/tests/unit/test_workflow_copilot_cancel.py
index 8c7c9e8db..7bfaa46aa 100644
--- a/tests/unit/test_workflow_copilot_cancel.py
+++ b/tests/unit/test_workflow_copilot_cancel.py
@@ -21,6 +21,8 @@ Covers:
from __future__ import annotations
import asyncio
+from collections.abc import AsyncIterator
+from contextlib import asynccontextmanager
from datetime import datetime, timedelta, timezone
from types import SimpleNamespace
from typing import Any
@@ -62,10 +64,11 @@ from skyvern.forge.sdk.schemas.workflow_copilot import (
WorkflowCopilotChat,
WorkflowCopilotChatRequest,
WorkflowCopilotChatSender,
+ WorkflowCopilotStreamErrorUpdate,
WorkflowCopilotStreamMessageType,
)
from tests.unit._copilot_workflow_fakes import fake_workflow
-from tests.unit.copilot_route_test_support import install_fake_create, setup_new_copilot_mocks
+from tests.unit.copilot_route_test_support import install_fake_create, no_finalisation_fence, setup_new_copilot_mocks
class _FakeCache:
@@ -1274,10 +1277,13 @@ async def test_watcher_reports_the_gesture_the_cancel_named() -> None:
@pytest.mark.asyncio
[email protected]("claim", ["claimed", "reconciling"])
async def test_operational_cancel_records_the_turn_as_interrupted_and_still_re_raises(
- monkeypatch: pytest.MonkeyPatch,
+ monkeypatch: pytest.MonkeyPatch, claim: str
) -> None:
- """task.cancel() without user_cancel_observed[0] -> an interrupted row, never a user-cancel row."""
+ """task.cancel() without user_cancel_observed[0] -> an interrupted row, never a user-cancel row.
+
+ Once reconcile owns the turn, its row is the only one; the drain must not write another."""
captured = install_fake_create(monkeypatch)
chat = SimpleNamespace(
@@ -1325,6 +1331,8 @@ async def test_operational_cancel_records_the_turn_as_interrupted_and_still_re_r
start_copilot_turn=AsyncMock(
return_value=SimpleNamespace(created_at=datetime(2026, 4, 27, tzinfo=timezone.utc))
),
+ claim_pending_copilot_turn_for_finalisation=AsyncMock(return_value=claim),
+ hold_copilot_turn_finalisation=no_finalisation_fence,
clear_pending_copilot_turn=AsyncMock(),
)
app.DATABASE.workflow_params = workflow_params
@@ -1360,10 +1368,14 @@ async def test_operational_cancel_records_the_turn_as_interrupted_and_still_re_r
contents = [c.kwargs.get("content") for c in insert_calls]
assert MINIMAL_CANCEL_STOP not in contents
interrupted = [content for content in contents if content and "interrupted" in content.lower()]
+ if claim == "reconciling":
+ assert interrupted == []
+ return
assert len(interrupted) == 1
assert "wpid-1" in interrupted[0]
written = insert_calls[-1].kwargs
assert written["turn_outcome"].terminal_reason == INTERRUPTED_TERMINAL_REASON
+ assert written["turn_outcome"].interrupted_row_final is True
assert written["narrative_payload"]["terminalMessage"] == written["content"]
@@ -1483,6 +1495,88 @@ async def test_cancel_during_shielded_finalisation_does_not_write_a_second_row(
assert interrupted == [], f"finalisation owns this turn's row; got a competing write: {interrupted}"
[email protected]("stall", ["claim", "finalise", "error recovery", "cancelled result"])
+async def test_a_drained_handler_keeps_the_finalisation_lock_until_its_shielded_writes_finish(
+ monkeypatch: pytest.MonkeyPatch, stall: str
+) -> None:
+ """Releasing on the cancel would let reconcile mark the row final while the finalise still writes,
+ and a cancel during the ownership claim must not turn a finished turn into an interrupted one."""
+ captured = install_fake_create(monkeypatch)
+ chat = _make_chat(auto_accept=False)
+ agent_result = AgentResult(
+ user_response="Here is your workflow.",
+ updated_workflow=None,
+ global_llm_context=None,
+ workflow_yaml=None,
+ cancelled=stall == "cancelled result",
+ turn_outcome=None,
+ )
+ _, workflow_params = setup_new_copilot_mocks(monkeypatch, chat, _make_original_workflow(), agent_result)
+ events: list[str] = []
+ stalled = asyncio.Event()
+ may_continue = asyncio.Event()
+
+ @asynccontextmanager
+ async def recording_fence(*_: object) -> AsyncIterator[None]:
+ if stall == "claim":
+ stalled.set()
+ await may_continue.wait()
+ events.append("locked")
+ try:
+ yield
+ finally:
+ events.append("released")
+
+ async def slow_finalise(*args: object, **kwargs: object) -> None:
+ del args, kwargs
+ if stall in ("finalise", "error recovery", "cancelled result"):
+ stalled.set()
+ await may_continue.wait()
+ events.append("finalise done")
+
+ workflow_params.hold_copilot_turn_finalisation = recording_fence
+ monkeypatch.setattr("skyvern.forge.sdk.routes.workflow_copilot._finalise_normal_turn", slow_finalise)
+ monkeypatch.setattr("skyvern.forge.sdk.routes.workflow_copilot._persist_cancel_turn", slow_finalise)
+ if stall == "error recovery":
+ monkeypatch.setattr(
+ "skyvern.forge.sdk.routes.workflow_copilot.run_copilot_agent",
+ AsyncMock(side_effect=RuntimeError("agent failed")),
+ )
+ monkeypatch.setattr(app._inst, "CACHE", None, raising=False)
+ request = MagicMock()
+ request.headers = {"x-api-key": "sk-test"}
+ await workflow_copilot_chat_post(
+ request, _make_chat_request(cancel_token=None), SimpleNamespace(organization_id="org-1")
+ )
+ stream = MagicMock()
+ stream.send = AsyncMock(return_value=True)
+ stream.is_disconnected = AsyncMock(return_value=False)
+
+ handler_task = asyncio.create_task(captured["handler"](stream))
+ await stalled.wait()
+ # A client disconnect then a drain: the second cancel lands while the release waits.
+ for _ in range(2):
+ handler_task.cancel()
+ for _ in range(5):
+ await asyncio.sleep(0)
+ assert "released" not in events
+ may_continue.set()
+ with pytest.raises(asyncio.CancelledError):
+ await handler_task
+
+ assert events == ["locked", "finalise done", "released"]
+ contents = [c.kwargs.get("content") for c in workflow_params.create_workflow_copilot_chat_message.await_args_list]
+ assert not [content for content in contents if content and "interrupted" in content.lower()]
+ if stall != "error recovery":
+ sent_errors = [
+ call.args[0].error
+ for call in stream.send.await_args_list
+ if isinstance(call.args[0], WorkflowCopilotStreamErrorUpdate)
+ ]
+ assert "The assistant didn't finish this turn. Please try again." not in sent_errors
+
+
@pytest.mark.asyncio
async def test_a_stop_before_the_agent_starts_still_reports_a_cause_when_its_row_cannot_be_written(
monkeypatch: pytest.MonkeyPatch,
diff --git a/tests/unit/test_workflow_copilot_route.py b/tests/unit/test_workflow_copilot_route.py
index 876752804..f0b247933 100644
--- a/tests/unit/test_workflow_copilot_route.py
+++ b/tests/unit/test_workflow_copilot_route.py
@@ -11,6 +11,7 @@ import asyncio
import contextlib
import json
import re
+from collections.abc import AsyncIterator
from datetime import UTC, datetime, timedelta
from pathlib import Path
from types import SimpleNamespace
@@ -18,9 +19,11 @@ from typing import Any
from unittest.mock import AsyncMock, MagicMock
import pytest
+import pytest_asyncio
import yaml
from fastapi import FastAPI, HTTPException
from httpx import ASGITransport, AsyncClient
+from sqlalchemy.engine import make_url
from sqlalchemy.ext.asyncio import AsyncEngine
from structlog.testing import capture_logs
@@ -63,6 +66,7 @@ from skyvern.forge.sdk.copilot.interruption import (
from skyvern.forge.sdk.copilot.tools import workflow_update
from skyvern.forge.sdk.copilot.turn_outcome import build_minimal_turn_outcome
from skyvern.forge.sdk.copilot.workflow_yaml import _process_workflow_yaml, dump_workflow_yaml
+from skyvern.forge.sdk.db.agent_db import _build_engine
from skyvern.forge.sdk.db.base_alchemy_db import BaseAlchemyDB
from skyvern.forge.sdk.db.exceptions import CopilotProposalConflictError, DatabaseConnectionUnavailableError
from skyvern.forge.sdk.db.repositories.workflow_parameters import (
@@ -6002,10 +6006,10 @@ async def test_a_turn_that_never_started_keeps_its_attachments_on_the_recovery_r
@pytest.mark.asyncio
-async def test_a_finished_turn_replaces_the_interrupted_row_rather_than_dropping_its_reply(
+async def test_an_interrupted_row_stored_before_the_final_flag_is_served_final(
monkeypatch: pytest.MonkeyPatch,
) -> None:
- """Recovery can reach a slow-but-live turn first; when that turn finishes, its reply is the truth."""
+ """Nothing replaces stored rows anymore, so an unflagged one must not hold the chat's save lock forever."""
chat = _make_persisted_chat([])
store, _ = _install_reconcile_store(monkeypatch, chat)
store.add_message(WorkflowCopilotChatSender.USER, "build me a scraper")
@@ -6014,34 +6018,111 @@ async def test_a_finished_turn_replaces_the_interrupted_row_rather_than_dropping
INTERRUPTED_TERMINAL_MESSAGE,
TurnOutcome(
response_kind=ResponseKind.RECOVER,
- reason_code=INTERRUPTED_TERMINAL_REASON,
terminal_reason=INTERRUPTED_TERMINAL_REASON,
copilot_turn_id="turn-a",
- user_message_id="wccm-0",
- request_cancel_token="cancel-turn-a",
),
)
- await _persist_turn_messages(
- chat=chat,
- turn_id="turn-a",
+ response = await _load_history()
+
+ outcome = response.chat_history[-1].turn_outcome
+ assert outcome is not None
+ assert outcome.interrupted_row_final is True
+
+
+async def _start_abandoned_turn(repo: WorkflowParametersRepository, turn_id: str) -> str:
+ chat = await repo.create_workflow_copilot_chat(organization_id="org", workflow_permanent_id="wpid")
+ await repo.start_copilot_turn(
+ organization_id="org",
+ workflow_copilot_chat_id=chat.workflow_copilot_chat_id,
+ pending_turn=CopilotPendingTurn(
+ turn_id=turn_id,
+ started_at=datetime.now(UTC) - timedelta(seconds=RECONCILE_ABANDON_AFTER_SECONDS + 60),
+ ),
user_message="build me a scraper",
- audio_artifact_id=None,
- user_row_already_persisted=True,
- sender=WorkflowCopilotChatSender.USER,
- assistant_content="Here is your workflow.",
- global_llm_context=None,
- turn_outcome=TurnOutcome(response_kind=ResponseKind.BUILD),
- narrative_payload=None,
)
+ return chat.workflow_copilot_chat_id
- assert len(store.assistant_messages) == 1
- assert store.assistant_messages[0].content == "Here is your workflow."
- outcome = store.assistant_messages[0].turn_outcome
- assert outcome is not None
- assert outcome.terminal_reason != INTERRUPTED_TERMINAL_REASON
- assert outcome.user_message_id == "wccm-0"
- assert outcome.request_cancel_token == "cancel-turn-a"
+
+async def test_a_turn_finishing_after_reconcile_claimed_it_cannot_write(
+ sqlite_engine: AsyncEngine, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ """Reconcile's interrupted row is final: the still-live turn loses its claim mid-recovery and after."""
+ repo = WorkflowParametersRepository(BaseAlchemyDB(sqlite_engine).Session)
+ monkeypatch.setattr(app.DATABASE, "workflow_params", repo)
+ mid_chat_id = await _start_abandoned_turn(repo, "turn-mid")
+ assert await repo.claim_pending_copilot_turn("org", mid_chat_id, "turn-mid", datetime.now(UTC))
+ mid_chat = await repo.get_workflow_copilot_chat_by_id("org", mid_chat_id)
+ assert not await workflow_copilot_route._claim_turn_finalisation(mid_chat, "turn-mid")
+
+ chat_id = await _start_abandoned_turn(repo, "turn-done")
+ chat = await repo.get_workflow_copilot_chat_by_id("org", chat_id)
+ await workflow_copilot_route._reconcile_interrupted_copilot_turns(chat, "org")
+
+ [row] = [m for m in await repo.get_workflow_copilot_chat_messages(chat_id) if m.sender == "ai"]
+ assert row.turn_outcome is not None
+ assert row.turn_outcome.terminal_reason == INTERRUPTED_TERMINAL_REASON
+ assert row.turn_outcome.interrupted_row_final is True
+ assert not await workflow_copilot_route._claim_turn_finalisation(chat, "turn-done")
+
+
+async def test_reconcile_leaves_a_turn_that_just_resumed_from_a_credential_pause(
+ sqlite_engine: AsyncEngine, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ """Two credential pauses outlast the abandon window from the turn's start while it is still working."""
+ repo = WorkflowParametersRepository(BaseAlchemyDB(sqlite_engine).Session)
+ monkeypatch.setattr(app.DATABASE, "workflow_params", repo)
+ chat_id = await _start_abandoned_turn(repo, "turn-a")
+ await repo.record_pending_copilot_turn_credential_resume("org", chat_id, "turn-a")
+ abandoned_before = datetime.now(UTC) - timedelta(seconds=RECONCILE_ABANDON_AFTER_SECONDS)
+ assert not await repo.claim_pending_copilot_turn("org", chat_id, "turn-a", abandoned_before)
+
+ await workflow_copilot_route._reconcile_interrupted_copilot_turns(
+ await repo.get_workflow_copilot_chat_by_id("org", chat_id), "org"
+ )
+
+ assert [m for m in await repo.get_workflow_copilot_chat_messages(chat_id) if m.sender == "ai"] == []
+ chat = await repo.get_workflow_copilot_chat_by_id("org", chat_id)
+ assert await workflow_copilot_route._claim_turn_finalisation(chat, "turn-a")
+
+
+@pytest_asyncio.fixture(params=["sqlite", "postgres"])
+async def fence_repo(
+ request: pytest.FixtureRequest, sqlite_engine: AsyncEngine
+) -> AsyncIterator[WorkflowParametersRepository]:
+ if request.param == "sqlite":
+ yield WorkflowParametersRepository(BaseAlchemyDB(sqlite_engine).Session, db_engine=sqlite_engine)
+ return
+ if make_url(str(settings.DATABASE_STRING)).get_backend_name() != "postgresql":
+ pytest.skip("requires PostgreSQL")
+ engine = _build_engine(settings.DATABASE_STRING)
+ try:
+ yield WorkflowParametersRepository(BaseAlchemyDB(engine).Session, db_engine=engine)
+ finally:
+ await engine.dispose()
+
+
+async def test_reconcile_waits_for_a_finalising_handler_and_answers_once_its_lock_is_gone(
+ fence_repo: WorkflowParametersRepository, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ """A stalled handler keeps its lock and so its turn; a dead one's connection takes the lock with it."""
+ monkeypatch.setattr(app.DATABASE, "workflow_params", fence_repo)
+ chat_id = await _start_abandoned_turn(fence_repo, "turn-a")
+
+ async def reconcile() -> list[WorkflowCopilotChatMessage]:
+ chat = await fence_repo.get_workflow_copilot_chat_by_id("org", chat_id)
+ await workflow_copilot_route._reconcile_interrupted_copilot_turns(chat, "org")
+ return [m for m in await fence_repo.get_workflow_copilot_chat_messages(chat_id) if m.sender == "ai"]
+
+ async with fence_repo.hold_copilot_turn_finalisation(chat_id, "turn-a"):
+ assert await reconcile() == []
+
+ [row] = await reconcile()
+ assert row.turn_outcome is not None
+ assert row.turn_outcome.interrupted_row_final is True
def test_reconcile_threshold_outlasts_the_turn_enforcement_ceiling() -> None:
@@ -6098,6 +6179,58 @@ async def test_marker_survives_a_finalizer_that_raises_with_no_assistant_row(
app.DATABASE.workflow_params.clear_pending_copilot_turn.assert_not_awaited()
[email protected]("ending", ["finished", "cancelled", "errored"])
+async def test_a_turn_reconcile_claimed_leaves_canonical_and_its_row_alone(
+ monkeypatch: pytest.MonkeyPatch, api_key_request: MagicMock, copilot_stream: MagicMock, ending: str
+) -> None:
+ """Rolling canonical back now could overwrite a save the user made once the interrupted row unlocked it."""
+ captured = install_fake_create(monkeypatch)
+ chat = SimpleNamespace(
+ workflow_copilot_chat_id="chat-1",
+ workflow_permanent_id="wpid-1",
+ organization_id="org-1",
+ proposed_workflow=None,
+ auto_accept=False,
+ )
+ original_workflow = SimpleNamespace(
+ workflow_id="wf-canonical",
+ title="Original",
+ description="Original description",
+ workflow_definition=None,
+ )
+ agent_result = AgentResult(
+ user_response="Here is your workflow.",
+ updated_workflow=None,
+ global_llm_context=None,
+ workflow_yaml=None,
+ workflow_was_persisted=True,
+ clear_proposed_workflow=False,
+ authoring_barred=False,
+ resolved_model=None,
+ turn_outcome=None,
+ cancelled=ending == "cancelled",
+ has_staged_proposal=False,
+ )
+ restore_mock, workflow_params = setup_new_copilot_mocks(monkeypatch, chat, original_workflow, agent_result)
+ workflow_params.claim_pending_copilot_turn_for_finalisation.return_value = "reconciling"
+ if ending == "errored":
+ monkeypatch.setattr(
+ workflow_copilot_route, "run_copilot_agent", AsyncMock(side_effect=RuntimeError("late failure"))
+ )
+ created_rows = workflow_params.create_workflow_copilot_chat_message.await_count
+
+ await workflow_copilot_chat_post(api_key_request, _make_chat_request(), SimpleNamespace(organization_id="org-1"))
+ handler = captured["handler"]
+ assert callable(handler)
+ await handler(copilot_stream)
+
+ restore_mock.assert_not_awaited()
+ workflow_params.update_workflow_copilot_chat.assert_not_awaited()
+ assert workflow_params.create_workflow_copilot_chat_message.await_count == created_rows
+ workflow_params.clear_pending_copilot_turn.assert_not_awaited()
+
+
@pytest.mark.asyncio
async def test_reconcile_writes_no_outcome_when_the_canonical_restore_fails(
monkeypatch: pytest.MonkeyPatch,