Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Release the Copilot save hold on an interrupted row the server marks final (SKY-17648) (#8775)
Andrew Neilsonbe9f86894949
diff --git a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.historyRace.test.tsx b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.historyRace.test.tsxindex 4ae1192b4..2b24a158d 100644--- a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.historyRace.test.tsx+++ b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.historyRace.test.tsx@@ -891,7 +891,7 @@ describe("WorkflowCopilotChat — recovery poll after a non-terminal stream clos});}- function interruptedHistory(): HistoryData {+ function interruptedHistory(final = false): HistoryData {return historyWithRow({sender: "ai",content: interruptedText,@@ -901,6 +901,7 @@ describe("WorkflowCopilotChat — recovery poll after a non-terminal stream closresponse_kind: "recover",terminal_reason: "interrupted",copilot_turn_id: turnId,+ ...(final ? { interrupted_row_final: true } : {}),},});}@@ -2790,6 +2791,26 @@ describe("WorkflowCopilotChat — recovery poll after a non-terminal stream closexpect(historyQueue).toHaveLength(0);});+ it("releases the save hold on the first read of an interrupted row the server marked final", async () => {+ await startTurn();+ await closeStreamWithoutTerminal();+ const readsBeforeRow = workflowGets.length;++ await advance(2_000);+ await resolveNextHistory(interruptedHistory(true));+ await act(async () => {+ await Promise.resolve();+ });++ expect(renderedText()).toContain(interruptedText);+ expect(renderedText()).not.toContain(+ "Could not confirm whether Copilot saved changes",+ );+ expect(screen.queryByRole("button", { name: "Retry" })).toBeNull();+ expect(workflowGets.length).toBeGreaterThan(readsBeforeRow);+ expect(useWorkflowYamlEditorStore.getState().copilotAcceptance).toBeNull();+ });+it("rechecks canonical after the terminal row despite unsaved edits", async () => {hasLocalChanges.current = true;await startTurn();diff --git a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.reviewGate.test.tsx b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.reviewGate.test.tsxindex 445b6f7a4..56ea68b5f 100644--- a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.reviewGate.test.tsx+++ b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.reviewGate.test.tsx@@ -15345,6 +15345,52 @@ describe("editor-state rollback lifecycle", () => {useWorkflowHasChangesStore.getState().saveGeneration,).toBeGreaterThan(0);});+ async function interruptStagedTurn(canonical: unknown) {+ const call = await stage();+ historyGet.mockImplementation((path: string) =>+ Promise.resolve(+ path === "/workflows/wpid_1" ? { data: canonical } : historyResponse,+ ),+ );+ vi.useFakeTimers();+ await act(async () => call.reject(new Error("Connection dropped")));+ historyResponse.data.request_turn_id = "turn-1";+ historyResponse.data.chat_history = [+ {+ sender: "ai",+ content: "This turn was interrupted before it could finish.",+ turn_outcome: {+ copilot_turn_id: "turn-1",+ terminal_reason: "interrupted",+ interrupted_row_final: true,+ },+ },+ ];+ await act(async () => vi.advanceTimersByTimeAsync(2_000));+ }+ it("does not credit Copilot with a canonical change it never made when its turn was interrupted", async () => {+ await renderChat();+ await interruptStagedTurn({+ ...proposedWorkflowPayload(),+ workflow_id: "wf_saved",+ version: 2,+ });++ expect(useWorkflowYamlEditorStore.getState().copilotAcceptance).toBeNull();+ expect(toast).not.toHaveBeenCalledWith(+ expect.objectContaining({ title: "The draft could not be restored" }),+ );+ });+ it("restores the pre-turn editor when an interrupted turn left canonical untouched", async () => {+ const restore = vi+ .fn<(snapshot: EditorStateSnapshot) => RestoreResult>()+ .mockImplementation(restoreLive);+ await renderChat({ onRestore: restore });+ await interruptStagedTurn(saveData.workflow);++ expect(restore).toHaveBeenCalledOnce();+ expect(useWorkflowYamlEditorStore.getState().copilotAcceptance).toBeNull();+ });it("reports a stale-workflow refusal during rollback and retains the reservation", async () => {changesState.hasChanges = true;const restore = vidiff --git a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsxindex 4c1c5e3a7..647a73e20 100644--- a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx+++ b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx@@ -374,6 +374,12 @@ const RECOVERY_POLL_STEADY_MS = 30_000;const RECOVERY_POLL_BUDGET_MS = 1_500_000;const CANONICAL_READ_TIMEOUT_MS = 5_000;const INTERRUPTED_TERMINAL_REASON = "interrupted";+// Only an older backend writes an interrupted row a late reply may still replace.+const isReplaceableInterruptedRow = (+ outcome: WorkflowCopilotChatHistoryMessage["turn_outcome"],+): boolean =>+ outcome?.terminal_reason === INTERRUPTED_TERMINAL_REASON &&+ outcome.interrupted_row_final !== true;const SEND_FAILED_MESSAGE = "Sorry, I encountered an error. Please try again.";// A severed stream is usually the client losing the network, so recovery reads// fail too. Few enough that an offline user gets the plain failure back in@@ -1492,6 +1498,9 @@ type CanonicalRecovery = {retryApply?: () => Promise<boolean>;rollback?: TurnSnapshot;restoreRollback?: boolean;+ // An interrupted turn commits canonical only if its process died between commit and reply, so a+ // canonical change is not credited to Copilot; the editor loads it either way.+ endedInterrupted?: boolean;waitingForUnlock: boolean;yaml: { draft: string; entrySnapshot: string } | null;};@@ -3999,7 +4008,8 @@ export function WorkflowCopilotChat({(recoveredNarrative?.draft || ownsChatProposal),);const recoveredStatus: RecordingRefinementStatus = interrupted- ? holdingReservation+ ? holdingReservation ||+ !isReplaceableInterruptedRow(row.turn_outcome)? "failed": "working": isCancelledRefinementTurn(@@ -4046,10 +4056,8 @@ export function WorkflowCopilotChat({autoAcceptWritesBeforeRead,);}- // An interrupted row can be replaced by the still-running finalizer.- if (- row.turn_outcome?.terminal_reason !== INTERRUPTED_TERMINAL_REASON- ) {+ // A replaceable interrupted row can be superseded by the still-running finalizer.+ if (!isReplaceableInterruptedRow(row.turn_outcome)) {if (!schedulesRefreshed) {schedulesRefreshed = true;void queryClient.invalidateQueries({@@ -4068,6 +4076,8 @@ export function WorkflowCopilotChat({canonicalRecovery.awaitingTurnId === rowTurnId) {canonicalRecovery.terminalConfirmed = true;+ canonicalRecovery.endedInterrupted =+ reason === INTERRUPTED_TERMINAL_REASON;canonicalRecovery.restoreRollback ||=isCancelledRefinementTurn(reason, narrative) ||narrative?.terminal === "error" ||@@ -4076,6 +4086,7 @@ export function WorkflowCopilotChat({"cancelled","error","copilot_recoverable_failure",+ INTERRUPTED_TERMINAL_REASON,].includes(reason));canonicalReadAttempted = true;await reconcileCanonicalWorkflowRef.current?.(@@ -4385,7 +4396,7 @@ export function WorkflowCopilotChat({data.chat_history.flatMap((message) =>message.sender === "ai" &&message.turn_outcome?.copilot_turn_id &&- message.turn_outcome.terminal_reason !== INTERRUPTED_TERMINAL_REASON+ !isReplaceableInterruptedRow(message.turn_outcome)? [message.turn_outcome.copilot_turn_id]: [],),@@ -5190,6 +5201,7 @@ export function WorkflowCopilotChat({if (pending) {if (!canonicalUnchanged &&+ !pending.endedInterrupted &&pending.rollback?.snapshot &&(pending.rollback.titlePersisted ||pending.rollback.workflowPersisted)diff --git a/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.ts b/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.tsindex 65fe39d2e..955c5117b 100644--- a/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.ts+++ b/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.ts@@ -223,6 +223,9 @@ export interface WorkflowCopilotChatHistoryMessage {// turn_start frame carries, so a client can correlate a row to its own send.copilot_turn_id?: string | null;terminal_reason?: string | null;+ // Set on every interrupted row the server serves, since no late reply replaces+ // a stored row. Absent only from a backend that still replaced rows.+ interrupted_row_final?: boolean;})| null;narrative_payload?: Record<string, unknown> | null;diff --git a/skyvern/forge/sdk/copilot/credential_pause.py b/skyvern/forge/sdk/copilot/credential_pause.pyindex 1b42b4a2a..69ec5a54d 100644--- a/skyvern/forge/sdk/copilot/credential_pause.py+++ b/skyvern/forge/sdk/copilot/credential_pause.py@@ -700,6 +700,14 @@ async def _run_credential_pause(# Can't act on a rescued resolution mid-unwind, only avoid corrupting state.await _invalidate_active_pause_record()raise+ try:+ await app.DATABASE.workflow_params.record_pending_copilot_turn_credential_resume(+ organization_id=organization_id,+ workflow_copilot_chat_id=chat_id,+ turn_id=turn_id,+ )+ except Exception:+ LOG.warning("Could not record the credential pause resume on the turn marker", exc_info=True)if resolution is None:resolution = await _invalidate_active_pause_record()diff --git a/skyvern/forge/sdk/db/agent_db.py b/skyvern/forge/sdk/db/agent_db.pyindex cf2ed680c..0536e9715 100644--- a/skyvern/forge/sdk/db/agent_db.py+++ b/skyvern/forge/sdk/db/agent_db.py@@ -406,7 +406,9 @@ class AgentDB(BaseAlchemyDB):db_engine=self.engine,sqlite_workflow_creation_lock=self._sqlite_workflow_creation_lock,)- self.workflow_params = WorkflowParametersRepository(self.Session, debug_enabled, self.is_retryable_error)+ self.workflow_params = WorkflowParametersRepository(+ self.Session, debug_enabled, self.is_retryable_error, db_engine=self.engine+ )self.workflow_run_attempts = WorkflowRunAttemptsRepository(self.Session, debug_enabled, self.is_retryable_error)self.workflow_run_credential_selections = WorkflowRunCredentialSelectionsRepository(self.Session, debug_enabled, self.is_retryable_errordiff --git a/skyvern/forge/sdk/db/repositories/workflow_parameters.py b/skyvern/forge/sdk/db/repositories/workflow_parameters.pyindex 9101165bc..2714376ca 100644--- a/skyvern/forge/sdk/db/repositories/workflow_parameters.py+++ b/skyvern/forge/sdk/db/repositories/workflow_parameters.py@@ -1,15 +1,17 @@from __future__ import annotationsimport json-from collections.abc import Mapping+from collections.abc import AsyncIterator, Callable, Mapping+from contextlib import asynccontextmanagerfrom datetime import datetime, timedelta, timezone-from typing import Any+from typing import Any, Literalimport structlog-from sqlalchemy import cast, func, or_, select+from sqlalchemy import cast, func, or_, select, textfrom sqlalchemy.dialects.postgresql import JSONB, JSONPATHfrom sqlalchemy.dialects.postgresql import insert as pg_insert-from sqlalchemy.ext.asyncio import AsyncSession+from sqlalchemy.exc import SQLAlchemyError+from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSessionfrom sqlalchemy.orm import deferfrom skyvern.config import settings@@ -24,7 +26,7 @@ from skyvern.forge.sdk.copilot.completion_criteria_store import criteria_from_jsfrom skyvern.forge.sdk.copilot.context import ProposalDisposition, TurnNarrativePayloadfrom skyvern.forge.sdk.db._error_handling import db_operationfrom skyvern.forge.sdk.db._sentinels import _UNSET-from skyvern.forge.sdk.db.base_alchemy_db import read_with_disconnect_recovery+from skyvern.forge.sdk.db.base_alchemy_db import _SessionFactory, read_with_disconnect_recoveryfrom skyvern.forge.sdk.db.base_repository import BaseRepositoryfrom skyvern.forge.sdk.db.datetime_utils import naive_utc_now, to_naive_utcfrom skyvern.forge.sdk.db.exceptions import (@@ -101,6 +103,19 @@ from skyvern.webeye.actions.actions import ActionLOG = structlog.get_logger()PENDING_TURN_RETENTION = timedelta(days=30)+# ponytail: SQLite has no advisory locks and runs in one process, so an in-process set stands in.+_SQLITE_FINALISING_TURNS: set[str] = set()+++def _turn_finalisation_lock_key(workflow_copilot_chat_id: str, turn_id: str) -> str:+ return f"copilot_turn_finalisation:{workflow_copilot_chat_id}:{turn_id}"+++async def _turn_finalisation_held(session: AsyncSession, key: str) -> bool:+ if session.bind is not None and session.bind.dialect.name == "sqlite":+ return key in _SQLITE_FINALISING_TURNS+ acquired = await session.scalar(text("SELECT pg_try_advisory_xact_lock(hashtext(:key))"), {"key": key})+ return not acquireddef _pending_turn_id_for_idempotency_digest(@@ -274,6 +289,16 @@ def _dump_attached_files(attached_files: list[CopilotAttachedFile] | None) -> liclass WorkflowParametersRepository(BaseRepository):"""Database operations for workflow parameters, copilot chat, task generation, actions, and runs."""+ def __init__(+ self,+ session_factory: _SessionFactory,+ debug_enabled: bool = False,+ is_retryable_error_fn: Callable[[SQLAlchemyError], bool] | None = None,+ db_engine: AsyncEngine | None = None,+ ) -> None:+ super().__init__(session_factory, debug_enabled, is_retryable_error_fn)+ self._db_engine = db_engine+@db_operation("create_workflow_parameter")async def create_workflow_parameter(self,@@ -1390,11 +1415,86 @@ class WorkflowParametersRepository(BaseRepository):recovering_at = _parse_pending_turn_timestamp(entry.get("recovering_at"))if recovering_at is not None and recovering_at > claim_before:return False+ if await _turn_finalisation_held(session, _turn_finalisation_lock_key(workflow_copilot_chat_id, turn_id)):+ return False+ credential_resumed_at = _parse_pending_turn_timestamp(entry.get("credential_resumed_at"))+ if credential_resumed_at is not None and credential_resumed_at > claim_before:+ return Falsepending[turn_id] = {**entry, "recovering_at": datetime.now(timezone.utc).isoformat()}chat.pending_turns = pendingawait session.commit()return True+ @db_operation("record_pending_copilot_turn_credential_resume")+ async def record_pending_copilot_turn_credential_resume(+ self,+ organization_id: str,+ workflow_copilot_chat_id: str,+ turn_id: str,+ ) -> None:+ async with self.Session() as session:+ chat = (+ await session.scalars(+ select(WorkflowCopilotChatModel)+ .where(WorkflowCopilotChatModel.organization_id == organization_id)+ .where(WorkflowCopilotChatModel.workflow_copilot_chat_id == workflow_copilot_chat_id)+ .with_for_update()+ )+ ).first()+ if chat is None:+ return+ pending = dict(chat.pending_turns or {})+ entry = pending.get(turn_id)+ if not isinstance(entry, dict):+ return+ pending[turn_id] = {**entry, "credential_resumed_at": datetime.now(timezone.utc).isoformat()}+ chat.pending_turns = pending+ await session.commit()++ @db_operation("claim_pending_copilot_turn_for_finalisation")+ async def claim_pending_copilot_turn_for_finalisation(+ self,+ organization_id: str,+ workflow_copilot_chat_id: str,+ turn_id: str,+ ) -> Literal["claimed", "reconciling", "no_marker"]:+ """ "claimed" means the live handler owns the turn and "reconciling" that reconcile does. Call it inside+ ``hold_copilot_turn_finalisation``: reconcile cannot claim a turn while that lock is held."""+ async with self.Session() as session:+ chat = (+ await session.scalars(+ select(WorkflowCopilotChatModel)+ .where(WorkflowCopilotChatModel.organization_id == organization_id)+ .where(WorkflowCopilotChatModel.workflow_copilot_chat_id == workflow_copilot_chat_id)+ .with_for_update()+ )+ ).first()+ if chat is None:+ return "no_marker"+ entry = (chat.pending_turns or {}).get(turn_id)+ if not isinstance(entry, dict):+ return "no_marker"+ return "reconciling" if entry.get("recovering_at") is not None else "claimed"++ @asynccontextmanager+ async def hold_copilot_turn_finalisation(self, workflow_copilot_chat_id: str, turn_id: str) -> AsyncIterator[None]:+ """Keep reconcile off this turn while the live handler writes its outcome. The lock lives on this+ transaction's connection, so a dead process releases it and a stalled live one keeps it."""+ if self._db_engine is None:+ raise RuntimeError("Copilot turn finalisation locking requires a database engine")+ key = _turn_finalisation_lock_key(workflow_copilot_chat_id, turn_id)+ if self._db_engine.dialect.name == "sqlite":+ _SQLITE_FINALISING_TURNS.add(key)+ try:+ yield+ finally:+ _SQLITE_FINALISING_TURNS.discard(key)+ return+ # A raw connection, never the ambient session: the handler's own writes commit on their own.+ async with self._db_engine.connect() as connection, connection.begin():+ await connection.execute(text("SELECT pg_advisory_xact_lock(hashtext(:key))"), {"key": key})+ yield+@db_operation("clear_pending_copilot_turn")async def clear_pending_copilot_turn(self,diff --git a/skyvern/forge/sdk/routes/workflow_copilot.py b/skyvern/forge/sdk/routes/workflow_copilot.pyindex 5c4691700..ae262352c 100644--- a/skyvern/forge/sdk/routes/workflow_copilot.py+++ b/skyvern/forge/sdk/routes/workflow_copilot.py@@ -7,10 +7,10 @@ import jsonimport reimport timeimport uuid-from collections.abc import Iterator+from collections.abc import Awaitable, Coroutine, Iteratorfrom contextlib import contextmanagerfrom datetime import UTC, datetime, timedelta-from typing import Any, Literal, cast, get_args+from typing import Any, Literal, TypeVar, cast, get_argsfrom urllib.parse import urlparseimport structlog@@ -190,6 +190,7 @@ ALLOWED_WORKFLOW_COPILOT_AUDIO_CONTENT_TYPES = {}LOG = structlog.get_logger()+_T = TypeVar("_T")async def _resolve_copilot_agent_handler(@@ -1175,6 +1176,23 @@ def _is_interrupted_recovery_row(message: WorkflowCopilotChatMessage) -> bool:return message.turn_outcome is not None and message.turn_outcome.terminal_reason == INTERRUPTED_TERMINAL_REASON+async def _claim_turn_finalisation(chat: WorkflowCopilotChat, turn_id: str) -> bool:+ """Whether the live turn still owns its post-agent writes; False once reconcile has claimed it."""+ claim = await app.DATABASE.workflow_params.claim_pending_copilot_turn_for_finalisation(+ organization_id=chat.organization_id,+ workflow_copilot_chat_id=chat.workflow_copilot_chat_id,+ turn_id=turn_id,+ )+ if claim == "reconciling":+ return False+ if claim == "claimed":+ return True+ # Reconcile drops the marker only after writing its interrupted row, so a missing marker+ # beside that row means reconcile already finished this turn.+ existing = await _assistant_row_for_turn(chat, turn_id)+ return existing is None or not _is_interrupted_recovery_row(existing)++async def _clear_pending_turn(chat: WorkflowCopilotChat, turn_id: str) -> None:# A failed clear is self-healing: the reconcile pass sees the turn already# answered, skips recovery and retries the clear.@@ -1251,46 +1269,7 @@ async def _persist_turn_messages(assistant_message: WorkflowCopilotChatMessage | None = Noneexisting = await _assistant_row_for_turn(chat, turn_id) if turn_id is not None else None- superseding_recovery = (- existing is not None- and _is_interrupted_recovery_row(existing)- and turn_outcome is not None- and turn_outcome.terminal_reason != INTERRUPTED_TERMINAL_REASON- )- if existing is not None and superseding_recovery and turn_outcome is not None:- replacement_updates: dict[str, str] = {}- if (- turn_outcome.user_message_id is None- and existing.turn_outcome is not None- and existing.turn_outcome.user_message_id is not None- ):- replacement_updates["user_message_id"] = existing.turn_outcome.user_message_id- if (- turn_outcome.request_cancel_token is None- and existing.turn_outcome is not None- and existing.turn_outcome.request_cancel_token is not None- ):- replacement_updates["request_cancel_token"] = existing.turn_outcome.request_cancel_token- if replacement_updates:- turn_outcome = turn_outcome.model_copy(update=replacement_updates)- # Recovery reached this turn first and wrote an interrupted row. The turn then finished,- # so its real reply is the truth and replaces that row rather than being dropped.- LOG.info(- "Copilot turn finished after being recovered; replacing the interrupted row with its reply",- workflow_copilot_chat_id=chat.workflow_copilot_chat_id,- turn_id=turn_id,- )- assistant_message = await asyncio.shield(- app.DATABASE.workflow_params.replace_workflow_copilot_chat_message(- organization_id=chat.organization_id,- workflow_copilot_chat_message_id=existing.workflow_copilot_chat_message_id,- content=assistant_content,- global_llm_context=global_llm_context,- turn_outcome=turn_outcome,- narrative_payload=narrative_payload,- )- )- elif existing is not None:+ if existing is not None:LOG.info("Copilot turn already has an assistant row; skipping duplicate persist",workflow_copilot_chat_id=chat.workflow_copilot_chat_id,@@ -1377,6 +1356,7 @@ def _interrupted_turn_outcome(),copilot_effective_mode=effective_mode,copilot_code_available=code_available or False,+ interrupted_row_final=True,)@@ -2286,7 +2266,7 @@ async def _new_copilot_chat_post(cancel_watcher: asyncio.Task[None] | None = Nonecurrent_code_available = Falseturn_index = 0- effective_mode = _effective_copilot_build_mode(chat_request)+ effective_mode: PersistedCopilotComposerMode | None = _effective_copilot_build_mode(chat_request)prior_turn_outcome: TurnOutcome | None = Nonedef capture_code_mode_opt_out_after_persist() -> None:@@ -2301,6 +2281,48 @@ async def _new_copilot_chat_post(turn_id=turn_id,)+ # Released in the route's finally, after every post-agent write, so reconcile cannot mark the+ # row final while this handler can still write.+ finalisation_fence = contextlib.AsyncExitStack()+ ownership_claim: list[asyncio.Future[bool]] = []+ post_agent_writes: list[asyncio.Future[Any]] = []++ async def claim_turn_ownership() -> bool:+ if chat is None or not turn_started:+ return True+ await finalisation_fence.enter_async_context(+ app.DATABASE.workflow_params.hold_copilot_turn_finalisation(chat.workflow_copilot_chat_id, turn_id)+ )+ owned = await _claim_turn_finalisation(chat, turn_id)+ if not owned:+ LOG.info(+ "Copilot turn finished after reconcile claimed it; leaving its interrupted row final",+ organization_id=organization.organization_id,+ workflow_copilot_chat_id=chat.workflow_copilot_chat_id,+ turn_id=turn_id,+ )+ return owned++ def turn_still_owned() -> Awaitable[bool]:+ # One claim per handler: a second concurrent acquire of the lock would wait on this one.+ if not ownership_claim:+ ownership_claim.append(asyncio.ensure_future(claim_turn_ownership()))+ return asyncio.shield(ownership_claim[0])++ async def turn_owned_or_left_to_reconcile() -> bool:+ # For error arms with nothing else to report: an unclaimed marker stays for reconcile.+ try:+ return await turn_still_owned()+ except Exception:+ LOG.warning("Could not claim the copilot turn's finalisation", turn_id=turn_id, exc_info=True)+ return False++ def tracked_write(write: Coroutine[Any, Any, _T]) -> Awaitable[_T]:+ # Shielded writes outlive a cancelled handler; the finally waits for them before releasing the lock.+ task = asyncio.ensure_future(write)+ post_agent_writes.append(task)+ return asyncio.shield(task)+async def _emit_unpersisted_failure(failure: RecoverableFailure,*,@@ -2348,7 +2370,17 @@ async def _new_copilot_chat_post(nonlocal failure_kind_for_log, route_exit, terminal_frame_emitted, turn_outcome_for_logroute_exit = "error"failure_kind_for_log = failure_kind- if any(isinstance(item, DatabaseConnectionUnavailableError) for item in iter_exception_chain(exc)):+ database_down = any(+ isinstance(item, DatabaseConnectionUnavailableError) for item in iter_exception_chain(exc)+ )+ claim_failure: Exception | None = None+ if not database_down:+ try:+ if not await turn_still_owned():+ return+ except Exception as error:+ claim_failure = error+ if database_down or claim_failure is not None:# Rolling the workflow back and writing the reply are both database work, against a# database that just exhausted a full reconnection budget. Report what is already# known rather than spending more of the caller's wait on writes that cannot land;@@ -2372,7 +2404,7 @@ async def _new_copilot_chat_post(await _emit_unpersisted_failure(failure,failure_kind=failure_kind,- writer_exc=exc,+ writer_exc=claim_failure or exc,)returnrestored = chat is not None and _should_restore_persisted_workflow(@@ -2453,7 +2485,7 @@ async def _new_copilot_chat_post(else chat_request.model_copy(update={"message": UNSCREENED_MESSAGE_PLACEHOLDER}))try:- await asyncio.shield(+ await tracked_write(_finalise_normal_turn(stream=stream,chat=chat,@@ -2860,25 +2892,39 @@ async def _new_copilot_chat_post(# carrying ``workflow_was_persisted`` so rollback proceeds normally.# Nobody pressed Stop on a cancellation the user never asked for, so# that turn is recorded as interrupted rather than as their intent.- await _persist_cancel_turn(- stream=stream,- chat=chat,- organization_id=organization.organization_id,- original_workflow=original_workflow,- user_message=chat_request.message,- agent_result=agent_result,- audio_artifact_id=chat_request.audio_artifact_id,- turn_id=turn_id,- keep_pending_proposal=chat_request.keep_pending_proposal,- user_row_already_persisted=turn_started,- sender=_turn_opener_sender(chat_request),- record_as_interrupted=not user_cancel_observed[0],- cancel_source=user_cancel_source[0],- effective_mode=effective_mode,- code_available=current_code_available,- request_cancel_token=chat_request.cancel_token,- )- terminal_frame_emitted = True+ # Set before the shielded write for the same reason as the success path below.+ finalise_started = True+ cancelled_agent_result = agent_result++ async def persist_cancel_if_owned() -> bool:+ nonlocal terminal_frame_emitted+ if not await turn_still_owned():+ return False+ await _persist_cancel_turn(+ stream=stream,+ chat=chat,+ organization_id=organization.organization_id,+ original_workflow=original_workflow,+ user_message=chat_request.message,+ agent_result=cancelled_agent_result,+ audio_artifact_id=chat_request.audio_artifact_id,+ turn_id=turn_id,+ keep_pending_proposal=chat_request.keep_pending_proposal,+ user_row_already_persisted=turn_started,+ sender=_turn_opener_sender(chat_request),+ record_as_interrupted=not user_cancel_observed[0],+ cancel_source=user_cancel_source[0],+ effective_mode=effective_mode,+ code_available=current_code_available,+ request_cancel_token=chat_request.cancel_token,+ )+ # Set inside the shield: a cancelled handler skips the line after its await,+ # and the finally would send a second, false terminal frame.+ terminal_frame_emitted = True+ return True++ if not await tracked_write(persist_cancel_if_owned()):+ returncapture_code_mode_opt_out_after_persist()LOG.info("Workflow copilot agent turn cancelled",@@ -2892,21 +2938,30 @@ async def _new_copilot_chat_post(# Atomic finalisation — a late cancel that fires here cannot tear# the success-path writes apart mid-way (no half-written turn,- # no duplicate user/AI rows).+ # no duplicate user/AI rows). The ownership claim sits inside the+ # shield for the same reason.finalise_started = True- await asyncio.shield(- _finalise_normal_turn(+ finalised_agent_result = agent_result++ async def finalise_if_owned() -> bool:+ nonlocal terminal_frame_emitted+ if not await turn_still_owned():+ return False+ await _finalise_normal_turn(stream=stream,chat=chat,organization_id=organization.organization_id,original_workflow=original_workflow,chat_request=chat_request,- agent_result=agent_result,+ agent_result=finalised_agent_result,turn_id=turn_id,user_row_already_persisted=turn_started,)- )- terminal_frame_emitted = True+ terminal_frame_emitted = True+ return True++ if not await tracked_write(finalise_if_owned()):+ returncapture_code_mode_opt_out_after_persist()route_exit = "completed"except HTTPException as exc:@@ -2924,9 +2979,13 @@ async def _new_copilot_chat_post(workflow_permanent_id=chat_request.workflow_permanent_id,exc_info=exc.status_code >= 500,)- if chat is not None and _should_restore_persisted_workflow(- chat.auto_accept,- agent_result,+ if (+ chat is not None+ and _should_restore_persisted_workflow(+ chat.auto_accept,+ agent_result,+ )+ and await turn_owned_or_left_to_reconcile()):try:await _restore_workflow_definition(original_workflow, organization.organization_id)@@ -2958,12 +3017,14 @@ async def _new_copilot_chat_post(except asyncio.CancelledError:route_exit = "cancelled"turn_outcome_for_log = agent_result.turn_outcome if agent_result is not None else None+ if not await turn_owned_or_left_to_reconcile():+ raiseif chat is not None and _should_restore_persisted_workflow(chat.auto_accept,agent_result,):try:- await asyncio.shield(_restore_workflow_definition(original_workflow, organization.organization_id))+ await tracked_write(_restore_workflow_definition(original_workflow, organization.organization_id))except Exception:LOG.warning("Workflow restore failed inside cancel-error handler",@@ -2975,7 +3036,7 @@ async def _new_copilot_chat_post(# the agent_result.cancelled branch above couldn't run.# _persist_cancel_turn skips rollback when agent_result is None.try:- await asyncio.shield(+ await tracked_write(_persist_cancel_turn(stream=stream,chat=chat,@@ -3037,7 +3098,7 @@ async def _new_copilot_chat_post(workflow_copilot_chat_id=chat.workflow_copilot_chat_id,turn_id=turn_id,):- await asyncio.shield(+ await tracked_write(_persist_interrupted_turn(chat,turn_id,@@ -3067,6 +3128,24 @@ async def _new_copilot_chat_post()finally:try:+ # A cancelled handler reaches here while its shielded writes still run. A further cancel+ # must not cut the wait short, or the lock drops under a live write; it is re-raised below.+ cancelled_while_waiting = False+ for pending_write in (*ownership_claim, *post_agent_writes):+ while not pending_write.done():+ try:+ await asyncio.shield(pending_write)+ except asyncio.CancelledError:+ cancelled_while_waiting = cancelled_while_waiting or not pending_write.done()+ except Exception:+ break+ try:+ await finalisation_fence.aclose()+ except Exception:+ with contained_effect("copilot turn finalisation lock release failure", turn_id=turn_id):+ LOG.warning(+ "Could not release the copilot turn finalisation lock", turn_id=turn_id, exc_info=True+ )if cancel_watcher is not None and not cancel_watcher.done():cancel_watcher.cancel()with contextlib.suppress(asyncio.CancelledError, Exception):@@ -3090,6 +3169,8 @@ async def _new_copilot_chat_post(browser_session_id,reason=BrowserSessionCloseReason.user_requested,)+ if cancelled_while_waiting:+ raise asyncio.CancelledErrorfinally:with contained_effect("copilot turn outcome log"):LOG.info(@@ -4431,7 +4512,7 @@ def convert_to_history_messages(feedback=getattr(message, "feedback", None),audio_artifact_id=message.audio_artifact_id,attached_files=message.attached_files,- turn_outcome=message.turn_outcome,+ turn_outcome=_served_turn_outcome(message.turn_outcome),created_at=message.created_at,modified_at=getattr(message, "modified_at", message.created_at),narrative_payload=message.narrative_payload,@@ -4440,6 +4521,14 @@ def convert_to_history_messages(]+def _served_turn_outcome(outcome: TurnOutcome | None) -> TurnOutcome | None:+ # Nothing in this backend replaces a stored assistant row, so an interrupted row written before+ # the flag existed is final too; serving it unflagged would hold the chat's save lock forever.+ if outcome is None or outcome.terminal_reason != INTERRUPTED_TERMINAL_REASON or outcome.interrupted_row_final:+ return outcome+ return outcome.model_copy(update={"interrupted_row_final": True})++@base_router.post("/workflow/copilot/suggest-goal", include_in_schema=False)async def workflow_copilot_suggest_goal(suggestion_request: WorkflowCopilotGoalSuggestionRequest,diff --git a/skyvern/forge/sdk/schemas/copilot_turn_outcome.py b/skyvern/forge/sdk/schemas/copilot_turn_outcome.pyindex 8003b5dbe..cf3b9cc3e 100644--- a/skyvern/forge/sdk/schemas/copilot_turn_outcome.py+++ b/skyvern/forge/sdk/schemas/copilot_turn_outcome.py@@ -117,6 +117,9 @@ class TurnOutcome(BaseModel):# Client-generated per-request correlation value. Retained after the pending# marker is cleared so a disconnected client can recover the exact chat.request_cancel_token: str | None = None+ # True on an interrupted row no late finalizer can replace. Chat history serves every stored+ # interrupted row this way; False only reaches a client from a backend that still replaced rows.+ interrupted_row_final: bool = Falseidempotency_digest: str | None = Noneunresolved_runtime_failure: UnresolvedRuntimeFailure | None = Noneconnected_account_choices: list[ConnectedAccountChoice] | None = Nonediff --git a/skyvern/forge/sdk/schemas/workflow_copilot.py b/skyvern/forge/sdk/schemas/workflow_copilot.pyindex 6f5e9d6af..62c7dbb69 100644--- a/skyvern/forge/sdk/schemas/workflow_copilot.py+++ b/skyvern/forge/sdk/schemas/workflow_copilot.py@@ -149,6 +149,9 @@ class CopilotPendingTurn(BaseModel):copilot_code_available: bool = Falseuser_message_id: str | None = Nonerecovering_at: datetime | None = None+ # A credential pause stops the turn's budget clock, so reconcile measures abandonment from here+ # too, as it does from a resolved question.+ credential_resumed_at: datetime | None = None# Fingerprint of the canonical workflow as this turn last left it. None means the turn# never wrote canonical, so it owns no write to roll back.canonical_write_fingerprint: str | None = Nonediff --git a/tests/unit/copilot_route_test_support.py b/tests/unit/copilot_route_test_support.pyindex 016f98869..965ba226f 100644--- a/tests/unit/copilot_route_test_support.py+++ b/tests/unit/copilot_route_test_support.py@@ -6,6 +6,8 @@ DB / LLM / agent mocks; these helpers keep that wiring in one place.from __future__ import annotations+from collections.abc import AsyncIterator+from contextlib import asynccontextmanagerfrom datetime import datetime, timezonefrom types import SimpleNamespacefrom typing import Any@@ -51,6 +53,11 @@ def narrative_payload_with_run(run_id: str | None) -> dict[str, Any]:}+@asynccontextmanager+async def no_finalisation_fence(*_: object) -> AsyncIterator[None]:+ yield++def install_fake_create(monkeypatch: pytest.MonkeyPatch) -> dict[str, object]:"""Capture the stream handler that the route hands to EventSourceStream."""captured: dict[str, object] = {}@@ -148,6 +155,8 @@ def setup_new_copilot_mocks(),replace_workflow_copilot_chat_message=AsyncMock(),claim_pending_copilot_turn=AsyncMock(return_value=True),+ claim_pending_copilot_turn_for_finalisation=AsyncMock(return_value="claimed"),+ hold_copilot_turn_finalisation=no_finalisation_fence,clear_pending_copilot_turn=AsyncMock(),)app.DATABASE.workflow_params = workflow_paramsdiff --git a/tests/unit/test_copilot_credential_pause.py b/tests/unit/test_copilot_credential_pause.pyindex 01d760a1e..2511621e7 100644--- a/tests/unit/test_copilot_credential_pause.py+++ b/tests/unit/test_copilot_credential_pause.py@@ -394,10 +394,14 @@ async def test_connected_action_mutates_policy_and_resolves(monkeypatch: pytest.)monkeypatch.setattr(credential_pause_module.app._inst, "CACHE", cache, raising=False)credential = _make_credential()+ record_resume = AsyncMock()monkeypatch.setattr(credential_pause_module.app,"DATABASE",- SimpleNamespace(credentials=SimpleNamespace(get_credentials_by_ids=AsyncMock(return_value=[credential]))),+ SimpleNamespace(+ credentials=SimpleNamespace(get_credentials_by_ids=AsyncMock(return_value=[credential])),+ workflow_params=SimpleNamespace(record_pending_copilot_turn_credential_resume=record_resume),+ ),)monkeypatch.setattr(credential_pause_module, "CREDENTIAL_RESPONSE_POLL_SECONDS", 0.01)@@ -407,6 +411,8 @@ async def test_connected_action_mutates_policy_and_resolves(monkeypatch: pytest.resume_msgs = await maybe_credential_pause(ctx, _fake_result(), stream, config)assert resume_msgs is not None+ # Reconcile measures abandonment from this stamp, so a resumed turn is not recovered as dead.+ record_resume.assert_awaited_with(organization_id="org-1", workflow_copilot_chat_id="chat-1", turn_id="turn-1")assert ctx.credential_pause_outcome == "connected"assert ctx.request_policy.resolved_credentials == [credential]assert ctx.request_policy.allow_run_blocks is Truediff --git a/tests/unit/test_workflow_copilot_cancel.py b/tests/unit/test_workflow_copilot_cancel.pyindex 8c7c9e8db..7bfaa46aa 100644--- a/tests/unit/test_workflow_copilot_cancel.py+++ b/tests/unit/test_workflow_copilot_cancel.py@@ -21,6 +21,8 @@ Covers:from __future__ import annotationsimport asyncio+from collections.abc import AsyncIterator+from contextlib import asynccontextmanagerfrom datetime import datetime, timedelta, timezonefrom types import SimpleNamespacefrom typing import Any@@ -62,10 +64,11 @@ from skyvern.forge.sdk.schemas.workflow_copilot import (WorkflowCopilotChat,WorkflowCopilotChatRequest,WorkflowCopilotChatSender,+ WorkflowCopilotStreamErrorUpdate,WorkflowCopilotStreamMessageType,)from tests.unit._copilot_workflow_fakes import fake_workflow-from tests.unit.copilot_route_test_support import install_fake_create, setup_new_copilot_mocks+from tests.unit.copilot_route_test_support import install_fake_create, no_finalisation_fence, setup_new_copilot_mocksclass _FakeCache:@@ -1274,10 +1277,13 @@ async def test_watcher_reports_the_gesture_the_cancel_named() -> None:@pytest.mark.asyncio[email protected]("claim", ["claimed", "reconciling"])async def test_operational_cancel_records_the_turn_as_interrupted_and_still_re_raises(- monkeypatch: pytest.MonkeyPatch,+ monkeypatch: pytest.MonkeyPatch, claim: str) -> None:- """task.cancel() without user_cancel_observed[0] -> an interrupted row, never a user-cancel row."""+ """task.cancel() without user_cancel_observed[0] -> an interrupted row, never a user-cancel row.++ Once reconcile owns the turn, its row is the only one; the drain must not write another."""captured = install_fake_create(monkeypatch)chat = SimpleNamespace(@@ -1325,6 +1331,8 @@ async def test_operational_cancel_records_the_turn_as_interrupted_and_still_re_rstart_copilot_turn=AsyncMock(return_value=SimpleNamespace(created_at=datetime(2026, 4, 27, tzinfo=timezone.utc))),+ claim_pending_copilot_turn_for_finalisation=AsyncMock(return_value=claim),+ hold_copilot_turn_finalisation=no_finalisation_fence,clear_pending_copilot_turn=AsyncMock(),)app.DATABASE.workflow_params = workflow_params@@ -1360,10 +1368,14 @@ async def test_operational_cancel_records_the_turn_as_interrupted_and_still_re_rcontents = [c.kwargs.get("content") for c in insert_calls]assert MINIMAL_CANCEL_STOP not in contentsinterrupted = [content for content in contents if content and "interrupted" in content.lower()]+ if claim == "reconciling":+ assert interrupted == []+ returnassert len(interrupted) == 1assert "wpid-1" in interrupted[0]written = insert_calls[-1].kwargsassert written["turn_outcome"].terminal_reason == INTERRUPTED_TERMINAL_REASON+ assert written["turn_outcome"].interrupted_row_final is Trueassert written["narrative_payload"]["terminalMessage"] == written["content"]@@ -1483,6 +1495,88 @@ async def test_cancel_during_shielded_finalisation_does_not_write_a_second_row(assert interrupted == [], f"finalisation owns this turn's row; got a competing write: {interrupted}"[email protected]("stall", ["claim", "finalise", "error recovery", "cancelled result"])+async def test_a_drained_handler_keeps_the_finalisation_lock_until_its_shielded_writes_finish(+ monkeypatch: pytest.MonkeyPatch, stall: str+) -> None:+ """Releasing on the cancel would let reconcile mark the row final while the finalise still writes,+ and a cancel during the ownership claim must not turn a finished turn into an interrupted one."""+ captured = install_fake_create(monkeypatch)+ chat = _make_chat(auto_accept=False)+ agent_result = AgentResult(+ user_response="Here is your workflow.",+ updated_workflow=None,+ global_llm_context=None,+ workflow_yaml=None,+ cancelled=stall == "cancelled result",+ turn_outcome=None,+ )+ _, workflow_params = setup_new_copilot_mocks(monkeypatch, chat, _make_original_workflow(), agent_result)+ events: list[str] = []+ stalled = asyncio.Event()+ may_continue = asyncio.Event()++ @asynccontextmanager+ async def recording_fence(*_: object) -> AsyncIterator[None]:+ if stall == "claim":+ stalled.set()+ await may_continue.wait()+ events.append("locked")+ try:+ yield+ finally:+ events.append("released")++ async def slow_finalise(*args: object, **kwargs: object) -> None:+ del args, kwargs+ if stall in ("finalise", "error recovery", "cancelled result"):+ stalled.set()+ await may_continue.wait()+ events.append("finalise done")++ workflow_params.hold_copilot_turn_finalisation = recording_fence+ monkeypatch.setattr("skyvern.forge.sdk.routes.workflow_copilot._finalise_normal_turn", slow_finalise)+ monkeypatch.setattr("skyvern.forge.sdk.routes.workflow_copilot._persist_cancel_turn", slow_finalise)+ if stall == "error recovery":+ monkeypatch.setattr(+ "skyvern.forge.sdk.routes.workflow_copilot.run_copilot_agent",+ AsyncMock(side_effect=RuntimeError("agent failed")),+ )+ monkeypatch.setattr(app._inst, "CACHE", None, raising=False)+ request = MagicMock()+ request.headers = {"x-api-key": "sk-test"}+ await workflow_copilot_chat_post(+ request, _make_chat_request(cancel_token=None), SimpleNamespace(organization_id="org-1")+ )+ stream = MagicMock()+ stream.send = AsyncMock(return_value=True)+ stream.is_disconnected = AsyncMock(return_value=False)++ handler_task = asyncio.create_task(captured["handler"](stream))+ await stalled.wait()+ # A client disconnect then a drain: the second cancel lands while the release waits.+ for _ in range(2):+ handler_task.cancel()+ for _ in range(5):+ await asyncio.sleep(0)+ assert "released" not in events+ may_continue.set()+ with pytest.raises(asyncio.CancelledError):+ await handler_task++ assert events == ["locked", "finalise done", "released"]+ contents = [c.kwargs.get("content") for c in workflow_params.create_workflow_copilot_chat_message.await_args_list]+ assert not [content for content in contents if content and "interrupted" in content.lower()]+ if stall != "error recovery":+ sent_errors = [+ call.args[0].error+ for call in stream.send.await_args_list+ if isinstance(call.args[0], WorkflowCopilotStreamErrorUpdate)+ ]+ assert "The assistant didn't finish this turn. Please try again." not in sent_errors++@pytest.mark.asyncioasync def test_a_stop_before_the_agent_starts_still_reports_a_cause_when_its_row_cannot_be_written(monkeypatch: pytest.MonkeyPatch,diff --git a/tests/unit/test_workflow_copilot_route.py b/tests/unit/test_workflow_copilot_route.pyindex 876752804..f0b247933 100644--- a/tests/unit/test_workflow_copilot_route.py+++ b/tests/unit/test_workflow_copilot_route.py@@ -11,6 +11,7 @@ import asyncioimport contextlibimport jsonimport re+from collections.abc import AsyncIteratorfrom datetime import UTC, datetime, timedeltafrom pathlib import Pathfrom types import SimpleNamespace@@ -18,9 +19,11 @@ from typing import Anyfrom unittest.mock import AsyncMock, MagicMockimport pytest+import pytest_asyncioimport yamlfrom fastapi import FastAPI, HTTPExceptionfrom httpx import ASGITransport, AsyncClient+from sqlalchemy.engine import make_urlfrom sqlalchemy.ext.asyncio import AsyncEnginefrom structlog.testing import capture_logs@@ -63,6 +66,7 @@ from skyvern.forge.sdk.copilot.interruption import (from skyvern.forge.sdk.copilot.tools import workflow_updatefrom skyvern.forge.sdk.copilot.turn_outcome import build_minimal_turn_outcomefrom skyvern.forge.sdk.copilot.workflow_yaml import _process_workflow_yaml, dump_workflow_yaml+from skyvern.forge.sdk.db.agent_db import _build_enginefrom skyvern.forge.sdk.db.base_alchemy_db import BaseAlchemyDBfrom skyvern.forge.sdk.db.exceptions import CopilotProposalConflictError, DatabaseConnectionUnavailableErrorfrom skyvern.forge.sdk.db.repositories.workflow_parameters import (@@ -6002,10 +6006,10 @@ async def test_a_turn_that_never_started_keeps_its_attachments_on_the_recovery_r@pytest.mark.asyncio-async def test_a_finished_turn_replaces_the_interrupted_row_rather_than_dropping_its_reply(+async def test_an_interrupted_row_stored_before_the_final_flag_is_served_final(monkeypatch: pytest.MonkeyPatch,) -> None:- """Recovery can reach a slow-but-live turn first; when that turn finishes, its reply is the truth."""+ """Nothing replaces stored rows anymore, so an unflagged one must not hold the chat's save lock forever."""chat = _make_persisted_chat([])store, _ = _install_reconcile_store(monkeypatch, chat)store.add_message(WorkflowCopilotChatSender.USER, "build me a scraper")@@ -6014,34 +6018,111 @@ async def test_a_finished_turn_replaces_the_interrupted_row_rather_than_droppingINTERRUPTED_TERMINAL_MESSAGE,TurnOutcome(response_kind=ResponseKind.RECOVER,- reason_code=INTERRUPTED_TERMINAL_REASON,terminal_reason=INTERRUPTED_TERMINAL_REASON,copilot_turn_id="turn-a",- user_message_id="wccm-0",- request_cancel_token="cancel-turn-a",),)- await _persist_turn_messages(- chat=chat,- turn_id="turn-a",+ response = await _load_history()++ outcome = response.chat_history[-1].turn_outcome+ assert outcome is not None+ assert outcome.interrupted_row_final is True+++async def _start_abandoned_turn(repo: WorkflowParametersRepository, turn_id: str) -> str:+ chat = await repo.create_workflow_copilot_chat(organization_id="org", workflow_permanent_id="wpid")+ await repo.start_copilot_turn(+ organization_id="org",+ workflow_copilot_chat_id=chat.workflow_copilot_chat_id,+ pending_turn=CopilotPendingTurn(+ turn_id=turn_id,+ started_at=datetime.now(UTC) - timedelta(seconds=RECONCILE_ABANDON_AFTER_SECONDS + 60),+ ),user_message="build me a scraper",- audio_artifact_id=None,- user_row_already_persisted=True,- sender=WorkflowCopilotChatSender.USER,- assistant_content="Here is your workflow.",- global_llm_context=None,- turn_outcome=TurnOutcome(response_kind=ResponseKind.BUILD),- narrative_payload=None,)+ return chat.workflow_copilot_chat_id- assert len(store.assistant_messages) == 1- assert store.assistant_messages[0].content == "Here is your workflow."- outcome = store.assistant_messages[0].turn_outcome- assert outcome is not None- assert outcome.terminal_reason != INTERRUPTED_TERMINAL_REASON- assert outcome.user_message_id == "wccm-0"- assert outcome.request_cancel_token == "cancel-turn-a"++async def test_a_turn_finishing_after_reconcile_claimed_it_cannot_write(+ sqlite_engine: AsyncEngine, monkeypatch: pytest.MonkeyPatch+) -> None:+ """Reconcile's interrupted row is final: the still-live turn loses its claim mid-recovery and after."""+ repo = WorkflowParametersRepository(BaseAlchemyDB(sqlite_engine).Session)+ monkeypatch.setattr(app.DATABASE, "workflow_params", repo)+ mid_chat_id = await _start_abandoned_turn(repo, "turn-mid")+ assert await repo.claim_pending_copilot_turn("org", mid_chat_id, "turn-mid", datetime.now(UTC))+ mid_chat = await repo.get_workflow_copilot_chat_by_id("org", mid_chat_id)+ assert not await workflow_copilot_route._claim_turn_finalisation(mid_chat, "turn-mid")++ chat_id = await _start_abandoned_turn(repo, "turn-done")+ chat = await repo.get_workflow_copilot_chat_by_id("org", chat_id)+ await workflow_copilot_route._reconcile_interrupted_copilot_turns(chat, "org")++ [row] = [m for m in await repo.get_workflow_copilot_chat_messages(chat_id) if m.sender == "ai"]+ assert row.turn_outcome is not None+ assert row.turn_outcome.terminal_reason == INTERRUPTED_TERMINAL_REASON+ assert row.turn_outcome.interrupted_row_final is True+ assert not await workflow_copilot_route._claim_turn_finalisation(chat, "turn-done")+++async def test_reconcile_leaves_a_turn_that_just_resumed_from_a_credential_pause(+ sqlite_engine: AsyncEngine, monkeypatch: pytest.MonkeyPatch+) -> None:+ """Two credential pauses outlast the abandon window from the turn's start while it is still working."""+ repo = WorkflowParametersRepository(BaseAlchemyDB(sqlite_engine).Session)+ monkeypatch.setattr(app.DATABASE, "workflow_params", repo)+ chat_id = await _start_abandoned_turn(repo, "turn-a")+ await repo.record_pending_copilot_turn_credential_resume("org", chat_id, "turn-a")+ abandoned_before = datetime.now(UTC) - timedelta(seconds=RECONCILE_ABANDON_AFTER_SECONDS)+ assert not await repo.claim_pending_copilot_turn("org", chat_id, "turn-a", abandoned_before)++ await workflow_copilot_route._reconcile_interrupted_copilot_turns(+ await repo.get_workflow_copilot_chat_by_id("org", chat_id), "org"+ )++ assert [m for m in await repo.get_workflow_copilot_chat_messages(chat_id) if m.sender == "ai"] == []+ chat = await repo.get_workflow_copilot_chat_by_id("org", chat_id)+ assert await workflow_copilot_route._claim_turn_finalisation(chat, "turn-a")+++@pytest_asyncio.fixture(params=["sqlite", "postgres"])+async def fence_repo(+ request: pytest.FixtureRequest, sqlite_engine: AsyncEngine+) -> AsyncIterator[WorkflowParametersRepository]:+ if request.param == "sqlite":+ yield WorkflowParametersRepository(BaseAlchemyDB(sqlite_engine).Session, db_engine=sqlite_engine)+ return+ if make_url(str(settings.DATABASE_STRING)).get_backend_name() != "postgresql":+ pytest.skip("requires PostgreSQL")+ engine = _build_engine(settings.DATABASE_STRING)+ try:+ yield WorkflowParametersRepository(BaseAlchemyDB(engine).Session, db_engine=engine)+ finally:+ await engine.dispose()+++async def test_reconcile_waits_for_a_finalising_handler_and_answers_once_its_lock_is_gone(+ fence_repo: WorkflowParametersRepository, monkeypatch: pytest.MonkeyPatch+) -> None:+ """A stalled handler keeps its lock and so its turn; a dead one's connection takes the lock with it."""+ monkeypatch.setattr(app.DATABASE, "workflow_params", fence_repo)+ chat_id = await _start_abandoned_turn(fence_repo, "turn-a")++ async def reconcile() -> list[WorkflowCopilotChatMessage]:+ chat = await fence_repo.get_workflow_copilot_chat_by_id("org", chat_id)+ await workflow_copilot_route._reconcile_interrupted_copilot_turns(chat, "org")+ return [m for m in await fence_repo.get_workflow_copilot_chat_messages(chat_id) if m.sender == "ai"]++ async with fence_repo.hold_copilot_turn_finalisation(chat_id, "turn-a"):+ assert await reconcile() == []++ [row] = await reconcile()+ assert row.turn_outcome is not None+ assert row.turn_outcome.interrupted_row_final is Truedef test_reconcile_threshold_outlasts_the_turn_enforcement_ceiling() -> None:@@ -6098,6 +6179,58 @@ async def test_marker_survives_a_finalizer_that_raises_with_no_assistant_row(app.DATABASE.workflow_params.clear_pending_copilot_turn.assert_not_awaited()[email protected]("ending", ["finished", "cancelled", "errored"])+async def test_a_turn_reconcile_claimed_leaves_canonical_and_its_row_alone(+ monkeypatch: pytest.MonkeyPatch, api_key_request: MagicMock, copilot_stream: MagicMock, ending: str+) -> None:+ """Rolling canonical back now could overwrite a save the user made once the interrupted row unlocked it."""+ captured = install_fake_create(monkeypatch)+ chat = SimpleNamespace(+ workflow_copilot_chat_id="chat-1",+ workflow_permanent_id="wpid-1",+ organization_id="org-1",+ proposed_workflow=None,+ auto_accept=False,+ )+ original_workflow = SimpleNamespace(+ workflow_id="wf-canonical",+ title="Original",+ description="Original description",+ workflow_definition=None,+ )+ agent_result = AgentResult(+ user_response="Here is your workflow.",+ updated_workflow=None,+ global_llm_context=None,+ workflow_yaml=None,+ workflow_was_persisted=True,+ clear_proposed_workflow=False,+ authoring_barred=False,+ resolved_model=None,+ turn_outcome=None,+ cancelled=ending == "cancelled",+ has_staged_proposal=False,+ )+ restore_mock, workflow_params = setup_new_copilot_mocks(monkeypatch, chat, original_workflow, agent_result)+ workflow_params.claim_pending_copilot_turn_for_finalisation.return_value = "reconciling"+ if ending == "errored":+ monkeypatch.setattr(+ workflow_copilot_route, "run_copilot_agent", AsyncMock(side_effect=RuntimeError("late failure"))+ )+ created_rows = workflow_params.create_workflow_copilot_chat_message.await_count++ await workflow_copilot_chat_post(api_key_request, _make_chat_request(), SimpleNamespace(organization_id="org-1"))+ handler = captured["handler"]+ assert callable(handler)+ await handler(copilot_stream)++ restore_mock.assert_not_awaited()+ workflow_params.update_workflow_copilot_chat.assert_not_awaited()+ assert workflow_params.create_workflow_copilot_chat_message.await_count == created_rows+ workflow_params.clear_pending_copilot_turn.assert_not_awaited()++@pytest.mark.asyncioasync def test_reconcile_writes_no_outcome_when_the_canonical_restore_fails(monkeypatch: pytest.MonkeyPatch,