Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
feat(SKY-9354): surface WIP workflow on copilot timeout / failed test (#5755)
Andrew Neilson4e1c46f7cf36
diff --git a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsxindex 451622284..ec18dde28 100644--- a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx+++ b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx@@ -738,7 +738,7 @@ export function WorkflowCopilotChat({};setMessages((prev) => [...prev, aiMessage]);- if (response.updated_workflow && autoAccept) {+ if (response.updated_workflow && autoAccept && !response.unvalidated) {applyWorkflowUpdate(response.updated_workflow);} else {setProposedWorkflow(response.updated_workflow ?? null);diff --git a/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.ts b/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.tsindex b3abca5b2..095ef3931 100644--- a/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.ts+++ b/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.ts@@ -83,6 +83,8 @@ export interface WorkflowCopilotStreamResponseUpdate {message: string;updated_workflow?: WorkflowApiResponse | null;response_time: string;+ // Clients must NOT auto-apply when true; render Accept/Reject explicitly.+ unvalidated?: boolean;}export interface WorkflowCopilotStreamErrorUpdate {diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.pyindex 6d564c7d8..438b2222c 100644--- a/skyvern/forge/sdk/copilot/agent.py+++ b/skyvern/forge/sdk/copilot/agent.py@@ -186,8 +186,9 @@ _FAILURE_FOLLOW_UP = {def _rewrite_failed_test_response(user_response: str, ctx: CopilotContext) -> str:- # Reshape replies when we cannot ship a proposal this turn so the user- # sees why nothing is being offered rather than an un-grounded claim.+ has_keepable_draft = ctx.last_workflow is not None and bool(ctx.last_workflow_yaml)+ keep_draft_affordance = " Keep the draft to iterate on, or discard." if has_keepable_draft else ""+if ctx.last_test_ok is False and ctx.last_update_block_count is not None:if ctx.last_update_block_count <= 0:draft_phrase = "a draft workflow"@@ -197,12 +198,17 @@ def _rewrite_failed_test_response(user_response: str, ctx: CopilotContext) -> stfailure_summary = _normalize_failure_reason(ctx.last_test_failure_reason)follow_up = _FAILURE_FOLLOW_UP.get(ctx.last_failure_category_top or "", "")- return f"I created {draft_phrase} and tested it, but the test failed. Failure: {failure_summary}.{follow_up}"+ return (+ f"I created {draft_phrase} and tested it, but the test failed. "+ f"Failure: {failure_summary}.{follow_up}{keep_draft_affordance}"+ )if ctx.last_test_ok is None and ctx.last_update_block_count is not None and ctx.last_workflow is not None:- # Agent edited the YAML but didn't verify it this turn; don't promise- # a re-run we can't durably execute (the restore helper rolls the- # mid-turn DB write back and there's no durable draft to re-test).+ if has_keepable_draft:+ return (+ "I drafted an update but wasn't able to verify it this turn. "+ "Keep the draft to iterate on it manually, or discard."+ )return ("I drafted an update but wasn't able to verify it this turn. ""Could you share more context about what you'd like me to do?"@@ -212,15 +218,8 @@ def _rewrite_failed_test_response(user_response: str, ctx: CopilotContext) -> stdef _verified_workflow_or_none(ctx: CopilotContext) -> tuple[Any, str | None]:- """Only surface a workflow proposal when it passed a test this turn.-- SKY-9143: the Accept/Reject UI must never reflect a workflow we haven't- proven works. Every agent exit path that builds an AgentResult directly- (cancel, max-turns, timeout, non-retriable nav error, catch-all Exception)- funnels through this so the strict invariant holds regardless of which- branch the run took.- """- if ctx.last_workflow is not None and ctx.last_test_ok is True:+ """Surface a proposal only when it passed a test this turn AND yaml is on hand."""+ if ctx.last_workflow is not None and ctx.last_workflow_yaml and ctx.last_test_ok is True:return ctx.last_workflow, ctx.last_workflow_yamlreturn None, None@@ -244,6 +243,76 @@ def _build_exit_result()+_TIMEOUT_REPLY_DEFAULT = "I ran out of time processing your request. Here's what I have so far."+_TIMEOUT_REPLY_UNVALIDATED = (+ "I ran out of time before I could finish testing. I have a draft workflow you can keep — "+ "accept it to save (note: it hasn't been verified end-to-end), or discard."+)+_TIMEOUT_REPLY_TESTED = "I ran out of time, but I have a tested draft for you. Accept it to save, or discard."++_MAX_TURNS_REPLY_DEFAULT = "I've reached the maximum number of steps. Here's what I have so far."+_MAX_TURNS_REPLY_UNVALIDATED = (+ "I've reached the maximum number of steps before I could finish testing. I have a draft "+ "workflow you can keep — accept it to save (note: it hasn't been verified end-to-end), or discard."+)+_MAX_TURNS_REPLY_TESTED = (+ "I've reached the maximum number of steps, but I have a tested draft for you. Accept it to save, or discard."+)+++def _build_capacity_exit_result(+ ctx: CopilotContext,+ global_llm_context: str | None,+ *,+ default_reply: str,+ unvalidated_reply: str,+ tested_reply: str,+) -> AgentResult:+ """Capacity-exhausted exits (timeout, max-turns) surface the most recent+ successfully-parsed workflow rather than discard the WIP."""+ # ``last_test_ok=None`` covers both "test never ran" and "test ran with+ # ambiguous output"; only the first case earns the carve-out (the REPLY+ # path is more permissive because its reply text carries the context).+ if (+ ctx.last_workflow is not None+ and ctx.last_workflow_yaml+ and ctx.last_test_ok is not False+ and not ctx.last_test_suspicious_success+ ):+ unvalidated = ctx.last_test_ok is not True+ reply = unvalidated_reply if unvalidated else tested_reply+ return AgentResult(+ user_response=reply,+ updated_workflow=ctx.last_workflow,+ global_llm_context=global_llm_context,+ workflow_yaml=ctx.last_workflow_yaml,+ workflow_was_persisted=ctx.workflow_persisted,+ total_tokens=ctx.total_tokens_used,+ unvalidated=unvalidated,+ )+ return _build_exit_result(ctx, default_reply, global_llm_context)+++def _build_timeout_exit_result(ctx: CopilotContext, global_llm_context: str | None) -> AgentResult:+ return _build_capacity_exit_result(+ ctx,+ global_llm_context,+ default_reply=_TIMEOUT_REPLY_DEFAULT,+ unvalidated_reply=_TIMEOUT_REPLY_UNVALIDATED,+ tested_reply=_TIMEOUT_REPLY_TESTED,+ )+++def _build_max_turns_exit_result(ctx: CopilotContext, global_llm_context: str | None) -> AgentResult:+ return _build_capacity_exit_result(+ ctx,+ global_llm_context,+ default_reply=_MAX_TURNS_REPLY_DEFAULT,+ unvalidated_reply=_MAX_TURNS_REPLY_UNVALIDATED,+ tested_reply=_MAX_TURNS_REPLY_TESTED,+ )++def _translate_to_agent_result(result: RunResultStreaming,ctx: CopilotContext,@@ -325,6 +394,12 @@ def _translate_to_agent_result(if resp_type != "ASK_QUESTION":user_response = _rewrite_failed_test_response(str(user_response), ctx)last_workflow, last_workflow_yaml = _verified_workflow_or_none(ctx)+ unvalidated = False+ if last_workflow is None and resp_type == "REPLY" and ctx.last_workflow is not None and ctx.last_workflow_yaml:+ # Failures are often environmental (captcha, transient block); surface the draft so the user can keep iterating.+ last_workflow = ctx.last_workflow+ last_workflow_yaml = ctx.last_workflow_yaml+ unvalidated = True# ASK_QUESTION blocks on user input — never surface a verified workflow# under it; auto_accept would silently apply a stepping-stone partial.@@ -353,6 +428,7 @@ def _translate_to_agent_result(workflow_was_persisted=ctx.workflow_persisted,total_tokens=ctx.total_tokens_used,clear_proposed_workflow=resp_type == "ASK_QUESTION",+ unvalidated=unvalidated,)@@ -564,17 +640,9 @@ async def run_copilot_agent(LOG.info("Copilot run cancelled by user")return _build_exit_result(ctx, "Cancelled by user.", global_llm_context, cancelled=True)except MaxTurnsExceeded:- return _build_exit_result(- ctx,- "I've reached the maximum number of steps. Here's what I have so far.",- global_llm_context,- )+ return _build_max_turns_exit_result(ctx, global_llm_context)except CopilotTotalTimeoutError:- return _build_exit_result(- ctx,- "I ran out of time processing your request. Here's what I have so far.",- global_llm_context,- )+ return _build_timeout_exit_result(ctx, global_llm_context)except CopilotNonRetriableNavError as exc:LOG.warning("Copilot run halted on non-retriable navigation error",diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.pyindex c0c4fc247..2b815a5b7 100644--- a/skyvern/forge/sdk/copilot/context.py+++ b/skyvern/forge/sdk/copilot/context.py@@ -127,6 +127,8 @@ class AgentResult:# persistence path (rollback + ``Cancelled by user.`` chat row) without# losing ``workflow_was_persisted`` the way a re-raise would.cancelled: bool = False+ # The route forces Accept/Reject regardless of ``auto_accept`` when this is True.+ unvalidated: bool = False@dataclassdiff --git a/skyvern/forge/sdk/routes/workflow_copilot.py b/skyvern/forge/sdk/routes/workflow_copilot.pyindex dcedb7951..e646db053 100644--- a/skyvern/forge/sdk/routes/workflow_copilot.py+++ b/skyvern/forge/sdk/routes/workflow_copilot.py@@ -170,20 +170,28 @@ async def _ensure_terminal_frame(stream: EventSourceStream, already_emitted: boopass-def _should_restore_persisted_workflow(auto_accept: bool | None, agent_result: object | None) -> bool:- """Return True when a persisted draft should be rolled back.+def _effective_auto_accept(auto_accept: bool | None, agent_result: object | None) -> bool:+ """``unvalidated`` overrides ``auto_accept=True`` to force explicit Accept/Reject."""+ if bool(getattr(agent_result, "unvalidated", False)):+ return False+ return auto_accept is True- SKY-9143: when the agent decided not to ship a proposal this turn- (``updated_workflow is None``) but ``_update_workflow`` already committed- a YAML to ``workflow_definition``, we must restore the original even under- ``auto_accept=True`` — otherwise an unverified edit becomes the live- workflow silently.- """++def _should_restore_persisted_workflow(auto_accept: bool | None, agent_result: object | None) -> bool:+ """Restore when a mid-turn ``_update_workflow`` commit isn't covered by an accepted proposal."""if not bool(getattr(agent_result, "workflow_was_persisted", False)):return Falseif getattr(agent_result, "updated_workflow", None) is None:return True- return auto_accept is not True+ return not _effective_auto_accept(auto_accept, agent_result)+++async def _clear_proposed_workflow(chat: Any) -> None:+ await app.DATABASE.workflow_params.update_workflow_copilot_chat(+ organization_id=chat.organization_id,+ workflow_copilot_chat_id=chat.workflow_copilot_chat_id,+ proposed_workflow=None,+ )async def _persist_cancel_turn(@@ -263,10 +271,13 @@ async def _finalise_normal_turn(if restored:await _restore_workflow_definition(original_workflow, organization_id)- if chat.auto_accept is not True and updated_workflow:+ auto_accept_effective = _effective_auto_accept(chat.auto_accept, agent_result)+ if not auto_accept_effective and updated_workflow:proposed_data = updated_workflow.model_dump(mode="json")if agent_result.workflow_yaml:proposed_data["_copilot_yaml"] = agent_result.workflow_yaml+ if agent_result.unvalidated:+ proposed_data["_copilot_unvalidated"] = Trueawait app.DATABASE.workflow_params.update_workflow_copilot_chat(organization_id=chat.organization_id,workflow_copilot_chat_id=chat.workflow_copilot_chat_id,@@ -277,11 +288,15 @@ async def _finalise_normal_turn(# so a reload does not resurrect a stale Accept/Reject card. Runs# under both auto_accept values — a stale proposal can survive an# auto-accept toggle.- await app.DATABASE.workflow_params.update_workflow_copilot_chat(- organization_id=chat.organization_id,- workflow_copilot_chat_id=chat.workflow_copilot_chat_id,- proposed_workflow=None,- )+ await _clear_proposed_workflow(chat)+ elif (+ auto_accept_effective+ and chat.proposed_workflow is not None+ and chat.proposed_workflow.get("_copilot_unvalidated") is True+ ):+ # The leftover unvalidated proposal is no longer attached to the chat+ # tail; clear it so reload doesn't resurrect a stale Accept/Reject card.+ await _clear_proposed_workflow(chat)await app.DATABASE.workflow_params.create_workflow_copilot_chat_message(organization_id=chat.organization_id,@@ -307,6 +322,7 @@ async def _finalise_normal_turn(response_time=assistant_message.created_at,total_tokens=getattr(agent_result, "total_tokens", None),response_type=getattr(agent_result, "response_type", "REPLY"),+ unvalidated=bool(getattr(agent_result, "unvalidated", False)),))diff --git a/skyvern/forge/sdk/schemas/workflow_copilot.py b/skyvern/forge/sdk/schemas/workflow_copilot.pyindex 89abd067b..9f91020a8 100644--- a/skyvern/forge/sdk/schemas/workflow_copilot.py+++ b/skyvern/forge/sdk/schemas/workflow_copilot.py@@ -117,6 +117,10 @@ class WorkflowCopilotStreamResponseUpdate(BaseModel):description="Total tokens consumed by the agent during this turn; None when no provider reported usage",)response_type: ResponseType = Field("REPLY", description="Agent response classification")+ unvalidated: bool = Field(+ False,+ description="When true, clients must not auto-apply; render Accept/Reject explicitly.",+ )class WorkflowCopilotStreamErrorUpdate(BaseModel):diff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.pyindex eea08109d..2d30aac4d 100644--- a/tests/unit/test_copilot_agent_helpers.py+++ b/tests/unit/test_copilot_agent_helpers.py@@ -110,6 +110,21 @@ class TestFailedTestResponseNormalization:original = "Let me know what you want to build."assert _rewrite_failed_test_response(original, ctx) == original+ def test_rewrite_appends_keep_draft_affordance_when_draft_on_hand(self) -> None:+ from skyvern.forge.sdk.copilot.agent import _rewrite_failed_test_response++ ctx = _ctx(+ last_workflow=object(),+ last_workflow_yaml="title: drafted",+ last_update_block_count=2,+ last_test_ok=False,+ last_test_failure_reason="A verification challenge is preventing submission.",+ )+ rewritten = _rewrite_failed_test_response("done", ctx)++ assert "test failed" in rewritten.lower()+ assert "keep the draft" in rewritten.lower()+class TestVerifiedWorkflowOrNone:"""SKY-9143 strict invariant: a proposal surfaces only after a passing test this turn."""@@ -167,6 +182,7 @@ class TestShouldRestorePersistedWorkflow:r = MagicMock()r.workflow_was_persisted = persistedr.updated_workflow = updated_workflow+ r.unvalidated = Falsereturn rdef test_restores_when_no_proposal_even_under_auto_accept(self) -> None:@@ -269,10 +285,10 @@ class TestTranslateToAgentResultGating:assert "validation error" in agent_result.user_response.lower()def test_ask_question_preserves_model_specific_question(self) -> None:- # The new prompt instructs the model to stop and ASK_QUESTION when it- # cannot test an edit. Row-3 of _rewrite_failed_test_response would- # clobber that specific unblocker with "Could you share more context";- # the resp_type==ASK_QUESTION guard must skip the rewrite.+ # The rewrite guard for ASK_QUESTION must hold: the agent's specific+ # clarifying question is not clobbered by the generic "share more+ # context" rewrite. SKY-9420 also drops any workflow under+ # ASK_QUESTION so an auto-accept user can't silently apply a partial.ctx = _ctx(last_update_block_count=1,last_test_ok=None,@@ -286,14 +302,11 @@ class TestTranslateToAgentResultGating:)assert agent_result.user_response == specific_question- # Even ASK_QUESTION must obey the strict gate — no verified workflow this turn.assert agent_result.updated_workflow is None+ assert agent_result.unvalidated is Falseassert agent_result.response_type == "ASK_QUESTION"def test_reply_still_rewrites_after_failed_test(self) -> None:- # Guard rail for the above: a plain REPLY after a failed test must- # still flow through the "test failed" rewrite so we don't regress- # the original SKY-9143 behavior.ctx = _ctx(last_update_block_count=2,last_test_ok=False,@@ -308,6 +321,44 @@ class TestTranslateToAgentResultGating:assert "test failed" in agent_result.user_response.lower()assert "All done" not in agent_result.user_responseassert agent_result.updated_workflow is None+ assert agent_result.unvalidated is False++ def test_reply_after_failed_test_surfaces_unvalidated_wip_when_draft_on_hand(self) -> None:+ wf = SimpleNamespace(name="drafted")+ ctx = _ctx(+ last_workflow=wf,+ last_workflow_yaml="title: drafted",+ last_update_block_count=4,+ last_test_ok=False,+ last_test_failure_reason="A verification challenge is preventing submission.",+ )+ result = _fake_run_result({"type": "REPLY", "user_response": "Done."})+ agent_result = agent_module._translate_to_agent_result(+ result, ctx, global_llm_context=None, chat_request=_chat_request(), organization_id="org-1"+ )++ assert agent_result.updated_workflow is wf+ assert agent_result.workflow_yaml == "title: drafted"+ assert agent_result.unvalidated is True+ assert "test failed" in agent_result.user_response.lower()+ assert "keep the draft" in agent_result.user_response.lower()++ def test_reply_after_suspicious_success_surfaces_unvalidated_wip(self) -> None:+ wf = SimpleNamespace(name="drafted")+ ctx = _ctx(+ last_workflow=wf,+ last_workflow_yaml="title: drafted",+ last_update_block_count=2,+ last_test_ok=None,+ last_test_suspicious_success=True,+ )+ result = _fake_run_result({"type": "REPLY", "user_response": "Done."})+ agent_result = agent_module._translate_to_agent_result(+ result, ctx, global_llm_context=None, chat_request=_chat_request(), organization_id="org-1"+ )++ assert agent_result.updated_workflow is wf+ assert agent_result.unvalidated is Truedef test_inline_replace_workflow_wraps_block_goals_with_user_message(self, monkeypatch) -> None:# SKY-9174 parity: update_and_run_blocks_tool wraps block goals withdiff --git a/tests/unit/test_copilot_timeout_wip.py b/tests/unit/test_copilot_timeout_wip.pynew file mode 100644index 000000000..ad8005d4a--- /dev/null+++ b/tests/unit/test_copilot_timeout_wip.py@@ -0,0 +1,172 @@+"""Tests for the capacity-exhausted WIP carve-outs (timeout, max-turns)."""++from __future__ import annotations++from unittest.mock import MagicMock++from skyvern.forge.sdk.copilot.agent import (+ _MAX_TURNS_REPLY_DEFAULT,+ _MAX_TURNS_REPLY_TESTED,+ _MAX_TURNS_REPLY_UNVALIDATED,+ _TIMEOUT_REPLY_DEFAULT,+ _TIMEOUT_REPLY_TESTED,+ _TIMEOUT_REPLY_UNVALIDATED,+ _build_max_turns_exit_result,+ _build_timeout_exit_result,+)+++def _ctx(+ *,+ last_workflow: object | None,+ last_workflow_yaml: str | None,+ last_test_ok: bool | None,+ last_test_suspicious_success: bool = False,+) -> MagicMock:+ ctx = MagicMock()+ ctx.last_workflow = last_workflow+ ctx.last_workflow_yaml = last_workflow_yaml+ ctx.last_test_ok = last_test_ok+ ctx.last_test_suspicious_success = last_test_suspicious_success+ ctx.workflow_persisted = last_workflow is not None+ ctx.total_tokens_used = None+ return ctx+++class TestBuildTimeoutExitResult:+ def test_no_workflow_falls_back_to_default_reply(self) -> None:+ ctx = _ctx(last_workflow=None, last_workflow_yaml=None, last_test_ok=None)++ result = _build_timeout_exit_result(ctx, global_llm_context=None)++ assert result.updated_workflow is None+ assert result.workflow_yaml is None+ assert result.unvalidated is False+ assert result.user_response == _TIMEOUT_REPLY_DEFAULT++ def test_untested_workflow_surfaces_as_unvalidated_wip(self) -> None:+ wf = MagicMock(name="wf")+ ctx = _ctx(last_workflow=wf, last_workflow_yaml="version: '1.0'", last_test_ok=None)++ result = _build_timeout_exit_result(ctx, global_llm_context=None)++ assert result.updated_workflow is wf+ assert result.workflow_yaml == "version: '1.0'"+ assert result.unvalidated is True+ assert result.user_response == _TIMEOUT_REPLY_UNVALIDATED++ def test_failed_test_drops_proposal(self) -> None:+ wf = MagicMock(name="wf")+ ctx = _ctx(last_workflow=wf, last_workflow_yaml="version: '1.0'", last_test_ok=False)++ result = _build_timeout_exit_result(ctx, global_llm_context=None)++ assert result.updated_workflow is None+ assert result.workflow_yaml is None+ assert result.unvalidated is False+ assert result.user_response == _TIMEOUT_REPLY_DEFAULT++ def test_passing_test_surfaces_as_tested_proposal(self) -> None:+ wf = MagicMock(name="wf")+ ctx = _ctx(last_workflow=wf, last_workflow_yaml="version: '1.0'", last_test_ok=True)++ result = _build_timeout_exit_result(ctx, global_llm_context=None)++ assert result.updated_workflow is wf+ assert result.workflow_yaml == "version: '1.0'"+ assert result.unvalidated is False+ assert result.user_response == _TIMEOUT_REPLY_TESTED++ def test_missing_yaml_drops_untested_proposal(self) -> None:+ wf = MagicMock(name="wf")+ ctx = _ctx(last_workflow=wf, last_workflow_yaml=None, last_test_ok=None)++ result = _build_timeout_exit_result(ctx, global_llm_context=None)++ assert result.updated_workflow is None+ assert result.workflow_yaml is None+ assert result.unvalidated is False+ assert result.user_response == _TIMEOUT_REPLY_DEFAULT++ def test_missing_yaml_drops_tested_proposal(self) -> None:+ wf = MagicMock(name="wf")+ ctx = _ctx(last_workflow=wf, last_workflow_yaml=None, last_test_ok=True)++ result = _build_timeout_exit_result(ctx, global_llm_context=None)++ assert result.updated_workflow is None+ assert result.workflow_yaml is None+ assert result.unvalidated is False+ assert result.user_response == _TIMEOUT_REPLY_DEFAULT++ def test_suspicious_success_drops_proposal(self) -> None:+ wf = MagicMock(name="wf")+ ctx = _ctx(+ last_workflow=wf,+ last_workflow_yaml="version: '1.0'",+ last_test_ok=None,+ last_test_suspicious_success=True,+ )++ result = _build_timeout_exit_result(ctx, global_llm_context=None)++ assert result.updated_workflow is None+ assert result.workflow_yaml is None+ assert result.unvalidated is False+ assert result.user_response == _TIMEOUT_REPLY_DEFAULT+++class TestBuildMaxTurnsExitResult:+ def test_no_workflow_falls_back_to_default_reply(self) -> None:+ ctx = _ctx(last_workflow=None, last_workflow_yaml=None, last_test_ok=None)++ result = _build_max_turns_exit_result(ctx, global_llm_context=None)++ assert result.updated_workflow is None+ assert result.unvalidated is False+ assert result.user_response == _MAX_TURNS_REPLY_DEFAULT++ def test_untested_workflow_surfaces_as_unvalidated_wip(self) -> None:+ wf = MagicMock(name="wf")+ ctx = _ctx(last_workflow=wf, last_workflow_yaml="version: '1.0'", last_test_ok=None)++ result = _build_max_turns_exit_result(ctx, global_llm_context=None)++ assert result.updated_workflow is wf+ assert result.unvalidated is True+ assert result.user_response == _MAX_TURNS_REPLY_UNVALIDATED++ def test_passing_test_surfaces_as_tested_proposal(self) -> None:+ wf = MagicMock(name="wf")+ ctx = _ctx(last_workflow=wf, last_workflow_yaml="version: '1.0'", last_test_ok=True)++ result = _build_max_turns_exit_result(ctx, global_llm_context=None)++ assert result.updated_workflow is wf+ assert result.unvalidated is False+ assert result.user_response == _MAX_TURNS_REPLY_TESTED++ def test_failed_test_drops_proposal(self) -> None:+ wf = MagicMock(name="wf")+ ctx = _ctx(last_workflow=wf, last_workflow_yaml="version: '1.0'", last_test_ok=False)++ result = _build_max_turns_exit_result(ctx, global_llm_context=None)++ assert result.updated_workflow is None+ assert result.unvalidated is False+ assert result.user_response == _MAX_TURNS_REPLY_DEFAULT++ def test_suspicious_success_drops_proposal(self) -> None:+ wf = MagicMock(name="wf")+ ctx = _ctx(+ last_workflow=wf,+ last_workflow_yaml="version: '1.0'",+ last_test_ok=None,+ last_test_suspicious_success=True,+ )++ result = _build_max_turns_exit_result(ctx, global_llm_context=None)++ assert result.updated_workflow is None+ assert result.unvalidated is False+ assert result.user_response == _MAX_TURNS_REPLY_DEFAULTdiff --git a/tests/unit/test_workflow_copilot_route.py b/tests/unit/test_workflow_copilot_route.pyindex a330a6974..1b2f8e95e 100644--- a/tests/unit/test_workflow_copilot_route.py+++ b/tests/unit/test_workflow_copilot_route.py@@ -228,6 +228,7 @@ async def test_flag_on_mid_stream_disconnect_restores_when_persisted_and_not_autworkflow_yaml=None,workflow_was_persisted=workflow_was_persisted,clear_proposed_workflow=False,+ unvalidated=False,)restore_mock = _setup_new_copilot_mocks(monkeypatch, chat, original_workflow, agent_result)@@ -290,6 +291,10 @@ async def test_flag_on_mid_stream_disconnect_restores_when_persisted_and_not_aut# No prior proposal => no DB write even when the clear flag is set.(False, False, False, None, True, False),(True, False, False, None, True, False),+ # auto_accept=True turn with a stale UNVALIDATED proposal clears it+ # via the third elif (no clear flag, no restore needed).+ (True, False, False, {"workflow_id": "stale", "_copilot_unvalidated": True}, False, True),+ (True, True, True, {"workflow_id": "stale", "_copilot_unvalidated": True}, False, True),],)async def test_proposed_workflow_cleared_on_restore(@@ -328,6 +333,7 @@ async def test_proposed_workflow_cleared_on_restore(workflow_yaml=None,workflow_was_persisted=workflow_was_persisted,clear_proposed_workflow=clear_proposed_flag,+ unvalidated=False,)_setup_new_copilot_mocks(monkeypatch, chat, original_workflow, agent_result)@@ -367,6 +373,79 @@ async def test_proposed_workflow_cleared_on_restore(assert response_frames[0].updated_workflow == expected_payload_workflow+async def test_unvalidated_timeout_wip_overrides_auto_accept(monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setattr(settings, "ENABLE_WORKFLOW_COPILOT_V2", True)++ captured = _install_fake_create(monkeypatch)++ chat = SimpleNamespace(+ workflow_copilot_chat_id="chat-1",+ workflow_permanent_id="wpid-1",+ organization_id="org-1",+ proposed_workflow=None,+ auto_accept=True,+ )+ original_workflow = SimpleNamespace(+ workflow_id="wf-canonical",+ title="Original",+ description="Original description",+ workflow_definition=None,+ )+ proposal = MagicMock(spec=["model_dump"])+ proposal.model_dump.return_value = {"workflow_id": "wf-canonical"}+ agent_result = SimpleNamespace(+ user_response="I ran out of time before I could finish testing.",+ updated_workflow=proposal,+ global_llm_context=None,+ workflow_yaml="title: WIP",+ workflow_was_persisted=True,+ clear_proposed_workflow=False,+ unvalidated=True,+ total_tokens=42,+ response_type="REPLY",+ )++ restore_mock = _setup_new_copilot_mocks(monkeypatch, chat, original_workflow, agent_result)++ request = MagicMock()+ request.headers = {"x-api-key": "sk-test-key"}+ organization = SimpleNamespace(organization_id="org-1")++ response = await workflow_copilot_chat_post(request, _make_chat_request(), organization)+ assert response is captured["sentinel"]++ sent_frames: list[object] = []+ stream = MagicMock()++ async def capture_send(payload: object) -> bool:+ sent_frames.append(payload)+ return True++ stream.send = capture_send+ stream.is_disconnected = AsyncMock(return_value=False)++ handler = captured["handler"]+ assert callable(handler)+ await handler(stream)++ restore_mock.assert_awaited_once()++ update_calls = app.DATABASE.workflow_params.update_workflow_copilot_chat.await_args_list+ proposal_writes = [c for c in update_calls if c.kwargs.get("proposed_workflow") is not None]+ assert len(proposal_writes) == 1+ proposed_data = proposal_writes[0].kwargs["proposed_workflow"]+ assert proposed_data.get("_copilot_unvalidated") is True+ assert proposed_data.get("_copilot_yaml") == "title: WIP"++ response_frame = next(+ (f for f in sent_frames if getattr(f, "type", None) and str(f.type).endswith("response")),+ None,+ )+ assert response_frame is not None+ assert getattr(response_frame, "unvalidated", False) is True++@pytest.mark.asyncioasync def test_env_on_short_circuits_posthog(monkeypatch: pytest.MonkeyPatch) -> None:"""Env var True must skip the PostHog check entirely and route to v2."""diff --git a/tests/unit/test_workflow_copilot_route_helpers.py b/tests/unit/test_workflow_copilot_route_helpers.pyindex fa160e6da..e9efd1f5f 100644--- a/tests/unit/test_workflow_copilot_route_helpers.py+++ b/tests/unit/test_workflow_copilot_route_helpers.py@@ -12,7 +12,11 @@ from __future__ import annotationsfrom unittest.mock import MagicMock-from skyvern.forge.sdk.routes.workflow_copilot import _normalize_copilot_yaml, _should_restore_persisted_workflow+from skyvern.forge.sdk.routes.workflow_copilot import (+ _effective_auto_accept,+ _normalize_copilot_yaml,+ _should_restore_persisted_workflow,+)class TestShouldRestorePersistedWorkflow:@@ -26,6 +30,7 @@ class TestShouldRestorePersistedWorkflow:def test_does_not_restore_for_auto_accept_or_unpersisted_result(self) -> None:persisted = MagicMock()persisted.workflow_was_persisted = True+ persisted.unvalidated = Falsenot_persisted = MagicMock()not_persisted.workflow_was_persisted = False@@ -33,6 +38,36 @@ class TestShouldRestorePersistedWorkflow:assert _should_restore_persisted_workflow(False, not_persisted) is Falseassert _should_restore_persisted_workflow(False, None) is False+ def test_unvalidated_timeout_wip_forces_rollback_under_auto_accept(self) -> None:+ agent_result = MagicMock()+ agent_result.workflow_was_persisted = True+ agent_result.unvalidated = True+ agent_result.updated_workflow = MagicMock()++ assert _should_restore_persisted_workflow(True, agent_result) is True+ assert _should_restore_persisted_workflow(False, agent_result) is True+++class TestEffectiveAutoAccept:+ def test_unvalidated_overrides_auto_accept(self) -> None:+ unvalidated = MagicMock()+ unvalidated.unvalidated = True++ assert _effective_auto_accept(True, unvalidated) is False+ assert _effective_auto_accept(False, unvalidated) is False++ def test_validated_proposal_respects_auto_accept_setting(self) -> None:+ validated = MagicMock()+ validated.unvalidated = False++ assert _effective_auto_accept(True, validated) is True+ assert _effective_auto_accept(False, validated) is False+ assert _effective_auto_accept(None, validated) is False++ def test_no_agent_result_falls_back_to_user_setting(self) -> None:+ assert _effective_auto_accept(True, None) is True+ assert _effective_auto_accept(False, None) is False+class TestNormalizeCopilotYamlTitleCoercion:def test_missing_top_level_title_is_coerced_to_empty(self) -> None: