Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix(SKY-9935): decouple WIP-rescue force-review from unvalidated proposal flag (#6094)
Andrew Neilson4264933701d5
diff --git a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.test.ts b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.test.tsnew file mode 100644index 000000000..675e1d38a--- /dev/null+++ b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.test.ts@@ -0,0 +1,73 @@+import { describe, expect, it } from "vitest";++import { shouldAutoApplyWorkflowResponse } from "./proposalDisposition";+import type { WorkflowCopilotStreamResponseUpdate } from "./workflowCopilotTypes";++const response = (+ overrides: Partial<WorkflowCopilotStreamResponseUpdate> = {},+): WorkflowCopilotStreamResponseUpdate => ({+ type: "response",+ workflow_copilot_chat_id: "wcc_1",+ message: "done",+ updated_workflow: { workflow_id: "wf_1" } as never,+ response_time: "2026-05-21T00:00:00Z",+ proposal_disposition: "auto_applicable",+ ...overrides,+});++describe("shouldAutoApplyWorkflowResponse", () => {+ it("auto-applies auto_applicable proposals when auto accept is enabled", () => {+ expect(shouldAutoApplyWorkflowResponse(response(), true, false)).toBe(true);+ });++ it.each(["review_untested", "review_tested"] as const)(+ "forces explicit review for %s proposals",+ (proposal_disposition) => {+ expect(+ shouldAutoApplyWorkflowResponse(+ response({ proposal_disposition }),+ true,+ false,+ ),+ ).toBe(false);+ },+ );++ it("uses the enum over legacy booleans when both are present", () => {+ expect(+ shouldAutoApplyWorkflowResponse(+ response({+ proposal_disposition: "auto_applicable",+ unvalidated: true,+ force_review: true,+ }),+ true,+ false,+ ),+ ).toBe(true);+ });++ it("falls back to legacy review booleans before the enum ships", () => {+ expect(+ shouldAutoApplyWorkflowResponse(+ response({+ proposal_disposition: undefined,+ force_review: true,+ }),+ true,+ false,+ ),+ ).toBe(false);+ });++ it("does not auto-apply cancelled turns", () => {+ expect(+ shouldAutoApplyWorkflowResponse(+ response({ cancelled: true }),+ true,+ false,+ ),+ ).toBe(false);+ expect(shouldAutoApplyWorkflowResponse(response(), true, true)).toBe(false);+ });+});diff --git a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsxindex 2e20c71a3..ec11e563d 100644--- a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx+++ b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx@@ -43,6 +43,7 @@ import {shouldQueuePromptForLiveBrowser,shouldWaitForLiveBrowser,} from "./browserReadiness";+import { shouldAutoApplyWorkflowResponse } from "./proposalDisposition";interface ChatMessage {id: string;@@ -865,10 +866,11 @@ export function WorkflowCopilotChat({cancelInFlightController.current === abortController;if (response.updated_workflow &&- autoAccept &&- !response.unvalidated &&- !response.cancelled &&- !userCancelledThisTurn+ shouldAutoApplyWorkflowResponse(+ response,+ autoAccept,+ userCancelledThisTurn,+ )) {applyWorkflowUpdate(response.updated_workflow);} else {diff --git a/skyvern-frontend/src/routes/workflows/copilot/proposalDisposition.ts b/skyvern-frontend/src/routes/workflows/copilot/proposalDisposition.tsnew file mode 100644index 000000000..dcab4bbbc--- /dev/null+++ b/skyvern-frontend/src/routes/workflows/copilot/proposalDisposition.ts@@ -0,0 +1,19 @@+import type { WorkflowCopilotStreamResponseUpdate } from "./workflowCopilotTypes";++export const shouldAutoApplyWorkflowResponse = (+ response: WorkflowCopilotStreamResponseUpdate,+ autoAccept: boolean,+ userCancelledThisTurn: boolean,+) => {+ const autoApplicable =+ response.proposal_disposition !== undefined+ ? response.proposal_disposition === "auto_applicable"+ : !response.unvalidated && !response.force_review;+ return Boolean(+ response.updated_workflow &&+ autoAccept &&+ autoApplicable &&+ !response.cancelled &&+ !userCancelledThisTurn,+ );+};diff --git a/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.ts b/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.tsindex 622e88027..d88c3e6bf 100644--- a/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.ts+++ b/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.ts@@ -4,6 +4,10 @@ import {} from "@/routes/workflows/types/workflowTypes";export type WorkflowCopilotChatSender = "user" | "ai";+export type ProposalDisposition =+ | "auto_applicable"+ | "review_untested"+ | "review_tested";export interface WorkflowCopilotChat {workflow_copilot_chat_id: string;@@ -83,10 +87,13 @@ export interface WorkflowCopilotStreamResponseUpdate {message: string;updated_workflow?: WorkflowApiResponse | null;response_time: string;- // Clients must NOT auto-apply when true; render Accept/Reject explicitly.+ proposal_disposition?: ProposalDisposition;+ // Legacy mirror for proposal_disposition === "review_untested".unvalidated?: boolean;- // Same auto-apply guard as unvalidated; cancel forces explicit review.+ // Cancel forces explicit review.cancelled?: boolean;+ // Legacy mirror for proposal_disposition === "review_tested".+ force_review?: boolean;}export interface WorkflowCopilotStreamErrorUpdate {diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.pyindex 3bc6d74ad..4536c4694 100644--- a/skyvern/forge/sdk/copilot/agent.py+++ b/skyvern/forge/sdk/copilot/agent.py@@ -745,10 +745,8 @@ def _build_wip_exit_result(cancelled: bool = False,) -> AgentResult:"""Selected non-success exits surface the most recent successfully parsed workflow."""- # When an unverified edit/run has overwritten ``last_workflow`` since the- # last verified shape, prefer the verified shape. ``unvalidated=True``- # triggers the route's rollback so auto-accept does not silently keep the- # failed/in-flight shape.+ # When an unverified edit/run has overwritten ``last_workflow``, prefer the+ # verified shape while still forcing explicit review.if (ctx.last_good_workflow is not Noneand ctx.last_good_workflow_yaml@@ -762,7 +760,7 @@ def _build_wip_exit_result(workflow_yaml=ctx.last_good_workflow_yaml,workflow_was_persisted=ctx.workflow_persisted,total_tokens=ctx.total_tokens_used,- unvalidated=True,+ proposal_disposition="review_tested",cancelled=cancelled,)if (@@ -780,7 +778,7 @@ def _build_wip_exit_result(workflow_yaml=ctx.last_workflow_yaml,workflow_was_persisted=ctx.workflow_persisted,total_tokens=ctx.total_tokens_used,- unvalidated=unvalidated,+ proposal_disposition="review_untested" if unvalidated else "auto_applicable",cancelled=cancelled,)return _build_exit_result(ctx, default_reply, global_llm_context, cancelled=cancelled)@@ -1089,7 +1087,7 @@ def _translate_to_agent_result(workflow_was_persisted=ctx.workflow_persisted,total_tokens=ctx.total_tokens_used,clear_proposed_workflow=resp_type == "ASK_QUESTION",- unvalidated=unvalidated,+ proposal_disposition="review_untested" if unvalidated else "auto_applicable",output_policy_diagnostics=output_policy_diagnostics,)diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.pyindex 1d1802992..380bda33d 100644--- a/skyvern/forge/sdk/copilot/context.py+++ b/skyvern/forge/sdk/copilot/context.py@@ -13,6 +13,7 @@ from skyvern.forge.sdk.workflow.models.workflow import WorkflowResponseType = Literal["REPLY", "ASK_QUESTION", "REPLACE_WORKFLOW"]COPILOT_RESPONSE_TYPES: tuple[ResponseType, ...] = get_args(ResponseType)+ProposalDisposition = Literal["auto_applicable", "review_untested", "review_tested"]if TYPE_CHECKING:from skyvern.forge.sdk.copilot.diagnosis_repair_contract import DiagnosisRepairContract@@ -131,10 +132,18 @@ class AgentResult:# persistence path (rollback + ``Cancelled by user.`` chat row) without# losing ``workflow_was_persisted`` the way a re-raise would.cancelled: bool = False- # The route forces Accept/Reject regardless of ``auto_accept`` when this is True.- unvalidated: bool = False+ # Controls whether the route may auto-apply the proposal or must force explicit review.+ proposal_disposition: ProposalDisposition = "auto_applicable"output_policy_diagnostics: dict[str, Any] | None = None+ @property+ def unvalidated(self) -> bool:+ return self.proposal_disposition == "review_untested"++ @property+ def force_review(self) -> bool:+ return self.proposal_disposition == "review_tested"+@dataclassclass CopilotContext(AgentContext):diff --git a/skyvern/forge/sdk/routes/workflow_copilot.py b/skyvern/forge/sdk/routes/workflow_copilot.pyindex 67f712a02..8f144d55d 100644--- a/skyvern/forge/sdk/routes/workflow_copilot.py+++ b/skyvern/forge/sdk/routes/workflow_copilot.py@@ -25,7 +25,7 @@ from skyvern.forge.sdk.copilot.agent import run_copilot_agentfrom skyvern.forge.sdk.copilot.attribution import is_copilot_born_initial_writefrom skyvern.forge.sdk.copilot.block_type_aliases import normalize_copilot_block_type_aliasfrom skyvern.forge.sdk.copilot.config import CopilotConfig-from skyvern.forge.sdk.copilot.context import AgentResult+from skyvern.forge.sdk.copilot.context import AgentResult, ProposalDispositionfrom skyvern.forge.sdk.copilot.output_utils import truncate_outputfrom skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.experimentation.llm_prompt_config import get_llm_handler_for_prompt_type@@ -253,9 +253,28 @@ async def _ensure_terminal_frame(stream: EventSourceStream, already_emitted: boopass+def _proposal_disposition(agent_result: object | None) -> ProposalDisposition:+ disposition = getattr(agent_result, "proposal_disposition", None)+ if disposition in ("auto_applicable", "review_untested", "review_tested"):+ return disposition+ if getattr(agent_result, "unvalidated", False) is True:+ return "review_untested"+ if getattr(agent_result, "force_review", False) is True:+ return "review_tested"+ return "auto_applicable"+++def _legacy_unvalidated(proposal_disposition: ProposalDisposition) -> bool:+ return proposal_disposition == "review_untested"+++def _legacy_force_review(proposal_disposition: ProposalDisposition) -> bool:+ return proposal_disposition == "review_tested"++def _effective_auto_accept(auto_accept: bool | None, agent_result: object | None) -> bool:- """``unvalidated`` and ``cancelled`` override ``auto_accept=True`` to force explicit Accept/Reject."""- if bool(getattr(agent_result, "unvalidated", False)) or bool(getattr(agent_result, "cancelled", False)):+ """Only auto-applicable proposals may honor ``auto_accept=True``."""+ if getattr(agent_result, "cancelled", False) is True or _proposal_disposition(agent_result) != "auto_applicable":return Falsereturn auto_accept is True@@ -281,7 +300,7 @@ def _build_proposed_workflow_data(updated_workflow: Workflow, agent_result: Agenproposed_data = dict(updated_workflow.model_dump(mode="json"))if agent_result.workflow_yaml:proposed_data["_copilot_yaml"] = agent_result.workflow_yaml- if agent_result.unvalidated:+ if _proposal_disposition(agent_result) == "review_untested":proposed_data["_copilot_unvalidated"] = Truereturn proposed_data@@ -331,7 +350,6 @@ async def _persist_cancel_turn(updated_global_llm_context = Nonetotal_tokens = Noneresponse_type = "REPLY"- unvalidated = Falseoutput_policy_diagnostics = Noneif chat.proposed_workflow is not None:await asyncio.shield(_clear_proposed_workflow(chat))@@ -348,7 +366,6 @@ async def _persist_cancel_turn(updated_global_llm_context = agent_result.global_llm_contexttotal_tokens = getattr(agent_result, "total_tokens", None)response_type = getattr(agent_result, "response_type", "REPLY")- unvalidated = bool(getattr(agent_result, "unvalidated", False))output_policy_diagnostics = getattr(agent_result, "output_policy_diagnostics", None)await asyncio.shield(@@ -368,6 +385,7 @@ async def _persist_cancel_turn(global_llm_context=updated_global_llm_context,))+ proposal_disposition = _proposal_disposition(agent_result)try:await asyncio.shield(stream.send(@@ -379,9 +397,11 @@ async def _persist_cancel_turn(response_time=assistant_message.created_at,total_tokens=total_tokens,response_type=response_type,- unvalidated=unvalidated,+ proposal_disposition=proposal_disposition,+ unvalidated=_legacy_unvalidated(proposal_disposition),cancelled=True,output_policy_diagnostics=output_policy_diagnostics,+ force_review=_legacy_force_review(proposal_disposition),)))@@ -444,6 +464,7 @@ async def _finalise_normal_turn(global_llm_context=updated_global_llm_context,)+ proposal_disposition = _proposal_disposition(agent_result)await stream.send(WorkflowCopilotStreamResponseUpdate(type=WorkflowCopilotStreamMessageType.RESPONSE,@@ -453,8 +474,10 @@ async def _finalise_normal_turn(response_time=assistant_message.created_at,total_tokens=getattr(agent_result, "total_tokens", None),response_type=getattr(agent_result, "response_type", "REPLY"),- unvalidated=bool(getattr(agent_result, "unvalidated", False)),+ proposal_disposition=proposal_disposition,+ unvalidated=_legacy_unvalidated(proposal_disposition),output_policy_diagnostics=getattr(agent_result, "output_policy_diagnostics", None),+ force_review=_legacy_force_review(proposal_disposition),))diff --git a/skyvern/forge/sdk/schemas/workflow_copilot.py b/skyvern/forge/sdk/schemas/workflow_copilot.pyindex 37f38a37a..3e05711ee 100644--- a/skyvern/forge/sdk/schemas/workflow_copilot.py+++ b/skyvern/forge/sdk/schemas/workflow_copilot.py@@ -4,7 +4,7 @@ from typing import Anyfrom pydantic import BaseModel, ConfigDict, Field-from skyvern.forge.sdk.copilot.context import ResponseType+from skyvern.forge.sdk.copilot.context import ProposalDisposition, ResponseTypeclass WorkflowCopilotChat(BaseModel):@@ -118,9 +118,13 @@ class WorkflowCopilotStreamResponseUpdate(BaseModel):description="Total tokens consumed by the agent during this turn; None when no provider reported usage",)response_type: ResponseType = Field("REPLY", description="Agent response classification")+ proposal_disposition: ProposalDisposition = Field(+ "auto_applicable",+ description="Whether this proposal may auto-apply or must be reviewed explicitly.",+ )unvalidated: bool = Field(False,- description="When true, clients must not auto-apply; render Accept/Reject explicitly.",+ description="Legacy mirror for proposal_disposition=review_untested.",)cancelled: bool = Field(False,@@ -130,6 +134,10 @@ class WorkflowCopilotStreamResponseUpdate(BaseModel):None,description="Diagnostic output-policy labels for raw-vs-final quality reporting.",)+ force_review: bool = Field(+ False,+ description="Legacy mirror for proposal_disposition=review_tested.",+ )class WorkflowCopilotStreamErrorUpdate(BaseModel):diff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.pyindex d84f8c122..9e30fdcdf 100644--- a/tests/unit/test_copilot_agent_helpers.py+++ b/tests/unit/test_copilot_agent_helpers.py@@ -477,6 +477,7 @@ class TestShouldRestorePersistedWorkflow:r.updated_workflow = updated_workflowr.unvalidated = Falser.cancelled = False+ r.force_review = Falsereturn rdef test_restores_when_no_proposal_even_under_auto_accept(self) -> None:diff --git a/tests/unit/test_copilot_mid_correction_guard.py b/tests/unit/test_copilot_mid_correction_guard.pyindex f6296d9e4..e9965a084 100644--- a/tests/unit/test_copilot_mid_correction_guard.py+++ b/tests/unit/test_copilot_mid_correction_guard.py@@ -134,7 +134,8 @@ def test_build_timeout_exit_result_surfaces_last_good_after_failure() -> None:result = _build_timeout_exit_result(ctx, global_llm_context=None)assert result.updated_workflow is ctx.last_good_workflowassert result.workflow_yaml == "yaml-good"- assert result.unvalidated is True+ assert result.unvalidated is False+ assert result.force_review is Trueassert result.user_response == _TIMEOUT_REPLY_TESTED@@ -150,7 +151,8 @@ def test_build_timeout_exit_result_surfaces_last_good_on_mid_flight_cancellationctx.last_good_workflow_yaml = "yaml-good"result = _build_timeout_exit_result(ctx, global_llm_context=None)assert result.updated_workflow is ctx.last_good_workflow- assert result.unvalidated is True+ assert result.unvalidated is False+ assert result.force_review is Truedef test_build_timeout_exit_result_falls_through_when_no_last_good() -> None:diff --git a/tests/unit/test_copilot_timeout_wip.py b/tests/unit/test_copilot_timeout_wip.pyindex 0da45fdce..ade6f32b2 100644--- a/tests/unit/test_copilot_timeout_wip.py+++ b/tests/unit/test_copilot_timeout_wip.py@@ -321,3 +321,72 @@ class TestBuildCancelExitResult:assert result.workflow_yaml is Noneassert result.unvalidated is Falseassert result.user_response == _CANCEL_REPLY_DEFAULT+++class TestWipExitSurfacesLastGoodWithForceReviewNotUnvalidated:+ """Mid-flight overwrite branch offers ``last_good_workflow`` with ``force_review=True, unvalidated=False``."""++ def _overwrite_ctx(self, *, last_test_ok: bool | None) -> MagicMock:+ good = MagicMock(name="wf-good")+ in_flight = MagicMock(name="wf-in-flight")+ return _ctx(+ last_workflow=in_flight,+ last_workflow_yaml="version: in-flight",+ last_test_ok=last_test_ok,+ last_good_workflow=good,+ last_good_workflow_yaml="version: good",+ )++ def test_cancel_with_overwrite_surfaces_last_good_as_tested_force_review(self) -> None:+ ctx = self._overwrite_ctx(last_test_ok=None)++ result = _build_cancel_exit_result(ctx, global_llm_context=None)++ assert result.cancelled is True+ assert result.updated_workflow is ctx.last_good_workflow+ assert result.workflow_yaml == "version: good"+ assert result.unvalidated is False+ assert result.force_review is True+ assert result.user_response == _CANCEL_REPLY_TESTED++ def test_timeout_with_overwrite_surfaces_last_good_as_tested_force_review(self) -> None:+ ctx = self._overwrite_ctx(last_test_ok=False)++ result = _build_timeout_exit_result(ctx, global_llm_context=None)++ assert result.updated_workflow is ctx.last_good_workflow+ assert result.unvalidated is False+ assert result.force_review is True+ assert result.user_response == _TIMEOUT_REPLY_TESTED++ def test_max_turns_with_overwrite_surfaces_last_good_as_tested_force_review(self) -> None:+ ctx = self._overwrite_ctx(last_test_ok=None)++ result = _build_max_turns_exit_result(ctx, global_llm_context=None)++ assert result.updated_workflow is ctx.last_good_workflow+ assert result.unvalidated is False+ assert result.force_review is True+ assert result.user_response == _MAX_TURNS_REPLY_TESTED++ def test_unexpected_error_with_overwrite_surfaces_last_good_as_tested_force_review(self) -> None:+ ctx = self._overwrite_ctx(last_test_ok=None)++ result = _build_unexpected_error_exit_result(ctx, global_llm_context=None)++ assert result.updated_workflow is ctx.last_good_workflow+ assert result.unvalidated is False+ assert result.force_review is True+ assert result.user_response == _UNEXPECTED_ERROR_REPLY_TESTED++ def test_cancelled_total_timeout_latch_uses_force_review_not_unvalidated(self) -> None:+ ctx = self._overwrite_ctx(last_test_ok=None)+ ctx.copilot_total_timeout_exceeded = True++ result = _build_cancelled_exit_result(ctx, global_llm_context=None)++ assert result.updated_workflow is ctx.last_good_workflow+ assert result.unvalidated is False+ assert result.force_review is True+ assert result.cancelled is False+ assert result.user_response == _TIMEOUT_REPLY_TESTEDdiff --git a/tests/unit/test_workflow_copilot_cancel.py b/tests/unit/test_workflow_copilot_cancel.pyindex c3b396f3f..a99e3222f 100644--- a/tests/unit/test_workflow_copilot_cancel.py+++ b/tests/unit/test_workflow_copilot_cancel.py@@ -380,6 +380,7 @@ async def test_route_cancel_branch_persists_wip_proposal_and_response_frame(frame = response_frames[0]assert frame.message == "Cancelled. I have a draft workflow you can keep."assert frame.updated_workflow == {"workflow_id": "wf-canonical", "title": "Draft"}+ assert frame.proposal_disposition == "review_untested"assert frame.unvalidated is Trueassert frame.cancelled is True@@ -430,6 +431,53 @@ async def test_route_cancel_tested_wip_with_auto_accept_still_persists_proposal(assert response_frames[0].cancelled is True+async def test_route_cancel_force_review_persists_proposal_without_unvalidated_marker(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """Cancel + WIP-rescue: persisted proposal must NOT carry the ``_copilot_unvalidated`` marker."""+ chat = _make_chat(auto_accept=True)+ original_workflow = _make_original_workflow()+ updated_workflow = MagicMock()+ updated_workflow.model_dump.return_value = {"workflow_id": "wf-canonical", "title": "Last Good Draft"}+ agent_result = SimpleNamespace(+ user_response="Cancelled. I have a tested draft for you. Accept it to save, or discard.",+ updated_workflow=updated_workflow,+ global_llm_context=None,+ workflow_yaml="title: Last Good Draft",+ workflow_was_persisted=True,+ clear_proposed_workflow=False,+ cancelled=True,+ total_tokens=456,+ response_type="REPLY",+ unvalidated=False,+ force_review=True,+ )+ restore_mock, workflow_params, sent_payloads = await _drive_cancel_route(+ monkeypatch, chat, original_workflow, agent_result+ )++ restore_mock.assert_awaited_once()++ workflow_params.update_workflow_copilot_chat.assert_awaited_once()+ proposed_workflow = workflow_params.update_workflow_copilot_chat.await_args.kwargs["proposed_workflow"]+ assert proposed_workflow == {+ "workflow_id": "wf-canonical",+ "title": "Last Good Draft",+ "_copilot_yaml": "title: Last Good Draft",+ }+ assert "_copilot_unvalidated" not in proposed_workflow++ response_frames = [+ p for p in sent_payloads if getattr(p, "type", None) == WorkflowCopilotStreamMessageType.RESPONSE+ ]+ assert len(response_frames) == 1+ assert response_frames[0].proposal_disposition == "review_tested"+ assert response_frames[0].unvalidated is False+ assert response_frames[0].cancelled is True+ assert response_frames[0].force_review is True++@pytest.mark.asyncioasync def test_route_cancel_clears_stale_proposed_workflow_when_no_wip(monkeypatch: pytest.MonkeyPatch,@@ -592,6 +640,7 @@ async def test_timeout_wip_result_streams_normal_response_frame(]assert len(response_frames) == 1assert response_frames[0].updated_workflow == {"workflow_id": "wf-draft", "title": "Draft"}+ assert response_frames[0].proposal_disposition == "review_untested"assert response_frames[0].unvalidated is Trueassert response_frames[0].total_tokens == 123@@ -601,6 +650,80 @@ async def test_timeout_wip_result_streams_normal_response_frame(assert "Cancelled by user." not in contents+async def test_timeout_wip_force_review_propagates_to_response_frame(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """Non-cancel WIP rescue propagates ``force_review`` to the SSE frame so the frontend skips auto-apply."""+ monkeypatch.setattr(settings, "ENABLE_WORKFLOW_COPILOT_V2", True)+ captured = _install_fake_create(monkeypatch)++ chat = SimpleNamespace(+ workflow_copilot_chat_id="chat-1",+ workflow_permanent_id="wpid-1",+ organization_id="org-1",+ proposed_workflow=None,+ auto_accept=True,+ )+ original_workflow = SimpleNamespace(+ workflow_id="wf-canonical",+ title="Original",+ description="Original description",+ workflow_definition=None,+ )+ updated_workflow = MagicMock()+ updated_workflow.model_dump.side_effect = lambda mode="json": {"workflow_id": "wf-good", "title": "Last Good"}+ agent_result = SimpleNamespace(+ user_response="I ran out of time, but I have a tested draft for you. Accept it to save, or discard.",+ updated_workflow=updated_workflow,+ global_llm_context=None,+ workflow_yaml="title: Last Good",+ workflow_was_persisted=True,+ clear_proposed_workflow=False,+ cancelled=False,+ total_tokens=789,+ response_type="REPLY",+ unvalidated=False,+ force_review=True,+ )+ restore_mock, workflow_params = _setup_route_mocks(monkeypatch, chat, original_workflow, agent_result)++ request = MagicMock()+ request.headers = {"x-api-key": "sk-test"}+ organization = SimpleNamespace(organization_id="org-1")++ response = await workflow_copilot_chat_post(request, _make_chat_request(), organization)+ assert response is captured["sentinel"]++ sent_payloads: list[Any] = []+ stream = MagicMock()++ async def _send(payload: Any) -> bool:+ sent_payloads.append(payload)+ return True++ stream.send = _send+ stream.is_disconnected = AsyncMock(return_value=False)++ handler = captured["handler"]+ assert callable(handler)+ await handler(stream)++ restore_mock.assert_awaited_once()+ proposal = workflow_params.update_workflow_copilot_chat.await_args.kwargs["proposed_workflow"]+ assert proposal["_copilot_yaml"] == "title: Last Good"+ assert "_copilot_unvalidated" not in proposal++ response_frames = [+ p for p in sent_payloads if getattr(p, "type", None) == WorkflowCopilotStreamMessageType.RESPONSE+ ]+ assert len(response_frames) == 1+ assert response_frames[0].proposal_disposition == "review_tested"+ assert response_frames[0].unvalidated is False+ assert response_frames[0].cancelled is False+ assert response_frames[0].force_review is True++# ---------------------------------------------------------------------------# /workflow/copilot/cancel endpoint# ---------------------------------------------------------------------------diff --git a/tests/unit/test_workflow_copilot_route_helpers.py b/tests/unit/test_workflow_copilot_route_helpers.pyindex b6404f959..156b7aaab 100644--- a/tests/unit/test_workflow_copilot_route_helpers.py+++ b/tests/unit/test_workflow_copilot_route_helpers.py@@ -19,6 +19,7 @@ from pydantic import ValidationErrorfrom skyvern.forge.sdk.routes.workflow_copilot import (_effective_auto_accept,_normalize_copilot_yaml,+ _proposal_disposition,_should_restore_persisted_workflow,)from skyvern.schemas.runs import ProxyLocation@@ -29,14 +30,19 @@ def _agent_result(persisted: bool,unvalidated: bool = False,cancelled: bool = False,+ force_review: bool = False,updated_workflow: Any = None,**kwargs: Any,) -> MagicMock:"""MagicMock with override flags explicitly set so a forgotten attr can't pass via MagicMock truthiness."""r = MagicMock()r.workflow_was_persisted = persisted+ r.proposal_disposition = (+ "review_untested" if unvalidated else "review_tested" if force_review else "auto_applicable"+ )r.unvalidated = unvalidatedr.cancelled = cancelled+ r.force_review = force_reviewr.updated_workflow = updated_workflowfor k, v in kwargs.items():setattr(r, k, v)@@ -70,12 +76,19 @@ class TestShouldRestorePersistedWorkflow:assert _should_restore_persisted_workflow(True, agent_result) is Trueassert _should_restore_persisted_workflow(False, agent_result) is True+ def test_force_review_wip_forces_rollback_under_auto_accept(self) -> None:+ agent_result = _agent_result(persisted=True, force_review=True, updated_workflow=MagicMock())++ assert _should_restore_persisted_workflow(True, agent_result) is True+ assert _should_restore_persisted_workflow(False, agent_result) is True+class TestEffectiveAutoAccept:def test_unvalidated_overrides_auto_accept(self) -> None:unvalidated = MagicMock()unvalidated.unvalidated = Trueunvalidated.cancelled = False+ unvalidated.force_review = Falseassert _effective_auto_accept(True, unvalidated) is Falseassert _effective_auto_accept(False, unvalidated) is False@@ -84,14 +97,41 @@ class TestEffectiveAutoAccept:cancelled = MagicMock()cancelled.unvalidated = Falsecancelled.cancelled = True+ cancelled.force_review = Falseassert _effective_auto_accept(True, cancelled) is Falseassert _effective_auto_accept(False, cancelled) is False+ def test_force_review_overrides_auto_accept(self) -> None:+ force_review = MagicMock()+ force_review.proposal_disposition = "review_tested"+ force_review.unvalidated = False+ force_review.cancelled = False+ force_review.force_review = True++ assert _effective_auto_accept(True, force_review) is False+ assert _effective_auto_accept(False, force_review) is False++ def test_review_untested_disposition_overrides_auto_accept(self) -> None:+ result = MagicMock()+ result.proposal_disposition = "review_untested"+ result.cancelled = False++ assert _effective_auto_accept(True, result) is False+ assert _effective_auto_accept(False, result) is False++ def test_legacy_flag_fallback_for_stacked_deploy(self) -> None:+ result = MagicMock(spec=["unvalidated", "force_review"])+ result.unvalidated = False+ result.force_review = True++ assert _proposal_disposition(result) == "review_tested"+def test_validated_proposal_respects_auto_accept_setting(self) -> None:validated = MagicMock()validated.unvalidated = Falsevalidated.cancelled = False+ validated.force_review = Falseassert _effective_auto_accept(True, validated) is Trueassert _effective_auto_accept(False, validated) is False