Skip to content

Automate browser based workflows with AI

fix(SKY-9935): decouple WIP-rescue force-review from unvalidated proposal flag (#6094)

Andrew Neilson4264933701d5
diff --git a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.test.ts b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.test.ts
new file mode 100644
index 000000000..675e1d38a
--- /dev/null
+++ b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.test.ts
@@ -0,0 +1,73 @@
+import { describe, expect, it } from "vitest";
+
+import { shouldAutoApplyWorkflowResponse } from "./proposalDisposition";
+import type { WorkflowCopilotStreamResponseUpdate } from "./workflowCopilotTypes";
+
+const response = (
+ overrides: Partial<WorkflowCopilotStreamResponseUpdate> = {},
+): WorkflowCopilotStreamResponseUpdate => ({
+ type: "response",
+ workflow_copilot_chat_id: "wcc_1",
+ message: "done",
+ updated_workflow: { workflow_id: "wf_1" } as never,
+ response_time: "2026-05-21T00:00:00Z",
+ proposal_disposition: "auto_applicable",
+ ...overrides,
+});
+
+describe("shouldAutoApplyWorkflowResponse", () => {
+ it("auto-applies auto_applicable proposals when auto accept is enabled", () => {
+ expect(shouldAutoApplyWorkflowResponse(response(), true, false)).toBe(true);
+ });
+
+ it.each(["review_untested", "review_tested"] as const)(
+ "forces explicit review for %s proposals",
+ (proposal_disposition) => {
+ expect(
+ shouldAutoApplyWorkflowResponse(
+ response({ proposal_disposition }),
+ true,
+ false,
+ ),
+ ).toBe(false);
+ },
+ );
+
+ it("uses the enum over legacy booleans when both are present", () => {
+ expect(
+ shouldAutoApplyWorkflowResponse(
+ response({
+ proposal_disposition: "auto_applicable",
+ unvalidated: true,
+ force_review: true,
+ }),
+ true,
+ false,
+ ),
+ ).toBe(true);
+ });
+
+ it("falls back to legacy review booleans before the enum ships", () => {
+ expect(
+ shouldAutoApplyWorkflowResponse(
+ response({
+ proposal_disposition: undefined,
+ force_review: true,
+ }),
+ true,
+ false,
+ ),
+ ).toBe(false);
+ });
+
+ it("does not auto-apply cancelled turns", () => {
+ expect(
+ shouldAutoApplyWorkflowResponse(
+ response({ cancelled: true }),
+ true,
+ false,
+ ),
+ ).toBe(false);
+ expect(shouldAutoApplyWorkflowResponse(response(), true, true)).toBe(false);
+ });
+});
diff --git a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx
index 2e20c71a3..ec11e563d 100644
--- a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx
+++ b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx
@@ -43,6 +43,7 @@ import {
shouldQueuePromptForLiveBrowser,
shouldWaitForLiveBrowser,
} from "./browserReadiness";
+import { shouldAutoApplyWorkflowResponse } from "./proposalDisposition";
interface ChatMessage {
id: string;
@@ -865,10 +866,11 @@ export function WorkflowCopilotChat({
cancelInFlightController.current === abortController;
if (
response.updated_workflow &&
- autoAccept &&
- !response.unvalidated &&
- !response.cancelled &&
- !userCancelledThisTurn
+ shouldAutoApplyWorkflowResponse(
+ response,
+ autoAccept,
+ userCancelledThisTurn,
+ )
) {
applyWorkflowUpdate(response.updated_workflow);
} else {
diff --git a/skyvern-frontend/src/routes/workflows/copilot/proposalDisposition.ts b/skyvern-frontend/src/routes/workflows/copilot/proposalDisposition.ts
new file mode 100644
index 000000000..dcab4bbbc
--- /dev/null
+++ b/skyvern-frontend/src/routes/workflows/copilot/proposalDisposition.ts
@@ -0,0 +1,19 @@
+import type { WorkflowCopilotStreamResponseUpdate } from "./workflowCopilotTypes";
+
+export const shouldAutoApplyWorkflowResponse = (
+ response: WorkflowCopilotStreamResponseUpdate,
+ autoAccept: boolean,
+ userCancelledThisTurn: boolean,
+) => {
+ const autoApplicable =
+ response.proposal_disposition !== undefined
+ ? response.proposal_disposition === "auto_applicable"
+ : !response.unvalidated && !response.force_review;
+ return Boolean(
+ response.updated_workflow &&
+ autoAccept &&
+ autoApplicable &&
+ !response.cancelled &&
+ !userCancelledThisTurn,
+ );
+};
diff --git a/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.ts b/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.ts
index 622e88027..d88c3e6bf 100644
--- a/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.ts
+++ b/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.ts
@@ -4,6 +4,10 @@ import {
} from "@/routes/workflows/types/workflowTypes";
export type WorkflowCopilotChatSender = "user" | "ai";
+export type ProposalDisposition =
+ | "auto_applicable"
+ | "review_untested"
+ | "review_tested";
export interface WorkflowCopilotChat {
workflow_copilot_chat_id: string;
@@ -83,10 +87,13 @@ export interface WorkflowCopilotStreamResponseUpdate {
message: string;
updated_workflow?: WorkflowApiResponse | null;
response_time: string;
- // Clients must NOT auto-apply when true; render Accept/Reject explicitly.
+ proposal_disposition?: ProposalDisposition;
+ // Legacy mirror for proposal_disposition === "review_untested".
unvalidated?: boolean;
- // Same auto-apply guard as unvalidated; cancel forces explicit review.
+ // Cancel forces explicit review.
cancelled?: boolean;
+ // Legacy mirror for proposal_disposition === "review_tested".
+ force_review?: boolean;
}
export interface WorkflowCopilotStreamErrorUpdate {
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.py
index 3bc6d74ad..4536c4694 100644
--- a/skyvern/forge/sdk/copilot/agent.py
+++ b/skyvern/forge/sdk/copilot/agent.py
@@ -745,10 +745,8 @@ def _build_wip_exit_result(
cancelled: bool = False,
) -> AgentResult:
"""Selected non-success exits surface the most recent successfully parsed workflow."""
- # When an unverified edit/run has overwritten ``last_workflow`` since the
- # last verified shape, prefer the verified shape. ``unvalidated=True``
- # triggers the route's rollback so auto-accept does not silently keep the
- # failed/in-flight shape.
+ # When an unverified edit/run has overwritten ``last_workflow``, prefer the
+ # verified shape while still forcing explicit review.
if (
ctx.last_good_workflow is not None
and ctx.last_good_workflow_yaml
@@ -762,7 +760,7 @@ def _build_wip_exit_result(
workflow_yaml=ctx.last_good_workflow_yaml,
workflow_was_persisted=ctx.workflow_persisted,
total_tokens=ctx.total_tokens_used,
- unvalidated=True,
+ proposal_disposition="review_tested",
cancelled=cancelled,
)
if (
@@ -780,7 +778,7 @@ def _build_wip_exit_result(
workflow_yaml=ctx.last_workflow_yaml,
workflow_was_persisted=ctx.workflow_persisted,
total_tokens=ctx.total_tokens_used,
- unvalidated=unvalidated,
+ proposal_disposition="review_untested" if unvalidated else "auto_applicable",
cancelled=cancelled,
)
return _build_exit_result(ctx, default_reply, global_llm_context, cancelled=cancelled)
@@ -1089,7 +1087,7 @@ def _translate_to_agent_result(
workflow_was_persisted=ctx.workflow_persisted,
total_tokens=ctx.total_tokens_used,
clear_proposed_workflow=resp_type == "ASK_QUESTION",
- unvalidated=unvalidated,
+ proposal_disposition="review_untested" if unvalidated else "auto_applicable",
output_policy_diagnostics=output_policy_diagnostics,
)
diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.py
index 1d1802992..380bda33d 100644
--- a/skyvern/forge/sdk/copilot/context.py
+++ b/skyvern/forge/sdk/copilot/context.py
@@ -13,6 +13,7 @@ from skyvern.forge.sdk.workflow.models.workflow import Workflow
ResponseType = Literal["REPLY", "ASK_QUESTION", "REPLACE_WORKFLOW"]
COPILOT_RESPONSE_TYPES: tuple[ResponseType, ...] = get_args(ResponseType)
+ProposalDisposition = Literal["auto_applicable", "review_untested", "review_tested"]
if TYPE_CHECKING:
from skyvern.forge.sdk.copilot.diagnosis_repair_contract import DiagnosisRepairContract
@@ -131,10 +132,18 @@ class AgentResult:
# persistence path (rollback + ``Cancelled by user.`` chat row) without
# losing ``workflow_was_persisted`` the way a re-raise would.
cancelled: bool = False
- # The route forces Accept/Reject regardless of ``auto_accept`` when this is True.
- unvalidated: bool = False
+ # Controls whether the route may auto-apply the proposal or must force explicit review.
+ proposal_disposition: ProposalDisposition = "auto_applicable"
output_policy_diagnostics: dict[str, Any] | None = None
+ @property
+ def unvalidated(self) -> bool:
+ return self.proposal_disposition == "review_untested"
+
+ @property
+ def force_review(self) -> bool:
+ return self.proposal_disposition == "review_tested"
+
@dataclass
class CopilotContext(AgentContext):
diff --git a/skyvern/forge/sdk/routes/workflow_copilot.py b/skyvern/forge/sdk/routes/workflow_copilot.py
index 67f712a02..8f144d55d 100644
--- a/skyvern/forge/sdk/routes/workflow_copilot.py
+++ b/skyvern/forge/sdk/routes/workflow_copilot.py
@@ -25,7 +25,7 @@ from skyvern.forge.sdk.copilot.agent import run_copilot_agent
from skyvern.forge.sdk.copilot.attribution import is_copilot_born_initial_write
from skyvern.forge.sdk.copilot.block_type_aliases import normalize_copilot_block_type_alias
from skyvern.forge.sdk.copilot.config import CopilotConfig
-from skyvern.forge.sdk.copilot.context import AgentResult
+from skyvern.forge.sdk.copilot.context import AgentResult, ProposalDisposition
from skyvern.forge.sdk.copilot.output_utils import truncate_output
from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.experimentation.llm_prompt_config import get_llm_handler_for_prompt_type
@@ -253,9 +253,28 @@ async def _ensure_terminal_frame(stream: EventSourceStream, already_emitted: boo
pass
+def _proposal_disposition(agent_result: object | None) -> ProposalDisposition:
+ disposition = getattr(agent_result, "proposal_disposition", None)
+ if disposition in ("auto_applicable", "review_untested", "review_tested"):
+ return disposition
+ if getattr(agent_result, "unvalidated", False) is True:
+ return "review_untested"
+ if getattr(agent_result, "force_review", False) is True:
+ return "review_tested"
+ return "auto_applicable"
+
+
+def _legacy_unvalidated(proposal_disposition: ProposalDisposition) -> bool:
+ return proposal_disposition == "review_untested"
+
+
+def _legacy_force_review(proposal_disposition: ProposalDisposition) -> bool:
+ return proposal_disposition == "review_tested"
+
+
def _effective_auto_accept(auto_accept: bool | None, agent_result: object | None) -> bool:
- """``unvalidated`` and ``cancelled`` override ``auto_accept=True`` to force explicit Accept/Reject."""
- if bool(getattr(agent_result, "unvalidated", False)) or bool(getattr(agent_result, "cancelled", False)):
+ """Only auto-applicable proposals may honor ``auto_accept=True``."""
+ if getattr(agent_result, "cancelled", False) is True or _proposal_disposition(agent_result) != "auto_applicable":
return False
return auto_accept is True
@@ -281,7 +300,7 @@ def _build_proposed_workflow_data(updated_workflow: Workflow, agent_result: Agen
proposed_data = dict(updated_workflow.model_dump(mode="json"))
if agent_result.workflow_yaml:
proposed_data["_copilot_yaml"] = agent_result.workflow_yaml
- if agent_result.unvalidated:
+ if _proposal_disposition(agent_result) == "review_untested":
proposed_data["_copilot_unvalidated"] = True
return proposed_data
@@ -331,7 +350,6 @@ async def _persist_cancel_turn(
updated_global_llm_context = None
total_tokens = None
response_type = "REPLY"
- unvalidated = False
output_policy_diagnostics = None
if chat.proposed_workflow is not None:
await asyncio.shield(_clear_proposed_workflow(chat))
@@ -348,7 +366,6 @@ async def _persist_cancel_turn(
updated_global_llm_context = agent_result.global_llm_context
total_tokens = getattr(agent_result, "total_tokens", None)
response_type = getattr(agent_result, "response_type", "REPLY")
- unvalidated = bool(getattr(agent_result, "unvalidated", False))
output_policy_diagnostics = getattr(agent_result, "output_policy_diagnostics", None)
await asyncio.shield(
@@ -368,6 +385,7 @@ async def _persist_cancel_turn(
global_llm_context=updated_global_llm_context,
)
)
+ proposal_disposition = _proposal_disposition(agent_result)
try:
await asyncio.shield(
stream.send(
@@ -379,9 +397,11 @@ async def _persist_cancel_turn(
response_time=assistant_message.created_at,
total_tokens=total_tokens,
response_type=response_type,
- unvalidated=unvalidated,
+ proposal_disposition=proposal_disposition,
+ unvalidated=_legacy_unvalidated(proposal_disposition),
cancelled=True,
output_policy_diagnostics=output_policy_diagnostics,
+ force_review=_legacy_force_review(proposal_disposition),
)
)
)
@@ -444,6 +464,7 @@ async def _finalise_normal_turn(
global_llm_context=updated_global_llm_context,
)
+ proposal_disposition = _proposal_disposition(agent_result)
await stream.send(
WorkflowCopilotStreamResponseUpdate(
type=WorkflowCopilotStreamMessageType.RESPONSE,
@@ -453,8 +474,10 @@ async def _finalise_normal_turn(
response_time=assistant_message.created_at,
total_tokens=getattr(agent_result, "total_tokens", None),
response_type=getattr(agent_result, "response_type", "REPLY"),
- unvalidated=bool(getattr(agent_result, "unvalidated", False)),
+ proposal_disposition=proposal_disposition,
+ unvalidated=_legacy_unvalidated(proposal_disposition),
output_policy_diagnostics=getattr(agent_result, "output_policy_diagnostics", None),
+ force_review=_legacy_force_review(proposal_disposition),
)
)
diff --git a/skyvern/forge/sdk/schemas/workflow_copilot.py b/skyvern/forge/sdk/schemas/workflow_copilot.py
index 37f38a37a..3e05711ee 100644
--- a/skyvern/forge/sdk/schemas/workflow_copilot.py
+++ b/skyvern/forge/sdk/schemas/workflow_copilot.py
@@ -4,7 +4,7 @@ from typing import Any
from pydantic import BaseModel, ConfigDict, Field
-from skyvern.forge.sdk.copilot.context import ResponseType
+from skyvern.forge.sdk.copilot.context import ProposalDisposition, ResponseType
class WorkflowCopilotChat(BaseModel):
@@ -118,9 +118,13 @@ class WorkflowCopilotStreamResponseUpdate(BaseModel):
description="Total tokens consumed by the agent during this turn; None when no provider reported usage",
)
response_type: ResponseType = Field("REPLY", description="Agent response classification")
+ proposal_disposition: ProposalDisposition = Field(
+ "auto_applicable",
+ description="Whether this proposal may auto-apply or must be reviewed explicitly.",
+ )
unvalidated: bool = Field(
False,
- description="When true, clients must not auto-apply; render Accept/Reject explicitly.",
+ description="Legacy mirror for proposal_disposition=review_untested.",
)
cancelled: bool = Field(
False,
@@ -130,6 +134,10 @@ class WorkflowCopilotStreamResponseUpdate(BaseModel):
None,
description="Diagnostic output-policy labels for raw-vs-final quality reporting.",
)
+ force_review: bool = Field(
+ False,
+ description="Legacy mirror for proposal_disposition=review_tested.",
+ )
class WorkflowCopilotStreamErrorUpdate(BaseModel):
diff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.py
index d84f8c122..9e30fdcdf 100644
--- a/tests/unit/test_copilot_agent_helpers.py
+++ b/tests/unit/test_copilot_agent_helpers.py
@@ -477,6 +477,7 @@ class TestShouldRestorePersistedWorkflow:
r.updated_workflow = updated_workflow
r.unvalidated = False
r.cancelled = False
+ r.force_review = False
return r
def test_restores_when_no_proposal_even_under_auto_accept(self) -> None:
diff --git a/tests/unit/test_copilot_mid_correction_guard.py b/tests/unit/test_copilot_mid_correction_guard.py
index f6296d9e4..e9965a084 100644
--- a/tests/unit/test_copilot_mid_correction_guard.py
+++ b/tests/unit/test_copilot_mid_correction_guard.py
@@ -134,7 +134,8 @@ def test_build_timeout_exit_result_surfaces_last_good_after_failure() -> None:
result = _build_timeout_exit_result(ctx, global_llm_context=None)
assert result.updated_workflow is ctx.last_good_workflow
assert result.workflow_yaml == "yaml-good"
- assert result.unvalidated is True
+ assert result.unvalidated is False
+ assert result.force_review is True
assert result.user_response == _TIMEOUT_REPLY_TESTED
@@ -150,7 +151,8 @@ def test_build_timeout_exit_result_surfaces_last_good_on_mid_flight_cancellation
ctx.last_good_workflow_yaml = "yaml-good"
result = _build_timeout_exit_result(ctx, global_llm_context=None)
assert result.updated_workflow is ctx.last_good_workflow
- assert result.unvalidated is True
+ assert result.unvalidated is False
+ assert result.force_review is True
def test_build_timeout_exit_result_falls_through_when_no_last_good() -> None:
diff --git a/tests/unit/test_copilot_timeout_wip.py b/tests/unit/test_copilot_timeout_wip.py
index 0da45fdce..ade6f32b2 100644
--- a/tests/unit/test_copilot_timeout_wip.py
+++ b/tests/unit/test_copilot_timeout_wip.py
@@ -321,3 +321,72 @@ class TestBuildCancelExitResult:
assert result.workflow_yaml is None
assert result.unvalidated is False
assert result.user_response == _CANCEL_REPLY_DEFAULT
+
+
+class TestWipExitSurfacesLastGoodWithForceReviewNotUnvalidated:
+ """Mid-flight overwrite branch offers ``last_good_workflow`` with ``force_review=True, unvalidated=False``."""
+
+ def _overwrite_ctx(self, *, last_test_ok: bool | None) -> MagicMock:
+ good = MagicMock(name="wf-good")
+ in_flight = MagicMock(name="wf-in-flight")
+ return _ctx(
+ last_workflow=in_flight,
+ last_workflow_yaml="version: in-flight",
+ last_test_ok=last_test_ok,
+ last_good_workflow=good,
+ last_good_workflow_yaml="version: good",
+ )
+
+ def test_cancel_with_overwrite_surfaces_last_good_as_tested_force_review(self) -> None:
+ ctx = self._overwrite_ctx(last_test_ok=None)
+
+ result = _build_cancel_exit_result(ctx, global_llm_context=None)
+
+ assert result.cancelled is True
+ assert result.updated_workflow is ctx.last_good_workflow
+ assert result.workflow_yaml == "version: good"
+ assert result.unvalidated is False
+ assert result.force_review is True
+ assert result.user_response == _CANCEL_REPLY_TESTED
+
+ def test_timeout_with_overwrite_surfaces_last_good_as_tested_force_review(self) -> None:
+ ctx = self._overwrite_ctx(last_test_ok=False)
+
+ result = _build_timeout_exit_result(ctx, global_llm_context=None)
+
+ assert result.updated_workflow is ctx.last_good_workflow
+ assert result.unvalidated is False
+ assert result.force_review is True
+ assert result.user_response == _TIMEOUT_REPLY_TESTED
+
+ def test_max_turns_with_overwrite_surfaces_last_good_as_tested_force_review(self) -> None:
+ ctx = self._overwrite_ctx(last_test_ok=None)
+
+ result = _build_max_turns_exit_result(ctx, global_llm_context=None)
+
+ assert result.updated_workflow is ctx.last_good_workflow
+ assert result.unvalidated is False
+ assert result.force_review is True
+ assert result.user_response == _MAX_TURNS_REPLY_TESTED
+
+ def test_unexpected_error_with_overwrite_surfaces_last_good_as_tested_force_review(self) -> None:
+ ctx = self._overwrite_ctx(last_test_ok=None)
+
+ result = _build_unexpected_error_exit_result(ctx, global_llm_context=None)
+
+ assert result.updated_workflow is ctx.last_good_workflow
+ assert result.unvalidated is False
+ assert result.force_review is True
+ assert result.user_response == _UNEXPECTED_ERROR_REPLY_TESTED
+
+ def test_cancelled_total_timeout_latch_uses_force_review_not_unvalidated(self) -> None:
+ ctx = self._overwrite_ctx(last_test_ok=None)
+ ctx.copilot_total_timeout_exceeded = True
+
+ result = _build_cancelled_exit_result(ctx, global_llm_context=None)
+
+ assert result.updated_workflow is ctx.last_good_workflow
+ assert result.unvalidated is False
+ assert result.force_review is True
+ assert result.cancelled is False
+ assert result.user_response == _TIMEOUT_REPLY_TESTED
diff --git a/tests/unit/test_workflow_copilot_cancel.py b/tests/unit/test_workflow_copilot_cancel.py
index c3b396f3f..a99e3222f 100644
--- a/tests/unit/test_workflow_copilot_cancel.py
+++ b/tests/unit/test_workflow_copilot_cancel.py
@@ -380,6 +380,7 @@ async def test_route_cancel_branch_persists_wip_proposal_and_response_frame(
frame = response_frames[0]
assert frame.message == "Cancelled. I have a draft workflow you can keep."
assert frame.updated_workflow == {"workflow_id": "wf-canonical", "title": "Draft"}
+ assert frame.proposal_disposition == "review_untested"
assert frame.unvalidated is True
assert frame.cancelled is True
@@ -430,6 +431,53 @@ async def test_route_cancel_tested_wip_with_auto_accept_still_persists_proposal(
assert response_frames[0].cancelled is True
+async def test_route_cancel_force_review_persists_proposal_without_unvalidated_marker(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """Cancel + WIP-rescue: persisted proposal must NOT carry the ``_copilot_unvalidated`` marker."""
+ chat = _make_chat(auto_accept=True)
+ original_workflow = _make_original_workflow()
+ updated_workflow = MagicMock()
+ updated_workflow.model_dump.return_value = {"workflow_id": "wf-canonical", "title": "Last Good Draft"}
+ agent_result = SimpleNamespace(
+ user_response="Cancelled. I have a tested draft for you. Accept it to save, or discard.",
+ updated_workflow=updated_workflow,
+ global_llm_context=None,
+ workflow_yaml="title: Last Good Draft",
+ workflow_was_persisted=True,
+ clear_proposed_workflow=False,
+ cancelled=True,
+ total_tokens=456,
+ response_type="REPLY",
+ unvalidated=False,
+ force_review=True,
+ )
+ restore_mock, workflow_params, sent_payloads = await _drive_cancel_route(
+ monkeypatch, chat, original_workflow, agent_result
+ )
+
+ restore_mock.assert_awaited_once()
+
+ workflow_params.update_workflow_copilot_chat.assert_awaited_once()
+ proposed_workflow = workflow_params.update_workflow_copilot_chat.await_args.kwargs["proposed_workflow"]
+ assert proposed_workflow == {
+ "workflow_id": "wf-canonical",
+ "title": "Last Good Draft",
+ "_copilot_yaml": "title: Last Good Draft",
+ }
+ assert "_copilot_unvalidated" not in proposed_workflow
+
+ response_frames = [
+ p for p in sent_payloads if getattr(p, "type", None) == WorkflowCopilotStreamMessageType.RESPONSE
+ ]
+ assert len(response_frames) == 1
+ assert response_frames[0].proposal_disposition == "review_tested"
+ assert response_frames[0].unvalidated is False
+ assert response_frames[0].cancelled is True
+ assert response_frames[0].force_review is True
+
+
@pytest.mark.asyncio
async def test_route_cancel_clears_stale_proposed_workflow_when_no_wip(
monkeypatch: pytest.MonkeyPatch,
@@ -592,6 +640,7 @@ async def test_timeout_wip_result_streams_normal_response_frame(
]
assert len(response_frames) == 1
assert response_frames[0].updated_workflow == {"workflow_id": "wf-draft", "title": "Draft"}
+ assert response_frames[0].proposal_disposition == "review_untested"
assert response_frames[0].unvalidated is True
assert response_frames[0].total_tokens == 123
@@ -601,6 +650,80 @@ async def test_timeout_wip_result_streams_normal_response_frame(
assert "Cancelled by user." not in contents
+async def test_timeout_wip_force_review_propagates_to_response_frame(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """Non-cancel WIP rescue propagates ``force_review`` to the SSE frame so the frontend skips auto-apply."""
+ monkeypatch.setattr(settings, "ENABLE_WORKFLOW_COPILOT_V2", True)
+ captured = _install_fake_create(monkeypatch)
+
+ chat = SimpleNamespace(
+ workflow_copilot_chat_id="chat-1",
+ workflow_permanent_id="wpid-1",
+ organization_id="org-1",
+ proposed_workflow=None,
+ auto_accept=True,
+ )
+ original_workflow = SimpleNamespace(
+ workflow_id="wf-canonical",
+ title="Original",
+ description="Original description",
+ workflow_definition=None,
+ )
+ updated_workflow = MagicMock()
+ updated_workflow.model_dump.side_effect = lambda mode="json": {"workflow_id": "wf-good", "title": "Last Good"}
+ agent_result = SimpleNamespace(
+ user_response="I ran out of time, but I have a tested draft for you. Accept it to save, or discard.",
+ updated_workflow=updated_workflow,
+ global_llm_context=None,
+ workflow_yaml="title: Last Good",
+ workflow_was_persisted=True,
+ clear_proposed_workflow=False,
+ cancelled=False,
+ total_tokens=789,
+ response_type="REPLY",
+ unvalidated=False,
+ force_review=True,
+ )
+ restore_mock, workflow_params = _setup_route_mocks(monkeypatch, chat, original_workflow, agent_result)
+
+ request = MagicMock()
+ request.headers = {"x-api-key": "sk-test"}
+ organization = SimpleNamespace(organization_id="org-1")
+
+ response = await workflow_copilot_chat_post(request, _make_chat_request(), organization)
+ assert response is captured["sentinel"]
+
+ sent_payloads: list[Any] = []
+ stream = MagicMock()
+
+ async def _send(payload: Any) -> bool:
+ sent_payloads.append(payload)
+ return True
+
+ stream.send = _send
+ stream.is_disconnected = AsyncMock(return_value=False)
+
+ handler = captured["handler"]
+ assert callable(handler)
+ await handler(stream)
+
+ restore_mock.assert_awaited_once()
+ proposal = workflow_params.update_workflow_copilot_chat.await_args.kwargs["proposed_workflow"]
+ assert proposal["_copilot_yaml"] == "title: Last Good"
+ assert "_copilot_unvalidated" not in proposal
+
+ response_frames = [
+ p for p in sent_payloads if getattr(p, "type", None) == WorkflowCopilotStreamMessageType.RESPONSE
+ ]
+ assert len(response_frames) == 1
+ assert response_frames[0].proposal_disposition == "review_tested"
+ assert response_frames[0].unvalidated is False
+ assert response_frames[0].cancelled is False
+ assert response_frames[0].force_review is True
+
+
# ---------------------------------------------------------------------------
# /workflow/copilot/cancel endpoint
# ---------------------------------------------------------------------------
diff --git a/tests/unit/test_workflow_copilot_route_helpers.py b/tests/unit/test_workflow_copilot_route_helpers.py
index b6404f959..156b7aaab 100644
--- a/tests/unit/test_workflow_copilot_route_helpers.py
+++ b/tests/unit/test_workflow_copilot_route_helpers.py
@@ -19,6 +19,7 @@ from pydantic import ValidationError
from skyvern.forge.sdk.routes.workflow_copilot import (
_effective_auto_accept,
_normalize_copilot_yaml,
+ _proposal_disposition,
_should_restore_persisted_workflow,
)
from skyvern.schemas.runs import ProxyLocation
@@ -29,14 +30,19 @@ def _agent_result(
persisted: bool,
unvalidated: bool = False,
cancelled: bool = False,
+ force_review: bool = False,
updated_workflow: Any = None,
**kwargs: Any,
) -> MagicMock:
"""MagicMock with override flags explicitly set so a forgotten attr can't pass via MagicMock truthiness."""
r = MagicMock()
r.workflow_was_persisted = persisted
+ r.proposal_disposition = (
+ "review_untested" if unvalidated else "review_tested" if force_review else "auto_applicable"
+ )
r.unvalidated = unvalidated
r.cancelled = cancelled
+ r.force_review = force_review
r.updated_workflow = updated_workflow
for k, v in kwargs.items():
setattr(r, k, v)
@@ -70,12 +76,19 @@ class TestShouldRestorePersistedWorkflow:
assert _should_restore_persisted_workflow(True, agent_result) is True
assert _should_restore_persisted_workflow(False, agent_result) is True
+ def test_force_review_wip_forces_rollback_under_auto_accept(self) -> None:
+ agent_result = _agent_result(persisted=True, force_review=True, updated_workflow=MagicMock())
+
+ assert _should_restore_persisted_workflow(True, agent_result) is True
+ assert _should_restore_persisted_workflow(False, agent_result) is True
+
class TestEffectiveAutoAccept:
def test_unvalidated_overrides_auto_accept(self) -> None:
unvalidated = MagicMock()
unvalidated.unvalidated = True
unvalidated.cancelled = False
+ unvalidated.force_review = False
assert _effective_auto_accept(True, unvalidated) is False
assert _effective_auto_accept(False, unvalidated) is False
@@ -84,14 +97,41 @@ class TestEffectiveAutoAccept:
cancelled = MagicMock()
cancelled.unvalidated = False
cancelled.cancelled = True
+ cancelled.force_review = False
assert _effective_auto_accept(True, cancelled) is False
assert _effective_auto_accept(False, cancelled) is False
+ def test_force_review_overrides_auto_accept(self) -> None:
+ force_review = MagicMock()
+ force_review.proposal_disposition = "review_tested"
+ force_review.unvalidated = False
+ force_review.cancelled = False
+ force_review.force_review = True
+
+ assert _effective_auto_accept(True, force_review) is False
+ assert _effective_auto_accept(False, force_review) is False
+
+ def test_review_untested_disposition_overrides_auto_accept(self) -> None:
+ result = MagicMock()
+ result.proposal_disposition = "review_untested"
+ result.cancelled = False
+
+ assert _effective_auto_accept(True, result) is False
+ assert _effective_auto_accept(False, result) is False
+
+ def test_legacy_flag_fallback_for_stacked_deploy(self) -> None:
+ result = MagicMock(spec=["unvalidated", "force_review"])
+ result.unvalidated = False
+ result.force_review = True
+
+ assert _proposal_disposition(result) == "review_tested"
+
def test_validated_proposal_respects_auto_accept_setting(self) -> None:
validated = MagicMock()
validated.unvalidated = False
validated.cancelled = False
+ validated.force_review = False
assert _effective_auto_accept(True, validated) is True
assert _effective_auto_accept(False, validated) is False