Skip to content

Automate browser based workflows with AI

feat(SKY-9354): surface WIP workflow on copilot timeout / failed test (#5755)

Andrew Neilson4e1c46f7cf36
diff --git a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx
index 451622284..ec18dde28 100644
--- a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx
+++ b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx
@@ -738,7 +738,7 @@ export function WorkflowCopilotChat({
};
setMessages((prev) => [...prev, aiMessage]);
- if (response.updated_workflow && autoAccept) {
+ if (response.updated_workflow && autoAccept && !response.unvalidated) {
applyWorkflowUpdate(response.updated_workflow);
} else {
setProposedWorkflow(response.updated_workflow ?? null);
diff --git a/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.ts b/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.ts
index b3abca5b2..095ef3931 100644
--- a/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.ts
+++ b/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.ts
@@ -83,6 +83,8 @@ export interface WorkflowCopilotStreamResponseUpdate {
message: string;
updated_workflow?: WorkflowApiResponse | null;
response_time: string;
+ // Clients must NOT auto-apply when true; render Accept/Reject explicitly.
+ unvalidated?: boolean;
}
export interface WorkflowCopilotStreamErrorUpdate {
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.py
index 6d564c7d8..438b2222c 100644
--- a/skyvern/forge/sdk/copilot/agent.py
+++ b/skyvern/forge/sdk/copilot/agent.py
@@ -186,8 +186,9 @@ _FAILURE_FOLLOW_UP = {
def _rewrite_failed_test_response(user_response: str, ctx: CopilotContext) -> str:
- # Reshape replies when we cannot ship a proposal this turn so the user
- # sees why nothing is being offered rather than an un-grounded claim.
+ has_keepable_draft = ctx.last_workflow is not None and bool(ctx.last_workflow_yaml)
+ keep_draft_affordance = " Keep the draft to iterate on, or discard." if has_keepable_draft else ""
+
if ctx.last_test_ok is False and ctx.last_update_block_count is not None:
if ctx.last_update_block_count <= 0:
draft_phrase = "a draft workflow"
@@ -197,12 +198,17 @@ def _rewrite_failed_test_response(user_response: str, ctx: CopilotContext) -> st
failure_summary = _normalize_failure_reason(ctx.last_test_failure_reason)
follow_up = _FAILURE_FOLLOW_UP.get(ctx.last_failure_category_top or "", "")
- return f"I created {draft_phrase} and tested it, but the test failed. Failure: {failure_summary}.{follow_up}"
+ return (
+ f"I created {draft_phrase} and tested it, but the test failed. "
+ f"Failure: {failure_summary}.{follow_up}{keep_draft_affordance}"
+ )
if ctx.last_test_ok is None and ctx.last_update_block_count is not None and ctx.last_workflow is not None:
- # Agent edited the YAML but didn't verify it this turn; don't promise
- # a re-run we can't durably execute (the restore helper rolls the
- # mid-turn DB write back and there's no durable draft to re-test).
+ if has_keepable_draft:
+ return (
+ "I drafted an update but wasn't able to verify it this turn. "
+ "Keep the draft to iterate on it manually, or discard."
+ )
return (
"I drafted an update but wasn't able to verify it this turn. "
"Could you share more context about what you'd like me to do?"
@@ -212,15 +218,8 @@ def _rewrite_failed_test_response(user_response: str, ctx: CopilotContext) -> st
def _verified_workflow_or_none(ctx: CopilotContext) -> tuple[Any, str | None]:
- """Only surface a workflow proposal when it passed a test this turn.
-
- SKY-9143: the Accept/Reject UI must never reflect a workflow we haven't
- proven works. Every agent exit path that builds an AgentResult directly
- (cancel, max-turns, timeout, non-retriable nav error, catch-all Exception)
- funnels through this so the strict invariant holds regardless of which
- branch the run took.
- """
- if ctx.last_workflow is not None and ctx.last_test_ok is True:
+ """Surface a proposal only when it passed a test this turn AND yaml is on hand."""
+ if ctx.last_workflow is not None and ctx.last_workflow_yaml and ctx.last_test_ok is True:
return ctx.last_workflow, ctx.last_workflow_yaml
return None, None
@@ -244,6 +243,76 @@ def _build_exit_result(
)
+_TIMEOUT_REPLY_DEFAULT = "I ran out of time processing your request. Here's what I have so far."
+_TIMEOUT_REPLY_UNVALIDATED = (
+ "I ran out of time before I could finish testing. I have a draft workflow you can keep — "
+ "accept it to save (note: it hasn't been verified end-to-end), or discard."
+)
+_TIMEOUT_REPLY_TESTED = "I ran out of time, but I have a tested draft for you. Accept it to save, or discard."
+
+_MAX_TURNS_REPLY_DEFAULT = "I've reached the maximum number of steps. Here's what I have so far."
+_MAX_TURNS_REPLY_UNVALIDATED = (
+ "I've reached the maximum number of steps before I could finish testing. I have a draft "
+ "workflow you can keep — accept it to save (note: it hasn't been verified end-to-end), or discard."
+)
+_MAX_TURNS_REPLY_TESTED = (
+ "I've reached the maximum number of steps, but I have a tested draft for you. Accept it to save, or discard."
+)
+
+
+def _build_capacity_exit_result(
+ ctx: CopilotContext,
+ global_llm_context: str | None,
+ *,
+ default_reply: str,
+ unvalidated_reply: str,
+ tested_reply: str,
+) -> AgentResult:
+ """Capacity-exhausted exits (timeout, max-turns) surface the most recent
+ successfully-parsed workflow rather than discard the WIP."""
+ # ``last_test_ok=None`` covers both "test never ran" and "test ran with
+ # ambiguous output"; only the first case earns the carve-out (the REPLY
+ # path is more permissive because its reply text carries the context).
+ if (
+ ctx.last_workflow is not None
+ and ctx.last_workflow_yaml
+ and ctx.last_test_ok is not False
+ and not ctx.last_test_suspicious_success
+ ):
+ unvalidated = ctx.last_test_ok is not True
+ reply = unvalidated_reply if unvalidated else tested_reply
+ return AgentResult(
+ user_response=reply,
+ updated_workflow=ctx.last_workflow,
+ global_llm_context=global_llm_context,
+ workflow_yaml=ctx.last_workflow_yaml,
+ workflow_was_persisted=ctx.workflow_persisted,
+ total_tokens=ctx.total_tokens_used,
+ unvalidated=unvalidated,
+ )
+ return _build_exit_result(ctx, default_reply, global_llm_context)
+
+
+def _build_timeout_exit_result(ctx: CopilotContext, global_llm_context: str | None) -> AgentResult:
+ return _build_capacity_exit_result(
+ ctx,
+ global_llm_context,
+ default_reply=_TIMEOUT_REPLY_DEFAULT,
+ unvalidated_reply=_TIMEOUT_REPLY_UNVALIDATED,
+ tested_reply=_TIMEOUT_REPLY_TESTED,
+ )
+
+
+def _build_max_turns_exit_result(ctx: CopilotContext, global_llm_context: str | None) -> AgentResult:
+ return _build_capacity_exit_result(
+ ctx,
+ global_llm_context,
+ default_reply=_MAX_TURNS_REPLY_DEFAULT,
+ unvalidated_reply=_MAX_TURNS_REPLY_UNVALIDATED,
+ tested_reply=_MAX_TURNS_REPLY_TESTED,
+ )
+
+
def _translate_to_agent_result(
result: RunResultStreaming,
ctx: CopilotContext,
@@ -325,6 +394,12 @@ def _translate_to_agent_result(
if resp_type != "ASK_QUESTION":
user_response = _rewrite_failed_test_response(str(user_response), ctx)
last_workflow, last_workflow_yaml = _verified_workflow_or_none(ctx)
+ unvalidated = False
+ if last_workflow is None and resp_type == "REPLY" and ctx.last_workflow is not None and ctx.last_workflow_yaml:
+ # Failures are often environmental (captcha, transient block); surface the draft so the user can keep iterating.
+ last_workflow = ctx.last_workflow
+ last_workflow_yaml = ctx.last_workflow_yaml
+ unvalidated = True
# ASK_QUESTION blocks on user input — never surface a verified workflow
# under it; auto_accept would silently apply a stepping-stone partial.
@@ -353,6 +428,7 @@ def _translate_to_agent_result(
workflow_was_persisted=ctx.workflow_persisted,
total_tokens=ctx.total_tokens_used,
clear_proposed_workflow=resp_type == "ASK_QUESTION",
+ unvalidated=unvalidated,
)
@@ -564,17 +640,9 @@ async def run_copilot_agent(
LOG.info("Copilot run cancelled by user")
return _build_exit_result(ctx, "Cancelled by user.", global_llm_context, cancelled=True)
except MaxTurnsExceeded:
- return _build_exit_result(
- ctx,
- "I've reached the maximum number of steps. Here's what I have so far.",
- global_llm_context,
- )
+ return _build_max_turns_exit_result(ctx, global_llm_context)
except CopilotTotalTimeoutError:
- return _build_exit_result(
- ctx,
- "I ran out of time processing your request. Here's what I have so far.",
- global_llm_context,
- )
+ return _build_timeout_exit_result(ctx, global_llm_context)
except CopilotNonRetriableNavError as exc:
LOG.warning(
"Copilot run halted on non-retriable navigation error",
diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.py
index c0c4fc247..2b815a5b7 100644
--- a/skyvern/forge/sdk/copilot/context.py
+++ b/skyvern/forge/sdk/copilot/context.py
@@ -127,6 +127,8 @@ class AgentResult:
# persistence path (rollback + ``Cancelled by user.`` chat row) without
# losing ``workflow_was_persisted`` the way a re-raise would.
cancelled: bool = False
+ # The route forces Accept/Reject regardless of ``auto_accept`` when this is True.
+ unvalidated: bool = False
@dataclass
diff --git a/skyvern/forge/sdk/routes/workflow_copilot.py b/skyvern/forge/sdk/routes/workflow_copilot.py
index dcedb7951..e646db053 100644
--- a/skyvern/forge/sdk/routes/workflow_copilot.py
+++ b/skyvern/forge/sdk/routes/workflow_copilot.py
@@ -170,20 +170,28 @@ async def _ensure_terminal_frame(stream: EventSourceStream, already_emitted: boo
pass
-def _should_restore_persisted_workflow(auto_accept: bool | None, agent_result: object | None) -> bool:
- """Return True when a persisted draft should be rolled back.
+def _effective_auto_accept(auto_accept: bool | None, agent_result: object | None) -> bool:
+ """``unvalidated`` overrides ``auto_accept=True`` to force explicit Accept/Reject."""
+ if bool(getattr(agent_result, "unvalidated", False)):
+ return False
+ return auto_accept is True
- SKY-9143: when the agent decided not to ship a proposal this turn
- (``updated_workflow is None``) but ``_update_workflow`` already committed
- a YAML to ``workflow_definition``, we must restore the original even under
- ``auto_accept=True`` — otherwise an unverified edit becomes the live
- workflow silently.
- """
+
+def _should_restore_persisted_workflow(auto_accept: bool | None, agent_result: object | None) -> bool:
+ """Restore when a mid-turn ``_update_workflow`` commit isn't covered by an accepted proposal."""
if not bool(getattr(agent_result, "workflow_was_persisted", False)):
return False
if getattr(agent_result, "updated_workflow", None) is None:
return True
- return auto_accept is not True
+ return not _effective_auto_accept(auto_accept, agent_result)
+
+
+async def _clear_proposed_workflow(chat: Any) -> None:
+ await app.DATABASE.workflow_params.update_workflow_copilot_chat(
+ organization_id=chat.organization_id,
+ workflow_copilot_chat_id=chat.workflow_copilot_chat_id,
+ proposed_workflow=None,
+ )
async def _persist_cancel_turn(
@@ -263,10 +271,13 @@ async def _finalise_normal_turn(
if restored:
await _restore_workflow_definition(original_workflow, organization_id)
- if chat.auto_accept is not True and updated_workflow:
+ auto_accept_effective = _effective_auto_accept(chat.auto_accept, agent_result)
+ if not auto_accept_effective and updated_workflow:
proposed_data = updated_workflow.model_dump(mode="json")
if agent_result.workflow_yaml:
proposed_data["_copilot_yaml"] = agent_result.workflow_yaml
+ if agent_result.unvalidated:
+ proposed_data["_copilot_unvalidated"] = True
await app.DATABASE.workflow_params.update_workflow_copilot_chat(
organization_id=chat.organization_id,
workflow_copilot_chat_id=chat.workflow_copilot_chat_id,
@@ -277,11 +288,15 @@ async def _finalise_normal_turn(
# so a reload does not resurrect a stale Accept/Reject card. Runs
# under both auto_accept values — a stale proposal can survive an
# auto-accept toggle.
- await app.DATABASE.workflow_params.update_workflow_copilot_chat(
- organization_id=chat.organization_id,
- workflow_copilot_chat_id=chat.workflow_copilot_chat_id,
- proposed_workflow=None,
- )
+ await _clear_proposed_workflow(chat)
+ elif (
+ auto_accept_effective
+ and chat.proposed_workflow is not None
+ and chat.proposed_workflow.get("_copilot_unvalidated") is True
+ ):
+ # The leftover unvalidated proposal is no longer attached to the chat
+ # tail; clear it so reload doesn't resurrect a stale Accept/Reject card.
+ await _clear_proposed_workflow(chat)
await app.DATABASE.workflow_params.create_workflow_copilot_chat_message(
organization_id=chat.organization_id,
@@ -307,6 +322,7 @@ async def _finalise_normal_turn(
response_time=assistant_message.created_at,
total_tokens=getattr(agent_result, "total_tokens", None),
response_type=getattr(agent_result, "response_type", "REPLY"),
+ unvalidated=bool(getattr(agent_result, "unvalidated", False)),
)
)
diff --git a/skyvern/forge/sdk/schemas/workflow_copilot.py b/skyvern/forge/sdk/schemas/workflow_copilot.py
index 89abd067b..9f91020a8 100644
--- a/skyvern/forge/sdk/schemas/workflow_copilot.py
+++ b/skyvern/forge/sdk/schemas/workflow_copilot.py
@@ -117,6 +117,10 @@ class WorkflowCopilotStreamResponseUpdate(BaseModel):
description="Total tokens consumed by the agent during this turn; None when no provider reported usage",
)
response_type: ResponseType = Field("REPLY", description="Agent response classification")
+ unvalidated: bool = Field(
+ False,
+ description="When true, clients must not auto-apply; render Accept/Reject explicitly.",
+ )
class WorkflowCopilotStreamErrorUpdate(BaseModel):
diff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.py
index eea08109d..2d30aac4d 100644
--- a/tests/unit/test_copilot_agent_helpers.py
+++ b/tests/unit/test_copilot_agent_helpers.py
@@ -110,6 +110,21 @@ class TestFailedTestResponseNormalization:
original = "Let me know what you want to build."
assert _rewrite_failed_test_response(original, ctx) == original
+ def test_rewrite_appends_keep_draft_affordance_when_draft_on_hand(self) -> None:
+ from skyvern.forge.sdk.copilot.agent import _rewrite_failed_test_response
+
+ ctx = _ctx(
+ last_workflow=object(),
+ last_workflow_yaml="title: drafted",
+ last_update_block_count=2,
+ last_test_ok=False,
+ last_test_failure_reason="A verification challenge is preventing submission.",
+ )
+ rewritten = _rewrite_failed_test_response("done", ctx)
+
+ assert "test failed" in rewritten.lower()
+ assert "keep the draft" in rewritten.lower()
+
class TestVerifiedWorkflowOrNone:
"""SKY-9143 strict invariant: a proposal surfaces only after a passing test this turn."""
@@ -167,6 +182,7 @@ class TestShouldRestorePersistedWorkflow:
r = MagicMock()
r.workflow_was_persisted = persisted
r.updated_workflow = updated_workflow
+ r.unvalidated = False
return r
def test_restores_when_no_proposal_even_under_auto_accept(self) -> None:
@@ -269,10 +285,10 @@ class TestTranslateToAgentResultGating:
assert "validation error" in agent_result.user_response.lower()
def test_ask_question_preserves_model_specific_question(self) -> None:
- # The new prompt instructs the model to stop and ASK_QUESTION when it
- # cannot test an edit. Row-3 of _rewrite_failed_test_response would
- # clobber that specific unblocker with "Could you share more context";
- # the resp_type==ASK_QUESTION guard must skip the rewrite.
+ # The rewrite guard for ASK_QUESTION must hold: the agent's specific
+ # clarifying question is not clobbered by the generic "share more
+ # context" rewrite. SKY-9420 also drops any workflow under
+ # ASK_QUESTION so an auto-accept user can't silently apply a partial.
ctx = _ctx(
last_update_block_count=1,
last_test_ok=None,
@@ -286,14 +302,11 @@ class TestTranslateToAgentResultGating:
)
assert agent_result.user_response == specific_question
- # Even ASK_QUESTION must obey the strict gate — no verified workflow this turn.
assert agent_result.updated_workflow is None
+ assert agent_result.unvalidated is False
assert agent_result.response_type == "ASK_QUESTION"
def test_reply_still_rewrites_after_failed_test(self) -> None:
- # Guard rail for the above: a plain REPLY after a failed test must
- # still flow through the "test failed" rewrite so we don't regress
- # the original SKY-9143 behavior.
ctx = _ctx(
last_update_block_count=2,
last_test_ok=False,
@@ -308,6 +321,44 @@ class TestTranslateToAgentResultGating:
assert "test failed" in agent_result.user_response.lower()
assert "All done" not in agent_result.user_response
assert agent_result.updated_workflow is None
+ assert agent_result.unvalidated is False
+
+ def test_reply_after_failed_test_surfaces_unvalidated_wip_when_draft_on_hand(self) -> None:
+ wf = SimpleNamespace(name="drafted")
+ ctx = _ctx(
+ last_workflow=wf,
+ last_workflow_yaml="title: drafted",
+ last_update_block_count=4,
+ last_test_ok=False,
+ last_test_failure_reason="A verification challenge is preventing submission.",
+ )
+ result = _fake_run_result({"type": "REPLY", "user_response": "Done."})
+ agent_result = agent_module._translate_to_agent_result(
+ result, ctx, global_llm_context=None, chat_request=_chat_request(), organization_id="org-1"
+ )
+
+ assert agent_result.updated_workflow is wf
+ assert agent_result.workflow_yaml == "title: drafted"
+ assert agent_result.unvalidated is True
+ assert "test failed" in agent_result.user_response.lower()
+ assert "keep the draft" in agent_result.user_response.lower()
+
+ def test_reply_after_suspicious_success_surfaces_unvalidated_wip(self) -> None:
+ wf = SimpleNamespace(name="drafted")
+ ctx = _ctx(
+ last_workflow=wf,
+ last_workflow_yaml="title: drafted",
+ last_update_block_count=2,
+ last_test_ok=None,
+ last_test_suspicious_success=True,
+ )
+ result = _fake_run_result({"type": "REPLY", "user_response": "Done."})
+ agent_result = agent_module._translate_to_agent_result(
+ result, ctx, global_llm_context=None, chat_request=_chat_request(), organization_id="org-1"
+ )
+
+ assert agent_result.updated_workflow is wf
+ assert agent_result.unvalidated is True
def test_inline_replace_workflow_wraps_block_goals_with_user_message(self, monkeypatch) -> None:
# SKY-9174 parity: update_and_run_blocks_tool wraps block goals with
diff --git a/tests/unit/test_copilot_timeout_wip.py b/tests/unit/test_copilot_timeout_wip.py
new file mode 100644
index 000000000..ad8005d4a
--- /dev/null
+++ b/tests/unit/test_copilot_timeout_wip.py
@@ -0,0 +1,172 @@
+"""Tests for the capacity-exhausted WIP carve-outs (timeout, max-turns)."""
+
+from __future__ import annotations
+
+from unittest.mock import MagicMock
+
+from skyvern.forge.sdk.copilot.agent import (
+ _MAX_TURNS_REPLY_DEFAULT,
+ _MAX_TURNS_REPLY_TESTED,
+ _MAX_TURNS_REPLY_UNVALIDATED,
+ _TIMEOUT_REPLY_DEFAULT,
+ _TIMEOUT_REPLY_TESTED,
+ _TIMEOUT_REPLY_UNVALIDATED,
+ _build_max_turns_exit_result,
+ _build_timeout_exit_result,
+)
+
+
+def _ctx(
+ *,
+ last_workflow: object | None,
+ last_workflow_yaml: str | None,
+ last_test_ok: bool | None,
+ last_test_suspicious_success: bool = False,
+) -> MagicMock:
+ ctx = MagicMock()
+ ctx.last_workflow = last_workflow
+ ctx.last_workflow_yaml = last_workflow_yaml
+ ctx.last_test_ok = last_test_ok
+ ctx.last_test_suspicious_success = last_test_suspicious_success
+ ctx.workflow_persisted = last_workflow is not None
+ ctx.total_tokens_used = None
+ return ctx
+
+
+class TestBuildTimeoutExitResult:
+ def test_no_workflow_falls_back_to_default_reply(self) -> None:
+ ctx = _ctx(last_workflow=None, last_workflow_yaml=None, last_test_ok=None)
+
+ result = _build_timeout_exit_result(ctx, global_llm_context=None)
+
+ assert result.updated_workflow is None
+ assert result.workflow_yaml is None
+ assert result.unvalidated is False
+ assert result.user_response == _TIMEOUT_REPLY_DEFAULT
+
+ def test_untested_workflow_surfaces_as_unvalidated_wip(self) -> None:
+ wf = MagicMock(name="wf")
+ ctx = _ctx(last_workflow=wf, last_workflow_yaml="version: '1.0'", last_test_ok=None)
+
+ result = _build_timeout_exit_result(ctx, global_llm_context=None)
+
+ assert result.updated_workflow is wf
+ assert result.workflow_yaml == "version: '1.0'"
+ assert result.unvalidated is True
+ assert result.user_response == _TIMEOUT_REPLY_UNVALIDATED
+
+ def test_failed_test_drops_proposal(self) -> None:
+ wf = MagicMock(name="wf")
+ ctx = _ctx(last_workflow=wf, last_workflow_yaml="version: '1.0'", last_test_ok=False)
+
+ result = _build_timeout_exit_result(ctx, global_llm_context=None)
+
+ assert result.updated_workflow is None
+ assert result.workflow_yaml is None
+ assert result.unvalidated is False
+ assert result.user_response == _TIMEOUT_REPLY_DEFAULT
+
+ def test_passing_test_surfaces_as_tested_proposal(self) -> None:
+ wf = MagicMock(name="wf")
+ ctx = _ctx(last_workflow=wf, last_workflow_yaml="version: '1.0'", last_test_ok=True)
+
+ result = _build_timeout_exit_result(ctx, global_llm_context=None)
+
+ assert result.updated_workflow is wf
+ assert result.workflow_yaml == "version: '1.0'"
+ assert result.unvalidated is False
+ assert result.user_response == _TIMEOUT_REPLY_TESTED
+
+ def test_missing_yaml_drops_untested_proposal(self) -> None:
+ wf = MagicMock(name="wf")
+ ctx = _ctx(last_workflow=wf, last_workflow_yaml=None, last_test_ok=None)
+
+ result = _build_timeout_exit_result(ctx, global_llm_context=None)
+
+ assert result.updated_workflow is None
+ assert result.workflow_yaml is None
+ assert result.unvalidated is False
+ assert result.user_response == _TIMEOUT_REPLY_DEFAULT
+
+ def test_missing_yaml_drops_tested_proposal(self) -> None:
+ wf = MagicMock(name="wf")
+ ctx = _ctx(last_workflow=wf, last_workflow_yaml=None, last_test_ok=True)
+
+ result = _build_timeout_exit_result(ctx, global_llm_context=None)
+
+ assert result.updated_workflow is None
+ assert result.workflow_yaml is None
+ assert result.unvalidated is False
+ assert result.user_response == _TIMEOUT_REPLY_DEFAULT
+
+ def test_suspicious_success_drops_proposal(self) -> None:
+ wf = MagicMock(name="wf")
+ ctx = _ctx(
+ last_workflow=wf,
+ last_workflow_yaml="version: '1.0'",
+ last_test_ok=None,
+ last_test_suspicious_success=True,
+ )
+
+ result = _build_timeout_exit_result(ctx, global_llm_context=None)
+
+ assert result.updated_workflow is None
+ assert result.workflow_yaml is None
+ assert result.unvalidated is False
+ assert result.user_response == _TIMEOUT_REPLY_DEFAULT
+
+
+class TestBuildMaxTurnsExitResult:
+ def test_no_workflow_falls_back_to_default_reply(self) -> None:
+ ctx = _ctx(last_workflow=None, last_workflow_yaml=None, last_test_ok=None)
+
+ result = _build_max_turns_exit_result(ctx, global_llm_context=None)
+
+ assert result.updated_workflow is None
+ assert result.unvalidated is False
+ assert result.user_response == _MAX_TURNS_REPLY_DEFAULT
+
+ def test_untested_workflow_surfaces_as_unvalidated_wip(self) -> None:
+ wf = MagicMock(name="wf")
+ ctx = _ctx(last_workflow=wf, last_workflow_yaml="version: '1.0'", last_test_ok=None)
+
+ result = _build_max_turns_exit_result(ctx, global_llm_context=None)
+
+ assert result.updated_workflow is wf
+ assert result.unvalidated is True
+ assert result.user_response == _MAX_TURNS_REPLY_UNVALIDATED
+
+ def test_passing_test_surfaces_as_tested_proposal(self) -> None:
+ wf = MagicMock(name="wf")
+ ctx = _ctx(last_workflow=wf, last_workflow_yaml="version: '1.0'", last_test_ok=True)
+
+ result = _build_max_turns_exit_result(ctx, global_llm_context=None)
+
+ assert result.updated_workflow is wf
+ assert result.unvalidated is False
+ assert result.user_response == _MAX_TURNS_REPLY_TESTED
+
+ def test_failed_test_drops_proposal(self) -> None:
+ wf = MagicMock(name="wf")
+ ctx = _ctx(last_workflow=wf, last_workflow_yaml="version: '1.0'", last_test_ok=False)
+
+ result = _build_max_turns_exit_result(ctx, global_llm_context=None)
+
+ assert result.updated_workflow is None
+ assert result.unvalidated is False
+ assert result.user_response == _MAX_TURNS_REPLY_DEFAULT
+
+ def test_suspicious_success_drops_proposal(self) -> None:
+ wf = MagicMock(name="wf")
+ ctx = _ctx(
+ last_workflow=wf,
+ last_workflow_yaml="version: '1.0'",
+ last_test_ok=None,
+ last_test_suspicious_success=True,
+ )
+
+ result = _build_max_turns_exit_result(ctx, global_llm_context=None)
+
+ assert result.updated_workflow is None
+ assert result.unvalidated is False
+ assert result.user_response == _MAX_TURNS_REPLY_DEFAULT
diff --git a/tests/unit/test_workflow_copilot_route.py b/tests/unit/test_workflow_copilot_route.py
index a330a6974..1b2f8e95e 100644
--- a/tests/unit/test_workflow_copilot_route.py
+++ b/tests/unit/test_workflow_copilot_route.py
@@ -228,6 +228,7 @@ async def test_flag_on_mid_stream_disconnect_restores_when_persisted_and_not_aut
workflow_yaml=None,
workflow_was_persisted=workflow_was_persisted,
clear_proposed_workflow=False,
+ unvalidated=False,
)
restore_mock = _setup_new_copilot_mocks(monkeypatch, chat, original_workflow, agent_result)
@@ -290,6 +291,10 @@ async def test_flag_on_mid_stream_disconnect_restores_when_persisted_and_not_aut
# No prior proposal => no DB write even when the clear flag is set.
(False, False, False, None, True, False),
(True, False, False, None, True, False),
+ # auto_accept=True turn with a stale UNVALIDATED proposal clears it
+ # via the third elif (no clear flag, no restore needed).
+ (True, False, False, {"workflow_id": "stale", "_copilot_unvalidated": True}, False, True),
+ (True, True, True, {"workflow_id": "stale", "_copilot_unvalidated": True}, False, True),
],
)
async def test_proposed_workflow_cleared_on_restore(
@@ -328,6 +333,7 @@ async def test_proposed_workflow_cleared_on_restore(
workflow_yaml=None,
workflow_was_persisted=workflow_was_persisted,
clear_proposed_workflow=clear_proposed_flag,
+ unvalidated=False,
)
_setup_new_copilot_mocks(monkeypatch, chat, original_workflow, agent_result)
@@ -367,6 +373,79 @@ async def test_proposed_workflow_cleared_on_restore(
assert response_frames[0].updated_workflow == expected_payload_workflow
+async def test_unvalidated_timeout_wip_overrides_auto_accept(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr(settings, "ENABLE_WORKFLOW_COPILOT_V2", True)
+
+ captured = _install_fake_create(monkeypatch)
+
+ chat = SimpleNamespace(
+ workflow_copilot_chat_id="chat-1",
+ workflow_permanent_id="wpid-1",
+ organization_id="org-1",
+ proposed_workflow=None,
+ auto_accept=True,
+ )
+ original_workflow = SimpleNamespace(
+ workflow_id="wf-canonical",
+ title="Original",
+ description="Original description",
+ workflow_definition=None,
+ )
+ proposal = MagicMock(spec=["model_dump"])
+ proposal.model_dump.return_value = {"workflow_id": "wf-canonical"}
+ agent_result = SimpleNamespace(
+ user_response="I ran out of time before I could finish testing.",
+ updated_workflow=proposal,
+ global_llm_context=None,
+ workflow_yaml="title: WIP",
+ workflow_was_persisted=True,
+ clear_proposed_workflow=False,
+ unvalidated=True,
+ total_tokens=42,
+ response_type="REPLY",
+ )
+
+ restore_mock = _setup_new_copilot_mocks(monkeypatch, chat, original_workflow, agent_result)
+
+ request = MagicMock()
+ request.headers = {"x-api-key": "sk-test-key"}
+ organization = SimpleNamespace(organization_id="org-1")
+
+ response = await workflow_copilot_chat_post(request, _make_chat_request(), organization)
+ assert response is captured["sentinel"]
+
+ sent_frames: list[object] = []
+ stream = MagicMock()
+
+ async def capture_send(payload: object) -> bool:
+ sent_frames.append(payload)
+ return True
+
+ stream.send = capture_send
+ stream.is_disconnected = AsyncMock(return_value=False)
+
+ handler = captured["handler"]
+ assert callable(handler)
+ await handler(stream)
+
+ restore_mock.assert_awaited_once()
+
+ update_calls = app.DATABASE.workflow_params.update_workflow_copilot_chat.await_args_list
+ proposal_writes = [c for c in update_calls if c.kwargs.get("proposed_workflow") is not None]
+ assert len(proposal_writes) == 1
+ proposed_data = proposal_writes[0].kwargs["proposed_workflow"]
+ assert proposed_data.get("_copilot_unvalidated") is True
+ assert proposed_data.get("_copilot_yaml") == "title: WIP"
+
+ response_frame = next(
+ (f for f in sent_frames if getattr(f, "type", None) and str(f.type).endswith("response")),
+ None,
+ )
+ assert response_frame is not None
+ assert getattr(response_frame, "unvalidated", False) is True
+
+
@pytest.mark.asyncio
async def test_env_on_short_circuits_posthog(monkeypatch: pytest.MonkeyPatch) -> None:
"""Env var True must skip the PostHog check entirely and route to v2."""
diff --git a/tests/unit/test_workflow_copilot_route_helpers.py b/tests/unit/test_workflow_copilot_route_helpers.py
index fa160e6da..e9efd1f5f 100644
--- a/tests/unit/test_workflow_copilot_route_helpers.py
+++ b/tests/unit/test_workflow_copilot_route_helpers.py
@@ -12,7 +12,11 @@ from __future__ import annotations
from unittest.mock import MagicMock
-from skyvern.forge.sdk.routes.workflow_copilot import _normalize_copilot_yaml, _should_restore_persisted_workflow
+from skyvern.forge.sdk.routes.workflow_copilot import (
+ _effective_auto_accept,
+ _normalize_copilot_yaml,
+ _should_restore_persisted_workflow,
+)
class TestShouldRestorePersistedWorkflow:
@@ -26,6 +30,7 @@ class TestShouldRestorePersistedWorkflow:
def test_does_not_restore_for_auto_accept_or_unpersisted_result(self) -> None:
persisted = MagicMock()
persisted.workflow_was_persisted = True
+ persisted.unvalidated = False
not_persisted = MagicMock()
not_persisted.workflow_was_persisted = False
@@ -33,6 +38,36 @@ class TestShouldRestorePersistedWorkflow:
assert _should_restore_persisted_workflow(False, not_persisted) is False
assert _should_restore_persisted_workflow(False, None) is False
+ def test_unvalidated_timeout_wip_forces_rollback_under_auto_accept(self) -> None:
+ agent_result = MagicMock()
+ agent_result.workflow_was_persisted = True
+ agent_result.unvalidated = True
+ agent_result.updated_workflow = MagicMock()
+
+ assert _should_restore_persisted_workflow(True, agent_result) is True
+ assert _should_restore_persisted_workflow(False, agent_result) is True
+
+
+class TestEffectiveAutoAccept:
+ def test_unvalidated_overrides_auto_accept(self) -> None:
+ unvalidated = MagicMock()
+ unvalidated.unvalidated = True
+
+ assert _effective_auto_accept(True, unvalidated) is False
+ assert _effective_auto_accept(False, unvalidated) is False
+
+ def test_validated_proposal_respects_auto_accept_setting(self) -> None:
+ validated = MagicMock()
+ validated.unvalidated = False
+
+ assert _effective_auto_accept(True, validated) is True
+ assert _effective_auto_accept(False, validated) is False
+ assert _effective_auto_accept(None, validated) is False
+
+ def test_no_agent_result_falls_back_to_user_setting(self) -> None:
+ assert _effective_auto_accept(True, None) is True
+ assert _effective_auto_accept(False, None) is False
+
class TestNormalizeCopilotYamlTitleCoercion:
def test_missing_top_level_title_is_coerced_to_empty(self) -> None: