Skip to content

Automate browser based workflows with AI

fix(SKY-9143): never propose a known-broken workflow from copilot (#5574)

Andrew Neilsonbe31095e2f68
diff --git a/pyproject.toml b/pyproject.toml
index 8a9239a6f..023bf6368 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -222,6 +222,9 @@ exclude = [
"alembic/env.py",
"cloud/extensions",
"skyvern-frontend",
+ "skyvern/client",
+ "notebooks",
+ "internal",
]
line-length = 120
target-version = "py311"
diff --git a/skyvern/forge/prompts/skyvern/workflow-copilot-agent.j2 b/skyvern/forge/prompts/skyvern/workflow-copilot-agent.j2
index 1190f6c9e..cec6bae4f 100644
--- a/skyvern/forge/prompts/skyvern/workflow-copilot-agent.j2
+++ b/skyvern/forge/prompts/skyvern/workflow-copilot-agent.j2
@@ -82,6 +82,7 @@ When a run fails with "Access Denied", an Akamai/Cloudflare/Imperva reference, o
7. If still blocked, explain to the user that the site has anti-bot protection that requires proxy configuration, and describe exactly what you tried.
MANDATORY: After every workflow update, you MUST test it. Prefer update_and_run_blocks (single call) over separate update_workflow + run_blocks_and_collect_debug calls. Never skip testing. Grow the workflow incrementally (known-good blocks + next block) unless the user explicitly asks for a one-shot full draft. Feasibility decisions must be based on the latest test result. If you conclude the request is infeasible, explain the specific evidence from that latest run.
+MANDATORY: The Accept/Reject proposal UI only appears when your latest run_blocks_and_collect_debug or update_and_run_blocks returned ok=true this turn. An untested update_workflow is silently dropped from the user-visible proposal — if you want the user to be able to accept your change, you MUST verify it with a passing test in this same turn. If testing is impossible for any reason (missing credentials, required human input), stop and ask a clarifying question instead of leaving an untested draft.
MANDATORY: If the latest run only validates an intermediate step (for example, "reached homepage"), do NOT present that as the final proposal when the user asked for a larger outcome (for example, finding/downloading specific files). Continue internal iteration.
MANDATORY: For multi-step requests (for example, "go to X and then download Y"), do not stop after validating only the first step. The proposed workflow must cover all requested steps.
MANDATORY: Do NOT respond to the user after validating only a subset of the requested outcome. If the user asked for N distinct actions (e.g., "navigate AND download", "login AND search AND add to cart"), your workflow must have blocks covering ALL of those actions before you present a final response. A workflow that only navigates to a page is NOT a complete response for "go to X and download Y".
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.py
index 5bf271ee5..4e3cbe841 100644
--- a/skyvern/forge/sdk/copilot/agent.py
+++ b/skyvern/forge/sdk/copilot/agent.py
@@ -109,18 +109,66 @@ def _normalize_failure_reason(failure_reason: str | None) -> str:
return normalized or "The workflow test run failed."
+_FAILURE_FOLLOW_UP = {
+ "NAVIGATION_FAILURE": " Can you confirm the URL is correct?",
+ "PROXY_ERROR": " Want me to retry with a different proxy location?",
+ "PAGE_LOAD_TIMEOUT": " Can you confirm the URL and try again in a moment?",
+ "ANTI_BOT_DETECTION": " Want me to retry with a different proxy location?",
+ "AUTH_FAILURE": " The site rejected the login — is the stored password still valid?",
+ "CREDENTIAL_ERROR": " I couldn't find a credential to use — can you link one in Settings?",
+}
+
+
def _rewrite_failed_test_response(user_response: str, ctx: CopilotContext) -> str:
- if ctx.last_update_block_count is None or ctx.last_test_ok is not False:
- return user_response
+ # Reshape replies when we cannot ship a proposal this turn so the user
+ # sees why nothing is being offered rather than an un-grounded claim.
+ if ctx.last_test_ok is False and ctx.last_update_block_count is not None:
+ if ctx.last_update_block_count <= 0:
+ draft_phrase = "a draft workflow"
+ else:
+ block_word = "block" if ctx.last_update_block_count == 1 else "blocks"
+ draft_phrase = f"a draft workflow with {ctx.last_update_block_count} {block_word}"
+
+ failure_summary = _normalize_failure_reason(ctx.last_test_failure_reason)
+ follow_up = _FAILURE_FOLLOW_UP.get(ctx.last_failure_category_top or "", "")
+ return f"I created {draft_phrase} and tested it, but the test failed. Failure: {failure_summary}.{follow_up}"
+
+ if ctx.last_test_ok is None and ctx.last_update_block_count is not None and ctx.last_workflow is not None:
+ # Agent edited the YAML but didn't verify it this turn; don't promise
+ # a re-run we can't durably execute (the restore helper rolls the
+ # mid-turn DB write back and there's no durable draft to re-test).
+ return (
+ "I drafted an update but wasn't able to verify it this turn. "
+ "Could you share more context about what you'd like me to do?"
+ )
+
+ return user_response
+
+
+def _verified_workflow_or_none(ctx: CopilotContext) -> tuple[Any, str | None]:
+ """Only surface a workflow proposal when it passed a test this turn.
+
+ SKY-9143: the Accept/Reject UI must never reflect a workflow we haven't
+ proven works. Every agent exit path that builds an AgentResult directly
+ (cancel, max-turns, timeout, non-retriable nav error, catch-all Exception)
+ funnels through this so the strict invariant holds regardless of which
+ branch the run took.
+ """
+ if ctx.last_workflow is not None and ctx.last_test_ok is True:
+ return ctx.last_workflow, ctx.last_workflow_yaml
+ return None, None
- if ctx.last_update_block_count <= 0:
- draft_phrase = "a draft workflow"
- else:
- block_word = "block" if ctx.last_update_block_count == 1 else "blocks"
- draft_phrase = f"a draft workflow with {ctx.last_update_block_count} {block_word}"
- failure_summary = _normalize_failure_reason(ctx.last_test_failure_reason)
- return f"I created {draft_phrase} and tested it, but the test failed. Failure: {failure_summary}"
+def _build_exit_result(ctx: CopilotContext, user_response: str, global_llm_context: str | None) -> AgentResult:
+ """AgentResult for agent-loop exits that don't go through ``_translate_to_agent_result``."""
+ verified_workflow, verified_yaml = _verified_workflow_or_none(ctx)
+ return AgentResult(
+ user_response=user_response,
+ updated_workflow=verified_workflow,
+ global_llm_context=global_llm_context,
+ workflow_yaml=verified_yaml,
+ workflow_was_persisted=ctx.workflow_persisted,
+ )
def _translate_to_agent_result(
@@ -161,11 +209,17 @@ def _translate_to_agent_result(
f"Please ask me to fix it.)"
)
- user_response = _rewrite_failed_test_response(str(user_response), ctx)
+ # Inline REPLACE_WORKFLOW bypasses _update_workflow so ctx.last_workflow
+ # is still whatever the tool layer last saw. Writing the REPLACE_WORKFLOW
+ # result onto ctx ensures _verified_workflow_or_none sees the right
+ # candidate — though it still gates on last_test_ok, so an untested
+ # REPLACE is rejected too.
+ if action_data.get("type") == "REPLACE_WORKFLOW" and last_workflow is not ctx.last_workflow:
+ ctx.last_workflow = last_workflow
+ ctx.last_workflow_yaml = last_workflow_yaml
- if ctx.last_test_ok is False:
- last_workflow = None
- last_workflow_yaml = None
+ user_response = _rewrite_failed_test_response(str(user_response), ctx)
+ last_workflow, last_workflow_yaml = _verified_workflow_or_none(ctx)
resp_type = action_data.get("type", "REPLY")
if resp_type not in ("REPLY", "ASK_QUESTION", "REPLACE_WORKFLOW"):
@@ -394,28 +448,18 @@ async def run_copilot_agent(
)
except asyncio.CancelledError:
LOG.info("Copilot run cancelled")
- return AgentResult(
- user_response="Request cancelled.",
- updated_workflow=ctx.last_workflow,
- global_llm_context=global_llm_context,
- workflow_yaml=ctx.last_workflow_yaml,
- workflow_was_persisted=ctx.workflow_persisted,
- )
+ return _build_exit_result(ctx, "Request cancelled.", global_llm_context)
except MaxTurnsExceeded:
- return AgentResult(
- user_response="I've reached the maximum number of steps. Here's what I have so far.",
- updated_workflow=ctx.last_workflow,
- global_llm_context=global_llm_context,
- workflow_yaml=ctx.last_workflow_yaml,
- workflow_was_persisted=ctx.workflow_persisted,
+ return _build_exit_result(
+ ctx,
+ "I've reached the maximum number of steps. Here's what I have so far.",
+ global_llm_context,
)
except CopilotTotalTimeoutError:
- return AgentResult(
- user_response="I ran out of time processing your request. Here's what I have so far.",
- updated_workflow=ctx.last_workflow,
- global_llm_context=global_llm_context,
- workflow_yaml=ctx.last_workflow_yaml,
- workflow_was_persisted=ctx.workflow_persisted,
+ return _build_exit_result(
+ ctx,
+ "I ran out of time processing your request. Here's what I have so far.",
+ global_llm_context,
)
except CopilotNonRetriableNavError as exc:
LOG.warning(
@@ -424,25 +468,21 @@ async def run_copilot_agent(
error_message=exc.error_message,
organization_id=organization_id,
)
+ # Non-retriable nav errors prove the current workflow doesn't
+ # work; zero the proposal even if other tools succeeded.
return AgentResult(
user_response=(
f"The target URL could not be reached. Error: {exc.error_message}. "
"Please verify the URL and try again."
),
- updated_workflow=ctx.last_workflow,
+ updated_workflow=None,
global_llm_context=global_llm_context,
- workflow_yaml=ctx.last_workflow_yaml,
+ workflow_yaml=None,
workflow_was_persisted=ctx.workflow_persisted,
)
except Exception as e:
LOG.error("Copilot agent error", error=str(e), exc_info=True)
- return AgentResult(
- user_response="An unexpected error occurred. Please try again.",
- updated_workflow=ctx.last_workflow,
- global_llm_context=global_llm_context,
- workflow_yaml=ctx.last_workflow_yaml,
- workflow_was_persisted=ctx.workflow_persisted,
- )
+ return _build_exit_result(ctx, "An unexpected error occurred. Please try again.", global_llm_context)
finally:
_copilot_model_name.reset(model_token)
session.close()
diff --git a/skyvern/forge/sdk/routes/workflow_copilot.py b/skyvern/forge/sdk/routes/workflow_copilot.py
index 0657fa0c7..601105f34 100644
--- a/skyvern/forge/sdk/routes/workflow_copilot.py
+++ b/skyvern/forge/sdk/routes/workflow_copilot.py
@@ -83,8 +83,19 @@ class BlockRunInfo:
def _should_restore_persisted_workflow(auto_accept: bool | None, agent_result: object | None) -> bool:
- """Return True when a persisted draft should be rolled back."""
- return auto_accept is not True and bool(getattr(agent_result, "workflow_was_persisted", False))
+ """Return True when a persisted draft should be rolled back.
+
+ SKY-9143: when the agent decided not to ship a proposal this turn
+ (``updated_workflow is None``) but ``_update_workflow`` already committed
+ a YAML to ``workflow_definition``, we must restore the original even under
+ ``auto_accept=True`` — otherwise an unverified edit becomes the live
+ workflow silently.
+ """
+ if not bool(getattr(agent_result, "workflow_was_persisted", False)):
+ return False
+ if getattr(agent_result, "updated_workflow", None) is None:
+ return True
+ return auto_accept is not True
async def _restore_workflow_definition(original_workflow: Workflow | None, organization_id: str) -> None:
@@ -840,9 +851,17 @@ async def _new_copilot_chat_post(
# connected: the user needs to see the reply on reconnect.
# SKY-8986: client disconnect used to short-circuit this block
# and leave the chat history without the AI response.
+ #
+ # SKY-9143: restore runs outside the auto_accept wrapper so
+ # auto-accept turns that ended without a viable proposal still
+ # roll back a mid-turn _update_workflow write. The Accept/Reject
+ # panel state below stays gated on auto_accept — the frontend
+ # applies proposals via applyWorkflowUpdate when auto-accept is
+ # on.
+ if _should_restore_persisted_workflow(chat.auto_accept, agent_result):
+ await _restore_workflow_definition(original_workflow, organization.organization_id)
+
if chat.auto_accept is not True:
- if _should_restore_persisted_workflow(chat.auto_accept, agent_result):
- await _restore_workflow_definition(original_workflow, organization.organization_id)
if updated_workflow:
proposed_data = updated_workflow.model_dump(mode="json")
if agent_result.workflow_yaml:
diff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.py
index e5ef30d7f..3b6df82e0 100644
--- a/tests/unit/test_copilot_agent_helpers.py
+++ b/tests/unit/test_copilot_agent_helpers.py
@@ -5,18 +5,26 @@ from __future__ import annotations
from unittest.mock import MagicMock
+def _ctx(**overrides):
+ from skyvern.forge.sdk.copilot.context import CopilotContext
+
+ defaults = dict(
+ organization_id="org-1",
+ workflow_id="wf-1",
+ workflow_permanent_id="wfp-1",
+ workflow_yaml="",
+ browser_session_id=None,
+ stream=MagicMock(),
+ )
+ defaults.update(overrides)
+ return CopilotContext(**defaults)
+
+
class TestFailedTestResponseNormalization:
def test_rewrite_failed_test_response_avoids_success_language(self) -> None:
from skyvern.forge.sdk.copilot.agent import _rewrite_failed_test_response
- from skyvern.forge.sdk.copilot.context import CopilotContext
-
- ctx = CopilotContext(
- organization_id="org-1",
- workflow_id="wf-1",
- workflow_permanent_id="wfp-1",
- workflow_yaml="",
- browser_session_id=None,
- stream=MagicMock(),
+
+ ctx = _ctx(
last_update_block_count=2,
last_test_ok=False,
last_test_failure_reason=(
@@ -58,3 +66,126 @@ class TestFailedTestResponseNormalization:
assert ctx.last_workflow is sentinel_workflow
assert ctx.last_test_ok is False
assert ctx.last_test_failure_reason == "net::ERR_NAME_NOT_RESOLVED"
+
+ def test_rewrite_includes_navigation_follow_up_when_category_matches(self) -> None:
+ from skyvern.forge.sdk.copilot.agent import _rewrite_failed_test_response
+
+ ctx = _ctx(
+ last_update_block_count=1,
+ last_test_ok=False,
+ last_test_failure_reason="Failed to navigate to url https://bad.example.",
+ last_failure_category_top="NAVIGATION_FAILURE",
+ )
+ rewritten = _rewrite_failed_test_response("done", ctx)
+
+ assert "test failed" in rewritten.lower()
+ assert "confirm the url" in rewritten.lower()
+
+ def test_rewrite_untested_edit_asks_for_more_context(self) -> None:
+ # SKY-9143 row 3: agent updated YAML without testing. The reply must
+ # not promise the backend can re-run a durable draft — the restore
+ # helper rolled it back and there is nothing to re-test next turn.
+ from skyvern.forge.sdk.copilot.agent import _rewrite_failed_test_response
+
+ sentinel_workflow = object()
+ ctx = _ctx(
+ last_update_block_count=1,
+ last_test_ok=None,
+ last_workflow=sentinel_workflow,
+ )
+ rewritten = _rewrite_failed_test_response("Here's the updated YAML.", ctx)
+
+ assert "drafted an update" in rewritten.lower()
+ assert "run it" not in rewritten.lower()
+ assert "more context" in rewritten.lower() or "clarify" in rewritten.lower()
+
+ def test_rewrite_passes_through_when_no_update_or_failure(self) -> None:
+ from skyvern.forge.sdk.copilot.agent import _rewrite_failed_test_response
+
+ ctx = _ctx()
+ original = "Let me know what you want to build."
+ assert _rewrite_failed_test_response(original, ctx) == original
+
+
+class TestVerifiedWorkflowOrNone:
+ """SKY-9143 strict invariant: a proposal surfaces only after a passing test this turn."""
+
+ def _wf(self) -> object:
+ return object()
+
+ def test_passes_workflow_when_tested_successfully(self) -> None:
+ from skyvern.forge.sdk.copilot.agent import _verified_workflow_or_none
+
+ wf = self._wf()
+ ctx = _ctx(last_workflow=wf, last_workflow_yaml="foo: bar", last_test_ok=True)
+ assert _verified_workflow_or_none(ctx) == (wf, "foo: bar")
+
+ def test_zeros_when_test_failed(self) -> None:
+ from skyvern.forge.sdk.copilot.agent import _verified_workflow_or_none
+
+ ctx = _ctx(last_workflow=self._wf(), last_workflow_yaml="foo: bar", last_test_ok=False)
+ assert _verified_workflow_or_none(ctx) == (None, None)
+
+ def test_zeros_when_untested_update(self) -> None:
+ # Exactly the scenario where _record_workflow_update_result reset
+ # last_test_ok to None after a standalone update_workflow or after
+ # the agent edited post-failure without re-testing.
+ from skyvern.forge.sdk.copilot.agent import _verified_workflow_or_none
+
+ ctx = _ctx(last_workflow=self._wf(), last_workflow_yaml="foo: bar", last_test_ok=None)
+ assert _verified_workflow_or_none(ctx) == (None, None)
+
+ def test_zeros_when_no_last_workflow(self) -> None:
+ from skyvern.forge.sdk.copilot.agent import _verified_workflow_or_none
+
+ ctx = _ctx(last_workflow=None, last_test_ok=True)
+ assert _verified_workflow_or_none(ctx) == (None, None)
+
+ def test_zeros_on_suspicious_success(self) -> None:
+ # _record_run_blocks_result sets last_test_ok=None when blocks ran ok
+ # but produced no meaningful extraction data. Still an unverified
+ # outcome; must not surface a proposal.
+ from skyvern.forge.sdk.copilot.agent import _verified_workflow_or_none
+
+ ctx = _ctx(
+ last_workflow=self._wf(),
+ last_workflow_yaml="foo: bar",
+ last_test_ok=None,
+ last_test_suspicious_success=True,
+ )
+ assert _verified_workflow_or_none(ctx) == (None, None)
+
+
+class TestShouldRestorePersistedWorkflow:
+ """SKY-9143: auto_accept=True must still restore when no proposal shipped."""
+
+ def _result(self, *, persisted: bool, updated_workflow: object | None):
+ r = MagicMock()
+ r.workflow_was_persisted = persisted
+ r.updated_workflow = updated_workflow
+ return r
+
+ def test_restores_when_no_proposal_even_under_auto_accept(self) -> None:
+ from skyvern.forge.sdk.routes.workflow_copilot import _should_restore_persisted_workflow
+
+ r = self._result(persisted=True, updated_workflow=None)
+ assert _should_restore_persisted_workflow(True, r) is True
+
+ def test_keeps_persisted_write_under_auto_accept_when_proposal_valid(self) -> None:
+ from skyvern.forge.sdk.routes.workflow_copilot import _should_restore_persisted_workflow
+
+ r = self._result(persisted=True, updated_workflow=object())
+ assert _should_restore_persisted_workflow(True, r) is False
+
+ def test_restores_when_not_auto_accept_and_persisted(self) -> None:
+ from skyvern.forge.sdk.routes.workflow_copilot import _should_restore_persisted_workflow
+
+ r = self._result(persisted=True, updated_workflow=object())
+ assert _should_restore_persisted_workflow(False, r) is True
+
+ def test_noop_when_nothing_was_persisted(self) -> None:
+ from skyvern.forge.sdk.routes.workflow_copilot import _should_restore_persisted_workflow
+
+ r = self._result(persisted=False, updated_workflow=None)
+ assert _should_restore_persisted_workflow(True, r) is False
+ assert _should_restore_persisted_workflow(False, r) is False
diff --git a/tests/unit/test_workflow_copilot_route.py b/tests/unit/test_workflow_copilot_route.py
index 7d3ba35f7..c6db45a6d 100644
--- a/tests/unit/test_workflow_copilot_route.py
+++ b/tests/unit/test_workflow_copilot_route.py
@@ -179,17 +179,26 @@ def _setup_new_copilot_mocks(
@pytest.mark.asyncio
@pytest.mark.parametrize(
- ("auto_accept", "workflow_was_persisted", "expect_restore"),
+ ("auto_accept", "workflow_was_persisted", "has_valid_proposal", "expect_restore"),
[
- (True, True, False), # auto_accept True => no restore
- (False, False, False), # nothing persisted => nothing to restore
- (False, True, True), # mid-stream disconnect with a persisted draft => restore
+ # auto_accept + valid proposal => keep the DB write (frontend applied it)
+ (True, True, True, False),
+ # auto_accept + no proposal (SKY-9143) => restore the unverified mid-turn write
+ (True, True, False, True),
+ # Nothing was persisted => nothing to restore regardless of other flags
+ (False, False, False, False),
+ # Normal mid-stream disconnect with a persisted draft and no proposal => restore
+ (False, True, False, True),
+ # Normal mid-stream disconnect with a persisted draft and a valid proposal =>
+ # still restore, user accepts via the panel to re-apply
+ (False, True, True, True),
],
)
async def test_flag_on_mid_stream_disconnect_restores_when_persisted_and_not_auto_accept(
monkeypatch: pytest.MonkeyPatch,
auto_accept: bool,
workflow_was_persisted: bool,
+ has_valid_proposal: bool,
expect_restore: bool,
) -> None:
monkeypatch.setattr(settings, "ENABLE_WORKFLOW_COPILOT_V2", True)
@@ -209,9 +218,12 @@ async def test_flag_on_mid_stream_disconnect_restores_when_persisted_and_not_aut
description="Original description",
workflow_definition=None,
)
+ proposal = MagicMock(spec=["model_dump"]) if has_valid_proposal else None
+ if proposal is not None:
+ proposal.model_dump.return_value = {"workflow_id": "wf-canonical"}
agent_result = SimpleNamespace(
user_response="done",
- updated_workflow=None,
+ updated_workflow=proposal,
global_llm_context=None,
workflow_yaml=None,
workflow_was_persisted=workflow_was_persisted,