Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix(SKY-9143): never propose a known-broken workflow from copilot (#5574)
Andrew Neilsonbe31095e2f68
diff --git a/pyproject.toml b/pyproject.tomlindex 8a9239a6f..023bf6368 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -222,6 +222,9 @@ exclude = ["alembic/env.py","cloud/extensions","skyvern-frontend",+ "skyvern/client",+ "notebooks",+ "internal",]line-length = 120target-version = "py311"diff --git a/skyvern/forge/prompts/skyvern/workflow-copilot-agent.j2 b/skyvern/forge/prompts/skyvern/workflow-copilot-agent.j2index 1190f6c9e..cec6bae4f 100644--- a/skyvern/forge/prompts/skyvern/workflow-copilot-agent.j2+++ b/skyvern/forge/prompts/skyvern/workflow-copilot-agent.j2@@ -82,6 +82,7 @@ When a run fails with "Access Denied", an Akamai/Cloudflare/Imperva reference, o7. If still blocked, explain to the user that the site has anti-bot protection that requires proxy configuration, and describe exactly what you tried.MANDATORY: After every workflow update, you MUST test it. Prefer update_and_run_blocks (single call) over separate update_workflow + run_blocks_and_collect_debug calls. Never skip testing. Grow the workflow incrementally (known-good blocks + next block) unless the user explicitly asks for a one-shot full draft. Feasibility decisions must be based on the latest test result. If you conclude the request is infeasible, explain the specific evidence from that latest run.+MANDATORY: The Accept/Reject proposal UI only appears when your latest run_blocks_and_collect_debug or update_and_run_blocks returned ok=true this turn. An untested update_workflow is silently dropped from the user-visible proposal — if you want the user to be able to accept your change, you MUST verify it with a passing test in this same turn. If testing is impossible for any reason (missing credentials, required human input), stop and ask a clarifying question instead of leaving an untested draft.MANDATORY: If the latest run only validates an intermediate step (for example, "reached homepage"), do NOT present that as the final proposal when the user asked for a larger outcome (for example, finding/downloading specific files). Continue internal iteration.MANDATORY: For multi-step requests (for example, "go to X and then download Y"), do not stop after validating only the first step. The proposed workflow must cover all requested steps.MANDATORY: Do NOT respond to the user after validating only a subset of the requested outcome. If the user asked for N distinct actions (e.g., "navigate AND download", "login AND search AND add to cart"), your workflow must have blocks covering ALL of those actions before you present a final response. A workflow that only navigates to a page is NOT a complete response for "go to X and download Y".diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.pyindex 5bf271ee5..4e3cbe841 100644--- a/skyvern/forge/sdk/copilot/agent.py+++ b/skyvern/forge/sdk/copilot/agent.py@@ -109,18 +109,66 @@ def _normalize_failure_reason(failure_reason: str | None) -> str:return normalized or "The workflow test run failed."+_FAILURE_FOLLOW_UP = {+ "NAVIGATION_FAILURE": " Can you confirm the URL is correct?",+ "PROXY_ERROR": " Want me to retry with a different proxy location?",+ "PAGE_LOAD_TIMEOUT": " Can you confirm the URL and try again in a moment?",+ "ANTI_BOT_DETECTION": " Want me to retry with a different proxy location?",+ "AUTH_FAILURE": " The site rejected the login — is the stored password still valid?",+ "CREDENTIAL_ERROR": " I couldn't find a credential to use — can you link one in Settings?",+}++def _rewrite_failed_test_response(user_response: str, ctx: CopilotContext) -> str:- if ctx.last_update_block_count is None or ctx.last_test_ok is not False:- return user_response+ # Reshape replies when we cannot ship a proposal this turn so the user+ # sees why nothing is being offered rather than an un-grounded claim.+ if ctx.last_test_ok is False and ctx.last_update_block_count is not None:+ if ctx.last_update_block_count <= 0:+ draft_phrase = "a draft workflow"+ else:+ block_word = "block" if ctx.last_update_block_count == 1 else "blocks"+ draft_phrase = f"a draft workflow with {ctx.last_update_block_count} {block_word}"++ failure_summary = _normalize_failure_reason(ctx.last_test_failure_reason)+ follow_up = _FAILURE_FOLLOW_UP.get(ctx.last_failure_category_top or "", "")+ return f"I created {draft_phrase} and tested it, but the test failed. Failure: {failure_summary}.{follow_up}"++ if ctx.last_test_ok is None and ctx.last_update_block_count is not None and ctx.last_workflow is not None:+ # Agent edited the YAML but didn't verify it this turn; don't promise+ # a re-run we can't durably execute (the restore helper rolls the+ # mid-turn DB write back and there's no durable draft to re-test).+ return (+ "I drafted an update but wasn't able to verify it this turn. "+ "Could you share more context about what you'd like me to do?"+ )++ return user_response+++def _verified_workflow_or_none(ctx: CopilotContext) -> tuple[Any, str | None]:+ """Only surface a workflow proposal when it passed a test this turn.++ SKY-9143: the Accept/Reject UI must never reflect a workflow we haven't+ proven works. Every agent exit path that builds an AgentResult directly+ (cancel, max-turns, timeout, non-retriable nav error, catch-all Exception)+ funnels through this so the strict invariant holds regardless of which+ branch the run took.+ """+ if ctx.last_workflow is not None and ctx.last_test_ok is True:+ return ctx.last_workflow, ctx.last_workflow_yaml+ return None, None- if ctx.last_update_block_count <= 0:- draft_phrase = "a draft workflow"- else:- block_word = "block" if ctx.last_update_block_count == 1 else "blocks"- draft_phrase = f"a draft workflow with {ctx.last_update_block_count} {block_word}"- failure_summary = _normalize_failure_reason(ctx.last_test_failure_reason)- return f"I created {draft_phrase} and tested it, but the test failed. Failure: {failure_summary}"+def _build_exit_result(ctx: CopilotContext, user_response: str, global_llm_context: str | None) -> AgentResult:+ """AgentResult for agent-loop exits that don't go through ``_translate_to_agent_result``."""+ verified_workflow, verified_yaml = _verified_workflow_or_none(ctx)+ return AgentResult(+ user_response=user_response,+ updated_workflow=verified_workflow,+ global_llm_context=global_llm_context,+ workflow_yaml=verified_yaml,+ workflow_was_persisted=ctx.workflow_persisted,+ )def _translate_to_agent_result(@@ -161,11 +209,17 @@ def _translate_to_agent_result(f"Please ask me to fix it.)")- user_response = _rewrite_failed_test_response(str(user_response), ctx)+ # Inline REPLACE_WORKFLOW bypasses _update_workflow so ctx.last_workflow+ # is still whatever the tool layer last saw. Writing the REPLACE_WORKFLOW+ # result onto ctx ensures _verified_workflow_or_none sees the right+ # candidate — though it still gates on last_test_ok, so an untested+ # REPLACE is rejected too.+ if action_data.get("type") == "REPLACE_WORKFLOW" and last_workflow is not ctx.last_workflow:+ ctx.last_workflow = last_workflow+ ctx.last_workflow_yaml = last_workflow_yaml- if ctx.last_test_ok is False:- last_workflow = None- last_workflow_yaml = None+ user_response = _rewrite_failed_test_response(str(user_response), ctx)+ last_workflow, last_workflow_yaml = _verified_workflow_or_none(ctx)resp_type = action_data.get("type", "REPLY")if resp_type not in ("REPLY", "ASK_QUESTION", "REPLACE_WORKFLOW"):@@ -394,28 +448,18 @@ async def run_copilot_agent()except asyncio.CancelledError:LOG.info("Copilot run cancelled")- return AgentResult(- user_response="Request cancelled.",- updated_workflow=ctx.last_workflow,- global_llm_context=global_llm_context,- workflow_yaml=ctx.last_workflow_yaml,- workflow_was_persisted=ctx.workflow_persisted,- )+ return _build_exit_result(ctx, "Request cancelled.", global_llm_context)except MaxTurnsExceeded:- return AgentResult(- user_response="I've reached the maximum number of steps. Here's what I have so far.",- updated_workflow=ctx.last_workflow,- global_llm_context=global_llm_context,- workflow_yaml=ctx.last_workflow_yaml,- workflow_was_persisted=ctx.workflow_persisted,+ return _build_exit_result(+ ctx,+ "I've reached the maximum number of steps. Here's what I have so far.",+ global_llm_context,)except CopilotTotalTimeoutError:- return AgentResult(- user_response="I ran out of time processing your request. Here's what I have so far.",- updated_workflow=ctx.last_workflow,- global_llm_context=global_llm_context,- workflow_yaml=ctx.last_workflow_yaml,- workflow_was_persisted=ctx.workflow_persisted,+ return _build_exit_result(+ ctx,+ "I ran out of time processing your request. Here's what I have so far.",+ global_llm_context,)except CopilotNonRetriableNavError as exc:LOG.warning(@@ -424,25 +468,21 @@ async def run_copilot_agent(error_message=exc.error_message,organization_id=organization_id,)+ # Non-retriable nav errors prove the current workflow doesn't+ # work; zero the proposal even if other tools succeeded.return AgentResult(user_response=(f"The target URL could not be reached. Error: {exc.error_message}. ""Please verify the URL and try again."),- updated_workflow=ctx.last_workflow,+ updated_workflow=None,global_llm_context=global_llm_context,- workflow_yaml=ctx.last_workflow_yaml,+ workflow_yaml=None,workflow_was_persisted=ctx.workflow_persisted,)except Exception as e:LOG.error("Copilot agent error", error=str(e), exc_info=True)- return AgentResult(- user_response="An unexpected error occurred. Please try again.",- updated_workflow=ctx.last_workflow,- global_llm_context=global_llm_context,- workflow_yaml=ctx.last_workflow_yaml,- workflow_was_persisted=ctx.workflow_persisted,- )+ return _build_exit_result(ctx, "An unexpected error occurred. Please try again.", global_llm_context)finally:_copilot_model_name.reset(model_token)session.close()diff --git a/skyvern/forge/sdk/routes/workflow_copilot.py b/skyvern/forge/sdk/routes/workflow_copilot.pyindex 0657fa0c7..601105f34 100644--- a/skyvern/forge/sdk/routes/workflow_copilot.py+++ b/skyvern/forge/sdk/routes/workflow_copilot.py@@ -83,8 +83,19 @@ class BlockRunInfo:def _should_restore_persisted_workflow(auto_accept: bool | None, agent_result: object | None) -> bool:- """Return True when a persisted draft should be rolled back."""- return auto_accept is not True and bool(getattr(agent_result, "workflow_was_persisted", False))+ """Return True when a persisted draft should be rolled back.++ SKY-9143: when the agent decided not to ship a proposal this turn+ (``updated_workflow is None``) but ``_update_workflow`` already committed+ a YAML to ``workflow_definition``, we must restore the original even under+ ``auto_accept=True`` — otherwise an unverified edit becomes the live+ workflow silently.+ """+ if not bool(getattr(agent_result, "workflow_was_persisted", False)):+ return False+ if getattr(agent_result, "updated_workflow", None) is None:+ return True+ return auto_accept is not Trueasync def _restore_workflow_definition(original_workflow: Workflow | None, organization_id: str) -> None:@@ -840,9 +851,17 @@ async def _new_copilot_chat_post(# connected: the user needs to see the reply on reconnect.# SKY-8986: client disconnect used to short-circuit this block# and leave the chat history without the AI response.+ #+ # SKY-9143: restore runs outside the auto_accept wrapper so+ # auto-accept turns that ended without a viable proposal still+ # roll back a mid-turn _update_workflow write. The Accept/Reject+ # panel state below stays gated on auto_accept — the frontend+ # applies proposals via applyWorkflowUpdate when auto-accept is+ # on.+ if _should_restore_persisted_workflow(chat.auto_accept, agent_result):+ await _restore_workflow_definition(original_workflow, organization.organization_id)+if chat.auto_accept is not True:- if _should_restore_persisted_workflow(chat.auto_accept, agent_result):- await _restore_workflow_definition(original_workflow, organization.organization_id)if updated_workflow:proposed_data = updated_workflow.model_dump(mode="json")if agent_result.workflow_yaml:diff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.pyindex e5ef30d7f..3b6df82e0 100644--- a/tests/unit/test_copilot_agent_helpers.py+++ b/tests/unit/test_copilot_agent_helpers.py@@ -5,18 +5,26 @@ from __future__ import annotationsfrom unittest.mock import MagicMock+def _ctx(**overrides):+ from skyvern.forge.sdk.copilot.context import CopilotContext++ defaults = dict(+ organization_id="org-1",+ workflow_id="wf-1",+ workflow_permanent_id="wfp-1",+ workflow_yaml="",+ browser_session_id=None,+ stream=MagicMock(),+ )+ defaults.update(overrides)+ return CopilotContext(**defaults)++class TestFailedTestResponseNormalization:def test_rewrite_failed_test_response_avoids_success_language(self) -> None:from skyvern.forge.sdk.copilot.agent import _rewrite_failed_test_response- from skyvern.forge.sdk.copilot.context import CopilotContext-- ctx = CopilotContext(- organization_id="org-1",- workflow_id="wf-1",- workflow_permanent_id="wfp-1",- workflow_yaml="",- browser_session_id=None,- stream=MagicMock(),++ ctx = _ctx(last_update_block_count=2,last_test_ok=False,last_test_failure_reason=(@@ -58,3 +66,126 @@ class TestFailedTestResponseNormalization:assert ctx.last_workflow is sentinel_workflowassert ctx.last_test_ok is Falseassert ctx.last_test_failure_reason == "net::ERR_NAME_NOT_RESOLVED"++ def test_rewrite_includes_navigation_follow_up_when_category_matches(self) -> None:+ from skyvern.forge.sdk.copilot.agent import _rewrite_failed_test_response++ ctx = _ctx(+ last_update_block_count=1,+ last_test_ok=False,+ last_test_failure_reason="Failed to navigate to url https://bad.example.",+ last_failure_category_top="NAVIGATION_FAILURE",+ )+ rewritten = _rewrite_failed_test_response("done", ctx)++ assert "test failed" in rewritten.lower()+ assert "confirm the url" in rewritten.lower()++ def test_rewrite_untested_edit_asks_for_more_context(self) -> None:+ # SKY-9143 row 3: agent updated YAML without testing. The reply must+ # not promise the backend can re-run a durable draft — the restore+ # helper rolled it back and there is nothing to re-test next turn.+ from skyvern.forge.sdk.copilot.agent import _rewrite_failed_test_response++ sentinel_workflow = object()+ ctx = _ctx(+ last_update_block_count=1,+ last_test_ok=None,+ last_workflow=sentinel_workflow,+ )+ rewritten = _rewrite_failed_test_response("Here's the updated YAML.", ctx)++ assert "drafted an update" in rewritten.lower()+ assert "run it" not in rewritten.lower()+ assert "more context" in rewritten.lower() or "clarify" in rewritten.lower()++ def test_rewrite_passes_through_when_no_update_or_failure(self) -> None:+ from skyvern.forge.sdk.copilot.agent import _rewrite_failed_test_response++ ctx = _ctx()+ original = "Let me know what you want to build."+ assert _rewrite_failed_test_response(original, ctx) == original+++class TestVerifiedWorkflowOrNone:+ """SKY-9143 strict invariant: a proposal surfaces only after a passing test this turn."""++ def _wf(self) -> object:+ return object()++ def test_passes_workflow_when_tested_successfully(self) -> None:+ from skyvern.forge.sdk.copilot.agent import _verified_workflow_or_none++ wf = self._wf()+ ctx = _ctx(last_workflow=wf, last_workflow_yaml="foo: bar", last_test_ok=True)+ assert _verified_workflow_or_none(ctx) == (wf, "foo: bar")++ def test_zeros_when_test_failed(self) -> None:+ from skyvern.forge.sdk.copilot.agent import _verified_workflow_or_none++ ctx = _ctx(last_workflow=self._wf(), last_workflow_yaml="foo: bar", last_test_ok=False)+ assert _verified_workflow_or_none(ctx) == (None, None)++ def test_zeros_when_untested_update(self) -> None:+ # Exactly the scenario where _record_workflow_update_result reset+ # last_test_ok to None after a standalone update_workflow or after+ # the agent edited post-failure without re-testing.+ from skyvern.forge.sdk.copilot.agent import _verified_workflow_or_none++ ctx = _ctx(last_workflow=self._wf(), last_workflow_yaml="foo: bar", last_test_ok=None)+ assert _verified_workflow_or_none(ctx) == (None, None)++ def test_zeros_when_no_last_workflow(self) -> None:+ from skyvern.forge.sdk.copilot.agent import _verified_workflow_or_none++ ctx = _ctx(last_workflow=None, last_test_ok=True)+ assert _verified_workflow_or_none(ctx) == (None, None)++ def test_zeros_on_suspicious_success(self) -> None:+ # _record_run_blocks_result sets last_test_ok=None when blocks ran ok+ # but produced no meaningful extraction data. Still an unverified+ # outcome; must not surface a proposal.+ from skyvern.forge.sdk.copilot.agent import _verified_workflow_or_none++ ctx = _ctx(+ last_workflow=self._wf(),+ last_workflow_yaml="foo: bar",+ last_test_ok=None,+ last_test_suspicious_success=True,+ )+ assert _verified_workflow_or_none(ctx) == (None, None)+++class TestShouldRestorePersistedWorkflow:+ """SKY-9143: auto_accept=True must still restore when no proposal shipped."""++ def _result(self, *, persisted: bool, updated_workflow: object | None):+ r = MagicMock()+ r.workflow_was_persisted = persisted+ r.updated_workflow = updated_workflow+ return r++ def test_restores_when_no_proposal_even_under_auto_accept(self) -> None:+ from skyvern.forge.sdk.routes.workflow_copilot import _should_restore_persisted_workflow++ r = self._result(persisted=True, updated_workflow=None)+ assert _should_restore_persisted_workflow(True, r) is True++ def test_keeps_persisted_write_under_auto_accept_when_proposal_valid(self) -> None:+ from skyvern.forge.sdk.routes.workflow_copilot import _should_restore_persisted_workflow++ r = self._result(persisted=True, updated_workflow=object())+ assert _should_restore_persisted_workflow(True, r) is False++ def test_restores_when_not_auto_accept_and_persisted(self) -> None:+ from skyvern.forge.sdk.routes.workflow_copilot import _should_restore_persisted_workflow++ r = self._result(persisted=True, updated_workflow=object())+ assert _should_restore_persisted_workflow(False, r) is True++ def test_noop_when_nothing_was_persisted(self) -> None:+ from skyvern.forge.sdk.routes.workflow_copilot import _should_restore_persisted_workflow++ r = self._result(persisted=False, updated_workflow=None)+ assert _should_restore_persisted_workflow(True, r) is False+ assert _should_restore_persisted_workflow(False, r) is Falsediff --git a/tests/unit/test_workflow_copilot_route.py b/tests/unit/test_workflow_copilot_route.pyindex 7d3ba35f7..c6db45a6d 100644--- a/tests/unit/test_workflow_copilot_route.py+++ b/tests/unit/test_workflow_copilot_route.py@@ -179,17 +179,26 @@ def _setup_new_copilot_mocks(@pytest.mark.asyncio@pytest.mark.parametrize(- ("auto_accept", "workflow_was_persisted", "expect_restore"),+ ("auto_accept", "workflow_was_persisted", "has_valid_proposal", "expect_restore"),[- (True, True, False), # auto_accept True => no restore- (False, False, False), # nothing persisted => nothing to restore- (False, True, True), # mid-stream disconnect with a persisted draft => restore+ # auto_accept + valid proposal => keep the DB write (frontend applied it)+ (True, True, True, False),+ # auto_accept + no proposal (SKY-9143) => restore the unverified mid-turn write+ (True, True, False, True),+ # Nothing was persisted => nothing to restore regardless of other flags+ (False, False, False, False),+ # Normal mid-stream disconnect with a persisted draft and no proposal => restore+ (False, True, False, True),+ # Normal mid-stream disconnect with a persisted draft and a valid proposal =>+ # still restore, user accepts via the panel to re-apply+ (False, True, True, True),],)async def test_flag_on_mid_stream_disconnect_restores_when_persisted_and_not_auto_accept(monkeypatch: pytest.MonkeyPatch,auto_accept: bool,workflow_was_persisted: bool,+ has_valid_proposal: bool,expect_restore: bool,) -> None:monkeypatch.setattr(settings, "ENABLE_WORKFLOW_COPILOT_V2", True)@@ -209,9 +218,12 @@ async def test_flag_on_mid_stream_disconnect_restores_when_persisted_and_not_autdescription="Original description",workflow_definition=None,)+ proposal = MagicMock(spec=["model_dump"]) if has_valid_proposal else None+ if proposal is not None:+ proposal.model_dump.return_value = {"workflow_id": "wf-canonical"}agent_result = SimpleNamespace(user_response="done",- updated_workflow=None,+ updated_workflow=proposal,global_llm_context=None,workflow_yaml=None,workflow_was_persisted=workflow_was_persisted,