Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix(SKY-9420): suppress proposal panel under ASK_QUESTION turns (#5751)
Andrew Neilson810d0732ec1a
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.pyindex 9f630377b..6d564c7d8 100644--- a/skyvern/forge/sdk/copilot/agent.py+++ b/skyvern/forge/sdk/copilot/agent.py@@ -326,6 +326,12 @@ def _translate_to_agent_result(user_response = _rewrite_failed_test_response(str(user_response), ctx)last_workflow, last_workflow_yaml = _verified_workflow_or_none(ctx)+ # ASK_QUESTION blocks on user input — never surface a verified workflow+ # under it; auto_accept would silently apply a stepping-stone partial.+ if resp_type == "ASK_QUESTION":+ last_workflow = None+ last_workflow_yaml = None+llm_context_raw = action_data.get("global_llm_context")structured = StructuredContext.from_json_str(global_llm_context)if isinstance(llm_context_raw, dict):@@ -346,6 +352,7 @@ def _translate_to_agent_result(workflow_yaml=last_workflow_yaml,workflow_was_persisted=ctx.workflow_persisted,total_tokens=ctx.total_tokens_used,+ clear_proposed_workflow=resp_type == "ASK_QUESTION",)diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.pyindex f801c8cac..c0c4fc247 100644--- a/skyvern/forge/sdk/copilot/context.py+++ b/skyvern/forge/sdk/copilot/context.py@@ -114,9 +114,8 @@ class AgentResult:response_type: ResponseType = "REPLY"workflow_yaml: str | None = Noneworkflow_was_persisted: bool = False- # Feasibility-gate fast-path sets this True so the route can null any- # previously-persisted proposed_workflow. Regular in-loop ASK_QUESTION- # responses leave it False, preserving in-progress drafts.+ # Tells the route to null any persisted proposed_workflow. Set by the+ # feasibility-gate fast-path and by ASK_QUESTION turns.clear_proposed_workflow: bool = False# Actual API token usage accumulated across the agent run. None when no# provider reported usage on the stream — distinguishes "no data" fromdiff --git a/skyvern/forge/sdk/routes/workflow_copilot.py b/skyvern/forge/sdk/routes/workflow_copilot.pyindex 40461bb48..dcedb7951 100644--- a/skyvern/forge/sdk/routes/workflow_copilot.py+++ b/skyvern/forge/sdk/routes/workflow_copilot.py@@ -263,31 +263,25 @@ async def _finalise_normal_turn(if restored:await _restore_workflow_definition(original_workflow, organization_id)- if chat.auto_accept is not True:- if updated_workflow:- proposed_data = updated_workflow.model_dump(mode="json")- if agent_result.workflow_yaml:- proposed_data["_copilot_yaml"] = agent_result.workflow_yaml- await app.DATABASE.workflow_params.update_workflow_copilot_chat(- organization_id=chat.organization_id,- workflow_copilot_chat_id=chat.workflow_copilot_chat_id,- proposed_workflow=proposed_data,- )- elif (- restored or getattr(agent_result, "clear_proposed_workflow", False)- ) and chat.proposed_workflow is not None:- # Null any previously-persisted proposed_workflow so a- # page reload does not resurrect a stale Accept/Reject- # card next to an assistant message that just explained- # why no verified proposal is available. Covers:- # * feasibility-gate fast-path clarifications, and- # * SKY-9143 strict-gate turns where a mid-turn draft was- # rolled back (``restored=True``).- await app.DATABASE.workflow_params.update_workflow_copilot_chat(- organization_id=chat.organization_id,- workflow_copilot_chat_id=chat.workflow_copilot_chat_id,- proposed_workflow=None,- )+ if chat.auto_accept is not True and updated_workflow:+ proposed_data = updated_workflow.model_dump(mode="json")+ if agent_result.workflow_yaml:+ proposed_data["_copilot_yaml"] = agent_result.workflow_yaml+ await app.DATABASE.workflow_params.update_workflow_copilot_chat(+ organization_id=chat.organization_id,+ workflow_copilot_chat_id=chat.workflow_copilot_chat_id,+ proposed_workflow=proposed_data,+ )+ elif chat.proposed_workflow is not None and (restored or agent_result.clear_proposed_workflow):+ # Null any persisted proposed_workflow the assistant just invalidated+ # so a reload does not resurrect a stale Accept/Reject card. Runs+ # under both auto_accept values — a stale proposal can survive an+ # auto-accept toggle.+ await app.DATABASE.workflow_params.update_workflow_copilot_chat(+ organization_id=chat.organization_id,+ workflow_copilot_chat_id=chat.workflow_copilot_chat_id,+ proposed_workflow=None,+ )await app.DATABASE.workflow_params.create_workflow_copilot_chat_message(organization_id=chat.organization_id,diff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.pyindex 2a8f9be98..eea08109d 100644--- a/tests/unit/test_copilot_agent_helpers.py+++ b/tests/unit/test_copilot_agent_helpers.py@@ -6,6 +6,8 @@ import jsonfrom types import SimpleNamespacefrom unittest.mock import MagicMock+from skyvern.forge.sdk.copilot import agent as agent_module+def _ctx(**overrides):from skyvern.forge.sdk.copilot.context import CopilotContext@@ -214,8 +216,6 @@ class TestTranslateToAgentResultGating:# ctx.last_workflow=old_wf). The agent then emits inline REPLACE_WORKFLOW# with a different yaml. The translate helper must invalidate the prior# test result so _verified_workflow_or_none rejects the untested REPLACE.- from skyvern.forge.sdk.copilot import agent as agent_module-old_wf = SimpleNamespace(name="old")new_wf = SimpleNamespace(name="new-from-replace")monkeypatch.setattr(@@ -247,8 +247,6 @@ class TestTranslateToAgentResultGating:# so a prior tested workflow remains available.import yaml as yaml_mod- from skyvern.forge.sdk.copilot import agent as agent_module-tested_wf = SimpleNamespace(name="tested")def boom(**kwargs):@@ -275,8 +273,6 @@ class TestTranslateToAgentResultGating:# cannot test an edit. Row-3 of _rewrite_failed_test_response would# clobber that specific unblocker with "Could you share more context";# the resp_type==ASK_QUESTION guard must skip the rewrite.- from skyvern.forge.sdk.copilot import agent as agent_module-ctx = _ctx(last_update_block_count=1,last_test_ok=None,@@ -298,8 +294,6 @@ class TestTranslateToAgentResultGating:# Guard rail for the above: a plain REPLY after a failed test must# still flow through the "test failed" rewrite so we don't regress# the original SKY-9143 behavior.- from skyvern.forge.sdk.copilot import agent as agent_module-ctx = _ctx(last_update_block_count=2,last_test_ok=False,@@ -321,8 +315,6 @@ class TestTranslateToAgentResultGating:# inline path must do the same, otherwise the untested yaml latches# onto ctx without user-intent framing and any downstream block run# hits the verifier-on-confirmation-surface bug this PR fixes.- from skyvern.forge.sdk.copilot import agent as agent_module-captured: dict[str, str] = {}def fake_process(**kwargs):@@ -346,6 +338,48 @@ class TestTranslateToAgentResultGating:assert captured["yaml"] == "WRAPPED::Submit a contact form on example.com.::raw: yaml"assert ctx.last_workflow_yaml == "WRAPPED::Submit a contact form on example.com.::raw: yaml"+ def test_ask_question_with_verified_workflow_suppresses_and_clears(self) -> None:+ # A verified-but-non-terminal workflow built this turn must not surface+ # alongside the question; the clear flag also nulls any stale prior ghost.+ verified_wf = SimpleNamespace(name="verified-partial")+ ctx = _ctx(last_workflow=verified_wf, last_workflow_yaml="verified: yaml", last_test_ok=True)+ result = _fake_run_result({"type": "ASK_QUESTION", "user_response": "Need credentials before I can continue."})+ agent_result = agent_module._translate_to_agent_result(+ result, ctx, global_llm_context=None, chat_request=_chat_request(), organization_id="org-1"+ )++ assert agent_result.updated_workflow is None+ assert agent_result.workflow_yaml is None+ assert agent_result.response_type == "ASK_QUESTION"+ assert agent_result.clear_proposed_workflow is True++ def test_ask_question_without_workflow_still_sets_clear_flag(self) -> None:+ # An ASK_QUESTION turn with no draft this turn must still null any+ # prior persisted proposal so reload stays coherent.+ ctx = _ctx()+ result = _fake_run_result({"type": "ASK_QUESTION", "user_response": "Which site?"})+ agent_result = agent_module._translate_to_agent_result(+ result, ctx, global_llm_context=None, chat_request=_chat_request(), organization_id="org-1"+ )++ assert agent_result.updated_workflow is None+ assert agent_result.clear_proposed_workflow is True++ def test_reply_does_not_set_clear_proposed_flag(self) -> None:+ # Differential: a REPLY turn surfaces the verified workflow and leaves+ # any prior persisted proposal untouched.+ verified_wf = SimpleNamespace(name="final")+ ctx = _ctx(last_workflow=verified_wf, last_workflow_yaml="final: yaml", last_test_ok=True)+ result = _fake_run_result({"type": "REPLY", "user_response": "Here you go."})+ agent_result = agent_module._translate_to_agent_result(+ result, ctx, global_llm_context=None, chat_request=_chat_request(), organization_id="org-1"+ )++ assert agent_result.updated_workflow is verified_wf+ assert agent_result.workflow_yaml == "final: yaml"+ assert agent_result.response_type == "REPLY"+ assert agent_result.clear_proposed_workflow is False+class TestCredentialRefusalReachesAgent:"""Prove the SKY-9189 refusal rule is actually delivered to the agent.diff --git a/tests/unit/test_workflow_copilot_route.py b/tests/unit/test_workflow_copilot_route.pyindex 3821918ac..a330a6974 100644--- a/tests/unit/test_workflow_copilot_route.py+++ b/tests/unit/test_workflow_copilot_route.py@@ -24,7 +24,7 @@ import pytestfrom skyvern.config import settingsfrom skyvern.forge import appfrom skyvern.forge.sdk.routes.workflow_copilot import COPILOT_V2_FLAG_KEY, workflow_copilot_chat_post-from skyvern.forge.sdk.schemas.workflow_copilot import WorkflowCopilotChatRequest+from skyvern.forge.sdk.schemas.workflow_copilot import WorkflowCopilotChatRequest, WorkflowCopilotStreamResponseUpdatedef _make_chat_request() -> WorkflowCopilotChatRequest:@@ -257,22 +257,39 @@ async def test_flag_on_mid_stream_disconnect_restores_when_persisted_and_not_aut@pytest.mark.asyncio@pytest.mark.parametrize(- ("auto_accept", "workflow_was_persisted", "has_valid_proposal", "prior_proposal", "expect_clear_call"),+ (+ "auto_accept",+ "workflow_was_persisted",+ "has_valid_proposal",+ "prior_proposal",+ "clear_proposed_flag",+ "expect_clear_call",+ ),[- # SKY-9143: auto_accept=False + gated-out turn (no proposal) + mid-turn- # persisted draft => we restore AND clear the stale proposed_workflow- # card so reload doesn't resurrect a card the assistant just invalidated.- (False, True, False, {"workflow_id": "stale"}, True),- # Restore fires but there's nothing stale to clear: skip the no-op DB write.- (False, True, False, None, False),- # auto_accept=False chat-only turn (no persisted draft, no proposal)- # must NOT clear a prior turn's proposed_workflow.- (False, False, False, {"workflow_id": "stale"}, False),- # auto_accept=False turn with a valid proposal sets the new card (not clear).- (False, True, True, {"workflow_id": "stale"}, False),- # auto_accept=True never writes proposed_workflow at all.- (True, True, False, {"workflow_id": "stale"}, False),- (True, True, True, {"workflow_id": "stale"}, False),+ # Restore-and-clear: persisted draft, no proposal, stale prior.+ (False, True, False, {"workflow_id": "stale"}, False, True),+ # Restore fires but nothing stale to clear.+ (False, True, False, None, False, False),+ # Chat-only turn with no clear flag must not touch a prior proposal.+ (False, False, False, {"workflow_id": "stale"}, False, False),+ # New valid proposal stores via the if-branch, not the clear-branch.+ (False, True, True, {"workflow_id": "stale"}, False, False),+ # auto_accept=True default paths: nothing to write.+ (True, True, False, None, False, False),+ (True, True, True, {"workflow_id": "stale"}, False, False),+ # Agent ran run_blocks (persisted=True) then ASK_QUESTIONed with the+ # clear flag set: restore AND clear fire in the same turn.+ (False, True, False, {"workflow_id": "stale"}, True, True),+ # auto_accept=True restore-driven clear: a stale proposal that survived+ # an auto-accept toggle gets nulled when the assistant invalidates it.+ (True, True, False, {"workflow_id": "stale"}, False, True),+ # The clear flag nulls the stale proposal under both auto_accept values+ # even when nothing was persisted this turn.+ (False, False, False, {"workflow_id": "stale"}, True, True),+ (True, False, False, {"workflow_id": "stale"}, True, True),+ # No prior proposal => no DB write even when the clear flag is set.+ (False, False, False, None, True, False),+ (True, False, False, None, True, False),],)async def test_proposed_workflow_cleared_on_restore(@@ -281,6 +298,7 @@ async def test_proposed_workflow_cleared_on_restore(workflow_was_persisted: bool,has_valid_proposal: bool,prior_proposal: dict | None,+ clear_proposed_flag: bool,expect_clear_call: bool,) -> None:monkeypatch.setattr(settings, "ENABLE_WORKFLOW_COPILOT_V2", True)@@ -309,7 +327,7 @@ async def test_proposed_workflow_cleared_on_restore(global_llm_context=None,workflow_yaml=None,workflow_was_persisted=workflow_was_persisted,- clear_proposed_workflow=False,+ clear_proposed_workflow=clear_proposed_flag,)_setup_new_copilot_mocks(monkeypatch, chat, original_workflow, agent_result)@@ -337,6 +355,17 @@ async def test_proposed_workflow_cleared_on_restore(else:assert not clear_calls, f"did not expect a clear call, got {update_calls!r}"+ # The FE's auto_accept code path reads the SSE payload, not+ # chat.proposed_workflow, so the payload must mirror agent_result.+ response_frames = [+ call.args[0]+ for call in stream.send.await_args_list+ if isinstance(call.args[0], WorkflowCopilotStreamResponseUpdate)+ ]+ assert len(response_frames) == 1, f"expected exactly one RESPONSE frame, got {response_frames!r}"+ expected_payload_workflow = proposal.model_dump.return_value if has_valid_proposal else None+ assert response_frames[0].updated_workflow == expected_payload_workflow+@pytest.mark.asyncioasync def test_env_on_short_circuits_posthog(monkeypatch: pytest.MonkeyPatch) -> None: