Skip to content

Automate browser based workflows with AI

fix(SKY-9420): suppress proposal panel under ASK_QUESTION turns (#5751)

Andrew Neilson810d0732ec1a
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.py
index 9f630377b..6d564c7d8 100644
--- a/skyvern/forge/sdk/copilot/agent.py
+++ b/skyvern/forge/sdk/copilot/agent.py
@@ -326,6 +326,12 @@ def _translate_to_agent_result(
user_response = _rewrite_failed_test_response(str(user_response), ctx)
last_workflow, last_workflow_yaml = _verified_workflow_or_none(ctx)
+ # ASK_QUESTION blocks on user input — never surface a verified workflow
+ # under it; auto_accept would silently apply a stepping-stone partial.
+ if resp_type == "ASK_QUESTION":
+ last_workflow = None
+ last_workflow_yaml = None
+
llm_context_raw = action_data.get("global_llm_context")
structured = StructuredContext.from_json_str(global_llm_context)
if isinstance(llm_context_raw, dict):
@@ -346,6 +352,7 @@ def _translate_to_agent_result(
workflow_yaml=last_workflow_yaml,
workflow_was_persisted=ctx.workflow_persisted,
total_tokens=ctx.total_tokens_used,
+ clear_proposed_workflow=resp_type == "ASK_QUESTION",
)
diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.py
index f801c8cac..c0c4fc247 100644
--- a/skyvern/forge/sdk/copilot/context.py
+++ b/skyvern/forge/sdk/copilot/context.py
@@ -114,9 +114,8 @@ class AgentResult:
response_type: ResponseType = "REPLY"
workflow_yaml: str | None = None
workflow_was_persisted: bool = False
- # Feasibility-gate fast-path sets this True so the route can null any
- # previously-persisted proposed_workflow. Regular in-loop ASK_QUESTION
- # responses leave it False, preserving in-progress drafts.
+ # Tells the route to null any persisted proposed_workflow. Set by the
+ # feasibility-gate fast-path and by ASK_QUESTION turns.
clear_proposed_workflow: bool = False
# Actual API token usage accumulated across the agent run. None when no
# provider reported usage on the stream — distinguishes "no data" from
diff --git a/skyvern/forge/sdk/routes/workflow_copilot.py b/skyvern/forge/sdk/routes/workflow_copilot.py
index 40461bb48..dcedb7951 100644
--- a/skyvern/forge/sdk/routes/workflow_copilot.py
+++ b/skyvern/forge/sdk/routes/workflow_copilot.py
@@ -263,31 +263,25 @@ async def _finalise_normal_turn(
if restored:
await _restore_workflow_definition(original_workflow, organization_id)
- if chat.auto_accept is not True:
- if updated_workflow:
- proposed_data = updated_workflow.model_dump(mode="json")
- if agent_result.workflow_yaml:
- proposed_data["_copilot_yaml"] = agent_result.workflow_yaml
- await app.DATABASE.workflow_params.update_workflow_copilot_chat(
- organization_id=chat.organization_id,
- workflow_copilot_chat_id=chat.workflow_copilot_chat_id,
- proposed_workflow=proposed_data,
- )
- elif (
- restored or getattr(agent_result, "clear_proposed_workflow", False)
- ) and chat.proposed_workflow is not None:
- # Null any previously-persisted proposed_workflow so a
- # page reload does not resurrect a stale Accept/Reject
- # card next to an assistant message that just explained
- # why no verified proposal is available. Covers:
- # * feasibility-gate fast-path clarifications, and
- # * SKY-9143 strict-gate turns where a mid-turn draft was
- # rolled back (``restored=True``).
- await app.DATABASE.workflow_params.update_workflow_copilot_chat(
- organization_id=chat.organization_id,
- workflow_copilot_chat_id=chat.workflow_copilot_chat_id,
- proposed_workflow=None,
- )
+ if chat.auto_accept is not True and updated_workflow:
+ proposed_data = updated_workflow.model_dump(mode="json")
+ if agent_result.workflow_yaml:
+ proposed_data["_copilot_yaml"] = agent_result.workflow_yaml
+ await app.DATABASE.workflow_params.update_workflow_copilot_chat(
+ organization_id=chat.organization_id,
+ workflow_copilot_chat_id=chat.workflow_copilot_chat_id,
+ proposed_workflow=proposed_data,
+ )
+ elif chat.proposed_workflow is not None and (restored or agent_result.clear_proposed_workflow):
+ # Null any persisted proposed_workflow the assistant just invalidated
+ # so a reload does not resurrect a stale Accept/Reject card. Runs
+ # under both auto_accept values — a stale proposal can survive an
+ # auto-accept toggle.
+ await app.DATABASE.workflow_params.update_workflow_copilot_chat(
+ organization_id=chat.organization_id,
+ workflow_copilot_chat_id=chat.workflow_copilot_chat_id,
+ proposed_workflow=None,
+ )
await app.DATABASE.workflow_params.create_workflow_copilot_chat_message(
organization_id=chat.organization_id,
diff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.py
index 2a8f9be98..eea08109d 100644
--- a/tests/unit/test_copilot_agent_helpers.py
+++ b/tests/unit/test_copilot_agent_helpers.py
@@ -6,6 +6,8 @@ import json
from types import SimpleNamespace
from unittest.mock import MagicMock
+from skyvern.forge.sdk.copilot import agent as agent_module
+
def _ctx(**overrides):
from skyvern.forge.sdk.copilot.context import CopilotContext
@@ -214,8 +216,6 @@ class TestTranslateToAgentResultGating:
# ctx.last_workflow=old_wf). The agent then emits inline REPLACE_WORKFLOW
# with a different yaml. The translate helper must invalidate the prior
# test result so _verified_workflow_or_none rejects the untested REPLACE.
- from skyvern.forge.sdk.copilot import agent as agent_module
-
old_wf = SimpleNamespace(name="old")
new_wf = SimpleNamespace(name="new-from-replace")
monkeypatch.setattr(
@@ -247,8 +247,6 @@ class TestTranslateToAgentResultGating:
# so a prior tested workflow remains available.
import yaml as yaml_mod
- from skyvern.forge.sdk.copilot import agent as agent_module
-
tested_wf = SimpleNamespace(name="tested")
def boom(**kwargs):
@@ -275,8 +273,6 @@ class TestTranslateToAgentResultGating:
# cannot test an edit. Row-3 of _rewrite_failed_test_response would
# clobber that specific unblocker with "Could you share more context";
# the resp_type==ASK_QUESTION guard must skip the rewrite.
- from skyvern.forge.sdk.copilot import agent as agent_module
-
ctx = _ctx(
last_update_block_count=1,
last_test_ok=None,
@@ -298,8 +294,6 @@ class TestTranslateToAgentResultGating:
# Guard rail for the above: a plain REPLY after a failed test must
# still flow through the "test failed" rewrite so we don't regress
# the original SKY-9143 behavior.
- from skyvern.forge.sdk.copilot import agent as agent_module
-
ctx = _ctx(
last_update_block_count=2,
last_test_ok=False,
@@ -321,8 +315,6 @@ class TestTranslateToAgentResultGating:
# inline path must do the same, otherwise the untested yaml latches
# onto ctx without user-intent framing and any downstream block run
# hits the verifier-on-confirmation-surface bug this PR fixes.
- from skyvern.forge.sdk.copilot import agent as agent_module
-
captured: dict[str, str] = {}
def fake_process(**kwargs):
@@ -346,6 +338,48 @@ class TestTranslateToAgentResultGating:
assert captured["yaml"] == "WRAPPED::Submit a contact form on example.com.::raw: yaml"
assert ctx.last_workflow_yaml == "WRAPPED::Submit a contact form on example.com.::raw: yaml"
+ def test_ask_question_with_verified_workflow_suppresses_and_clears(self) -> None:
+ # A verified-but-non-terminal workflow built this turn must not surface
+ # alongside the question; the clear flag also nulls any stale prior ghost.
+ verified_wf = SimpleNamespace(name="verified-partial")
+ ctx = _ctx(last_workflow=verified_wf, last_workflow_yaml="verified: yaml", last_test_ok=True)
+ result = _fake_run_result({"type": "ASK_QUESTION", "user_response": "Need credentials before I can continue."})
+ agent_result = agent_module._translate_to_agent_result(
+ result, ctx, global_llm_context=None, chat_request=_chat_request(), organization_id="org-1"
+ )
+
+ assert agent_result.updated_workflow is None
+ assert agent_result.workflow_yaml is None
+ assert agent_result.response_type == "ASK_QUESTION"
+ assert agent_result.clear_proposed_workflow is True
+
+ def test_ask_question_without_workflow_still_sets_clear_flag(self) -> None:
+ # An ASK_QUESTION turn with no draft this turn must still null any
+ # prior persisted proposal so reload stays coherent.
+ ctx = _ctx()
+ result = _fake_run_result({"type": "ASK_QUESTION", "user_response": "Which site?"})
+ agent_result = agent_module._translate_to_agent_result(
+ result, ctx, global_llm_context=None, chat_request=_chat_request(), organization_id="org-1"
+ )
+
+ assert agent_result.updated_workflow is None
+ assert agent_result.clear_proposed_workflow is True
+
+ def test_reply_does_not_set_clear_proposed_flag(self) -> None:
+ # Differential: a REPLY turn surfaces the verified workflow and leaves
+ # any prior persisted proposal untouched.
+ verified_wf = SimpleNamespace(name="final")
+ ctx = _ctx(last_workflow=verified_wf, last_workflow_yaml="final: yaml", last_test_ok=True)
+ result = _fake_run_result({"type": "REPLY", "user_response": "Here you go."})
+ agent_result = agent_module._translate_to_agent_result(
+ result, ctx, global_llm_context=None, chat_request=_chat_request(), organization_id="org-1"
+ )
+
+ assert agent_result.updated_workflow is verified_wf
+ assert agent_result.workflow_yaml == "final: yaml"
+ assert agent_result.response_type == "REPLY"
+ assert agent_result.clear_proposed_workflow is False
+
class TestCredentialRefusalReachesAgent:
"""Prove the SKY-9189 refusal rule is actually delivered to the agent.
diff --git a/tests/unit/test_workflow_copilot_route.py b/tests/unit/test_workflow_copilot_route.py
index 3821918ac..a330a6974 100644
--- a/tests/unit/test_workflow_copilot_route.py
+++ b/tests/unit/test_workflow_copilot_route.py
@@ -24,7 +24,7 @@ import pytest
from skyvern.config import settings
from skyvern.forge import app
from skyvern.forge.sdk.routes.workflow_copilot import COPILOT_V2_FLAG_KEY, workflow_copilot_chat_post
-from skyvern.forge.sdk.schemas.workflow_copilot import WorkflowCopilotChatRequest
+from skyvern.forge.sdk.schemas.workflow_copilot import WorkflowCopilotChatRequest, WorkflowCopilotStreamResponseUpdate
def _make_chat_request() -> WorkflowCopilotChatRequest:
@@ -257,22 +257,39 @@ async def test_flag_on_mid_stream_disconnect_restores_when_persisted_and_not_aut
@pytest.mark.asyncio
@pytest.mark.parametrize(
- ("auto_accept", "workflow_was_persisted", "has_valid_proposal", "prior_proposal", "expect_clear_call"),
+ (
+ "auto_accept",
+ "workflow_was_persisted",
+ "has_valid_proposal",
+ "prior_proposal",
+ "clear_proposed_flag",
+ "expect_clear_call",
+ ),
[
- # SKY-9143: auto_accept=False + gated-out turn (no proposal) + mid-turn
- # persisted draft => we restore AND clear the stale proposed_workflow
- # card so reload doesn't resurrect a card the assistant just invalidated.
- (False, True, False, {"workflow_id": "stale"}, True),
- # Restore fires but there's nothing stale to clear: skip the no-op DB write.
- (False, True, False, None, False),
- # auto_accept=False chat-only turn (no persisted draft, no proposal)
- # must NOT clear a prior turn's proposed_workflow.
- (False, False, False, {"workflow_id": "stale"}, False),
- # auto_accept=False turn with a valid proposal sets the new card (not clear).
- (False, True, True, {"workflow_id": "stale"}, False),
- # auto_accept=True never writes proposed_workflow at all.
- (True, True, False, {"workflow_id": "stale"}, False),
- (True, True, True, {"workflow_id": "stale"}, False),
+ # Restore-and-clear: persisted draft, no proposal, stale prior.
+ (False, True, False, {"workflow_id": "stale"}, False, True),
+ # Restore fires but nothing stale to clear.
+ (False, True, False, None, False, False),
+ # Chat-only turn with no clear flag must not touch a prior proposal.
+ (False, False, False, {"workflow_id": "stale"}, False, False),
+ # New valid proposal stores via the if-branch, not the clear-branch.
+ (False, True, True, {"workflow_id": "stale"}, False, False),
+ # auto_accept=True default paths: nothing to write.
+ (True, True, False, None, False, False),
+ (True, True, True, {"workflow_id": "stale"}, False, False),
+ # Agent ran run_blocks (persisted=True) then ASK_QUESTIONed with the
+ # clear flag set: restore AND clear fire in the same turn.
+ (False, True, False, {"workflow_id": "stale"}, True, True),
+ # auto_accept=True restore-driven clear: a stale proposal that survived
+ # an auto-accept toggle gets nulled when the assistant invalidates it.
+ (True, True, False, {"workflow_id": "stale"}, False, True),
+ # The clear flag nulls the stale proposal under both auto_accept values
+ # even when nothing was persisted this turn.
+ (False, False, False, {"workflow_id": "stale"}, True, True),
+ (True, False, False, {"workflow_id": "stale"}, True, True),
+ # No prior proposal => no DB write even when the clear flag is set.
+ (False, False, False, None, True, False),
+ (True, False, False, None, True, False),
],
)
async def test_proposed_workflow_cleared_on_restore(
@@ -281,6 +298,7 @@ async def test_proposed_workflow_cleared_on_restore(
workflow_was_persisted: bool,
has_valid_proposal: bool,
prior_proposal: dict | None,
+ clear_proposed_flag: bool,
expect_clear_call: bool,
) -> None:
monkeypatch.setattr(settings, "ENABLE_WORKFLOW_COPILOT_V2", True)
@@ -309,7 +327,7 @@ async def test_proposed_workflow_cleared_on_restore(
global_llm_context=None,
workflow_yaml=None,
workflow_was_persisted=workflow_was_persisted,
- clear_proposed_workflow=False,
+ clear_proposed_workflow=clear_proposed_flag,
)
_setup_new_copilot_mocks(monkeypatch, chat, original_workflow, agent_result)
@@ -337,6 +355,17 @@ async def test_proposed_workflow_cleared_on_restore(
else:
assert not clear_calls, f"did not expect a clear call, got {update_calls!r}"
+ # The FE's auto_accept code path reads the SSE payload, not
+ # chat.proposed_workflow, so the payload must mirror agent_result.
+ response_frames = [
+ call.args[0]
+ for call in stream.send.await_args_list
+ if isinstance(call.args[0], WorkflowCopilotStreamResponseUpdate)
+ ]
+ assert len(response_frames) == 1, f"expected exactly one RESPONSE frame, got {response_frames!r}"
+ expected_payload_workflow = proposal.model_dump.return_value if has_valid_proposal else None
+ assert response_frames[0].updated_workflow == expected_payload_workflow
+
@pytest.mark.asyncio
async def test_env_on_short_circuits_posthog(monkeypatch: pytest.MonkeyPatch) -> None: