Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
SKY-9658 align unvalidated proposal replies (#5891)
Andrew Neilsona93a6ec0c191
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.pyindex 829a021af..2a82e71d8 100644--- a/skyvern/forge/sdk/copilot/agent.py+++ b/skyvern/forge/sdk/copilot/agent.py@@ -523,6 +523,20 @@ _UNBACKED_WORKFLOW_DELIVERY_REPLY = ("I wasn't able to produce a workflow proposal in this turn. Please try again, or provide the missing details ""so I can build and test it.")+_PROPOSAL_ACCEPT_UI_ACTION_RE = re.compile(r"\b(?:accept|always\s+accept)\b", re.IGNORECASE)+_PROPOSAL_REJECT_UI_ACTION_RE = re.compile(r"\b(?:reject|discard)\b", re.IGNORECASE)+_UNVALIDATED_PROPOSAL_AFFORDANCE = (+ "I have a draft workflow proposal. Use Review to inspect it, Accept to save it, or Reject to discard it. "+ "It has not been verified end-to-end."+)+++def _ensure_unvalidated_proposal_affordance(user_response: str) -> str:+ if _PROPOSAL_ACCEPT_UI_ACTION_RE.search(user_response) and _PROPOSAL_REJECT_UI_ACTION_RE.search(user_response):+ return user_response+ if user_response.strip():+ return f"{user_response}\n\n{_UNVALIDATED_PROPOSAL_AFFORDANCE}"+ return _UNVALIDATED_PROPOSAL_AFFORDANCEdef _build_wip_exit_result(@@ -767,6 +781,8 @@ def _translate_to_agent_result(and looks_like_workflow_delivery_claim(user_response)):user_response = _UNBACKED_WORKFLOW_DELIVERY_REPLY+ if resp_type == "REPLY" and last_workflow is not None and unvalidated:+ user_response = _ensure_unvalidated_proposal_affordance(str(user_response))llm_context_raw = action_data.get("global_llm_context")structured = StructuredContext.from_json_str(global_llm_context)diff --git a/skyvern/forge/sdk/copilot/output_utils.py b/skyvern/forge/sdk/copilot/output_utils.pyindex 9431c14cb..6025e0784 100644--- a/skyvern/forge/sdk/copilot/output_utils.py+++ b/skyvern/forge/sdk/copilot/output_utils.py@@ -49,6 +49,7 @@ _USER_RESPONSE_VALUE_RE = re.compile(r'"user_response"\s*:\s*"((?:[^"\\]|\\.)*)"_TYPE_VALUE_RE = re.compile(rf'"type"\s*:\s*"({_TYPE_ALTERNATION})"')_WORKFLOW_DELIVERY_CLAIM_PATTERNS = [re.compile(r"\bhere(?:'|’)?s\s+(?:the|a)\s+workflow\b", re.IGNORECASE),+ re.compile(r"\b(?:i(?:'|’)?ve|i\s+have)\s+drafted\b.{0,80}\bworkflow\b", re.IGNORECASE),re.compile(r"\b(?:created|built|drafted|generated)\s+(?:a|the)\s+(?:draft\s+)?workflow\b", re.IGNORECASE),re.compile(r"\byour\s+workflow\s+(?:is\s+)?(?:ready|complete|completed|set\s+up)\b", re.IGNORECASE),re.compile(r"\bworkflow\s+(?:is\s+)?(?:ready|complete|completed)\b", re.IGNORECASE),diff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.pyindex 978ce0410..bcea6eacd 100644--- a/tests/unit/test_copilot_agent_helpers.py+++ b/tests/unit/test_copilot_agent_helpers.py@@ -552,6 +552,49 @@ workflow_definition:assert agent_result.updated_workflow is wfassert agent_result.unvalidated is True+ assert "review" in agent_result.user_response.lower()+ assert "accept" in agent_result.user_response.lower()+ assert "reject" in agent_result.user_response.lower()+ assert "discard" in agent_result.user_response.lower()++ def test_unvalidated_wip_reply_adds_proposal_affordance(self) -> None:+ wf = SimpleNamespace(name="drafted")+ ctx = _ctx(+ last_workflow=wf,+ last_workflow_yaml="title: drafted",+ last_update_block_count=None,+ last_test_ok=None,+ )+ result = _fake_run_result({"type": "REPLY", "user_response": "Please provide credentials before I continue."})+ agent_result = agent_module._translate_to_agent_result(+ result, ctx, global_llm_context=None, chat_request=_chat_request(), organization_id="org-1"+ )++ assert agent_result.updated_workflow is wf+ assert agent_result.workflow_yaml == "title: drafted"+ assert agent_result.unvalidated is True+ assert "Please provide credentials before I continue." in agent_result.user_response+ assert "review" in agent_result.user_response.lower()+ assert "accept" in agent_result.user_response.lower()+ assert "reject" in agent_result.user_response.lower()+ assert "discard" in agent_result.user_response.lower()++ def test_unvalidated_wip_reply_keeps_existing_ui_affordance(self) -> None:+ wf = SimpleNamespace(name="drafted")+ ctx = _ctx(+ last_workflow=wf,+ last_workflow_yaml="title: drafted",+ last_update_block_count=None,+ last_test_ok=None,+ )+ response = "I have a draft proposal. Use Review to inspect it, Accept to save it, or Reject it."+ result = _fake_run_result({"type": "REPLY", "user_response": response})+ agent_result = agent_module._translate_to_agent_result(+ result, ctx, global_llm_context=None, chat_request=_chat_request(), organization_id="org-1"+ )++ assert agent_result.user_response == response+ assert agent_result.updated_workflow is wfdef test_goal_reached_false_flips_validated_proposal_to_unvalidated(self) -> None:# Agent-emitted goal_reached=False must override last_test_ok=True so@@ -660,6 +703,23 @@ workflow_definition:assert agent_result.updated_workflow is Noneassert agent_result.workflow_yaml is None+ def test_initial_part_workflow_claim_is_rewritten_without_proposal(self) -> None:+ ctx = _ctx(last_test_ok=None)+ result = _fake_run_result(+ {+ "type": "REPLY",+ "user_response": "In the meantime, I've drafted the initial part of your workflow with placeholders.",+ }+ )+ agent_result = agent_module._translate_to_agent_result(+ result, ctx, global_llm_context=None, chat_request=_chat_request(), organization_id="org-1"+ )++ assert "initial part of your workflow" not in agent_result.user_response.lower()+ assert "wasn't able to produce a workflow proposal" in agent_result.user_response+ assert agent_result.updated_workflow is None+ assert agent_result.workflow_yaml is None+def test_unbacked_workflow_claim_not_rewritten_when_proposal_exists(self) -> None:wf = SimpleNamespace(name="drafted")ctx = _ctx(last_workflow=wf, last_workflow_yaml="title: drafted", last_test_ok=True)diff --git a/tests/unit/test_copilot_mid_correction_guard.py b/tests/unit/test_copilot_mid_correction_guard.pyindex f102b71c8..f6296d9e4 100644--- a/tests/unit/test_copilot_mid_correction_guard.py+++ b/tests/unit/test_copilot_mid_correction_guard.py@@ -198,7 +198,10 @@ def test_translate_to_agent_result_salvages_last_good_on_failed_reply() -> None:assert agent_result.unvalidated is True# Failure rewrite would have replaced the agent's text with one based on# ``last_update_block_count=5``; salvage must skip the rewrite.- assert agent_result.user_response == agent_text+ assert agent_result.user_response.startswith(agent_text)+ assert "Use Review to inspect it" in agent_result.user_response+ assert "Accept to save it" in agent_result.user_response+ assert "Reject to discard it" in agent_result.user_responsedef test_translate_to_agent_result_salvages_after_failure_then_update_workflow() -> None:@@ -220,7 +223,10 @@ def test_translate_to_agent_result_salvages_after_failure_then_update_workflow())assert agent_result.updated_workflow is ctx.last_good_workflowassert agent_result.unvalidated is True- assert agent_result.user_response == agent_text+ assert agent_result.user_response.startswith(agent_text)+ assert "Use Review to inspect it" in agent_result.user_response+ assert "Accept to save it" in agent_result.user_response+ assert "Reject to discard it" in agent_result.user_responsedef test_translate_to_agent_result_does_not_salvage_on_standalone_edit() -> None:diff --git a/tests/unit/test_copilot_response_coverage_gate.py b/tests/unit/test_copilot_response_coverage_gate.pyindex 9049a629c..20851daf2 100644--- a/tests/unit/test_copilot_response_coverage_gate.py+++ b/tests/unit/test_copilot_response_coverage_gate.py@@ -137,6 +137,17 @@ def test_reply_claiming_workflow_without_update_fires_nudge() -> None:assert ctx.no_workflow_nudge_count == 1+def test_initial_part_workflow_claim_without_update_fires_nudge() -> None:+ ctx = _Ctx()+ parsed = {+ "type": "REPLY",+ "user_response": "In the meantime, I've drafted the initial part of your workflow with placeholders.",+ }++ assert _response_coverage_nudge(ctx, parsed) == POST_NO_WORKFLOW_DELIVERY_NUDGE+ assert ctx.no_workflow_nudge_count == 1++def test_no_workflow_delivery_nudge_respects_counter_cap() -> None:ctx = _Ctx()parsed = {"type": "REPLY", "user_response": "I created a workflow for this."}