Skip to content

Automate browser based workflows with AI

SKY-9658 align unvalidated proposal replies (#5891)

Andrew Neilsona93a6ec0c191
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.py
index 829a021af..2a82e71d8 100644
--- a/skyvern/forge/sdk/copilot/agent.py
+++ b/skyvern/forge/sdk/copilot/agent.py
@@ -523,6 +523,20 @@ _UNBACKED_WORKFLOW_DELIVERY_REPLY = (
"I wasn't able to produce a workflow proposal in this turn. Please try again, or provide the missing details "
"so I can build and test it."
)
+_PROPOSAL_ACCEPT_UI_ACTION_RE = re.compile(r"\b(?:accept|always\s+accept)\b", re.IGNORECASE)
+_PROPOSAL_REJECT_UI_ACTION_RE = re.compile(r"\b(?:reject|discard)\b", re.IGNORECASE)
+_UNVALIDATED_PROPOSAL_AFFORDANCE = (
+ "I have a draft workflow proposal. Use Review to inspect it, Accept to save it, or Reject to discard it. "
+ "It has not been verified end-to-end."
+)
+
+
+def _ensure_unvalidated_proposal_affordance(user_response: str) -> str:
+ if _PROPOSAL_ACCEPT_UI_ACTION_RE.search(user_response) and _PROPOSAL_REJECT_UI_ACTION_RE.search(user_response):
+ return user_response
+ if user_response.strip():
+ return f"{user_response}\n\n{_UNVALIDATED_PROPOSAL_AFFORDANCE}"
+ return _UNVALIDATED_PROPOSAL_AFFORDANCE
def _build_wip_exit_result(
@@ -767,6 +781,8 @@ def _translate_to_agent_result(
and looks_like_workflow_delivery_claim(user_response)
):
user_response = _UNBACKED_WORKFLOW_DELIVERY_REPLY
+ if resp_type == "REPLY" and last_workflow is not None and unvalidated:
+ user_response = _ensure_unvalidated_proposal_affordance(str(user_response))
llm_context_raw = action_data.get("global_llm_context")
structured = StructuredContext.from_json_str(global_llm_context)
diff --git a/skyvern/forge/sdk/copilot/output_utils.py b/skyvern/forge/sdk/copilot/output_utils.py
index 9431c14cb..6025e0784 100644
--- a/skyvern/forge/sdk/copilot/output_utils.py
+++ b/skyvern/forge/sdk/copilot/output_utils.py
@@ -49,6 +49,7 @@ _USER_RESPONSE_VALUE_RE = re.compile(r'"user_response"\s*:\s*"((?:[^"\\]|\\.)*)"
_TYPE_VALUE_RE = re.compile(rf'"type"\s*:\s*"({_TYPE_ALTERNATION})"')
_WORKFLOW_DELIVERY_CLAIM_PATTERNS = [
re.compile(r"\bhere(?:'|’)?s\s+(?:the|a)\s+workflow\b", re.IGNORECASE),
+ re.compile(r"\b(?:i(?:'|’)?ve|i\s+have)\s+drafted\b.{0,80}\bworkflow\b", re.IGNORECASE),
re.compile(r"\b(?:created|built|drafted|generated)\s+(?:a|the)\s+(?:draft\s+)?workflow\b", re.IGNORECASE),
re.compile(r"\byour\s+workflow\s+(?:is\s+)?(?:ready|complete|completed|set\s+up)\b", re.IGNORECASE),
re.compile(r"\bworkflow\s+(?:is\s+)?(?:ready|complete|completed)\b", re.IGNORECASE),
diff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.py
index 978ce0410..bcea6eacd 100644
--- a/tests/unit/test_copilot_agent_helpers.py
+++ b/tests/unit/test_copilot_agent_helpers.py
@@ -552,6 +552,49 @@ workflow_definition:
assert agent_result.updated_workflow is wf
assert agent_result.unvalidated is True
+ assert "review" in agent_result.user_response.lower()
+ assert "accept" in agent_result.user_response.lower()
+ assert "reject" in agent_result.user_response.lower()
+ assert "discard" in agent_result.user_response.lower()
+
+ def test_unvalidated_wip_reply_adds_proposal_affordance(self) -> None:
+ wf = SimpleNamespace(name="drafted")
+ ctx = _ctx(
+ last_workflow=wf,
+ last_workflow_yaml="title: drafted",
+ last_update_block_count=None,
+ last_test_ok=None,
+ )
+ result = _fake_run_result({"type": "REPLY", "user_response": "Please provide credentials before I continue."})
+ agent_result = agent_module._translate_to_agent_result(
+ result, ctx, global_llm_context=None, chat_request=_chat_request(), organization_id="org-1"
+ )
+
+ assert agent_result.updated_workflow is wf
+ assert agent_result.workflow_yaml == "title: drafted"
+ assert agent_result.unvalidated is True
+ assert "Please provide credentials before I continue." in agent_result.user_response
+ assert "review" in agent_result.user_response.lower()
+ assert "accept" in agent_result.user_response.lower()
+ assert "reject" in agent_result.user_response.lower()
+ assert "discard" in agent_result.user_response.lower()
+
+ def test_unvalidated_wip_reply_keeps_existing_ui_affordance(self) -> None:
+ wf = SimpleNamespace(name="drafted")
+ ctx = _ctx(
+ last_workflow=wf,
+ last_workflow_yaml="title: drafted",
+ last_update_block_count=None,
+ last_test_ok=None,
+ )
+ response = "I have a draft proposal. Use Review to inspect it, Accept to save it, or Reject it."
+ result = _fake_run_result({"type": "REPLY", "user_response": response})
+ agent_result = agent_module._translate_to_agent_result(
+ result, ctx, global_llm_context=None, chat_request=_chat_request(), organization_id="org-1"
+ )
+
+ assert agent_result.user_response == response
+ assert agent_result.updated_workflow is wf
def test_goal_reached_false_flips_validated_proposal_to_unvalidated(self) -> None:
# Agent-emitted goal_reached=False must override last_test_ok=True so
@@ -660,6 +703,23 @@ workflow_definition:
assert agent_result.updated_workflow is None
assert agent_result.workflow_yaml is None
+ def test_initial_part_workflow_claim_is_rewritten_without_proposal(self) -> None:
+ ctx = _ctx(last_test_ok=None)
+ result = _fake_run_result(
+ {
+ "type": "REPLY",
+ "user_response": "In the meantime, I've drafted the initial part of your workflow with placeholders.",
+ }
+ )
+ agent_result = agent_module._translate_to_agent_result(
+ result, ctx, global_llm_context=None, chat_request=_chat_request(), organization_id="org-1"
+ )
+
+ assert "initial part of your workflow" not in agent_result.user_response.lower()
+ assert "wasn't able to produce a workflow proposal" in agent_result.user_response
+ assert agent_result.updated_workflow is None
+ assert agent_result.workflow_yaml is None
+
def test_unbacked_workflow_claim_not_rewritten_when_proposal_exists(self) -> None:
wf = SimpleNamespace(name="drafted")
ctx = _ctx(last_workflow=wf, last_workflow_yaml="title: drafted", last_test_ok=True)
diff --git a/tests/unit/test_copilot_mid_correction_guard.py b/tests/unit/test_copilot_mid_correction_guard.py
index f102b71c8..f6296d9e4 100644
--- a/tests/unit/test_copilot_mid_correction_guard.py
+++ b/tests/unit/test_copilot_mid_correction_guard.py
@@ -198,7 +198,10 @@ def test_translate_to_agent_result_salvages_last_good_on_failed_reply() -> None:
assert agent_result.unvalidated is True
# Failure rewrite would have replaced the agent's text with one based on
# ``last_update_block_count=5``; salvage must skip the rewrite.
- assert agent_result.user_response == agent_text
+ assert agent_result.user_response.startswith(agent_text)
+ assert "Use Review to inspect it" in agent_result.user_response
+ assert "Accept to save it" in agent_result.user_response
+ assert "Reject to discard it" in agent_result.user_response
def test_translate_to_agent_result_salvages_after_failure_then_update_workflow() -> None:
@@ -220,7 +223,10 @@ def test_translate_to_agent_result_salvages_after_failure_then_update_workflow()
)
assert agent_result.updated_workflow is ctx.last_good_workflow
assert agent_result.unvalidated is True
- assert agent_result.user_response == agent_text
+ assert agent_result.user_response.startswith(agent_text)
+ assert "Use Review to inspect it" in agent_result.user_response
+ assert "Accept to save it" in agent_result.user_response
+ assert "Reject to discard it" in agent_result.user_response
def test_translate_to_agent_result_does_not_salvage_on_standalone_edit() -> None:
diff --git a/tests/unit/test_copilot_response_coverage_gate.py b/tests/unit/test_copilot_response_coverage_gate.py
index 9049a629c..20851daf2 100644
--- a/tests/unit/test_copilot_response_coverage_gate.py
+++ b/tests/unit/test_copilot_response_coverage_gate.py
@@ -137,6 +137,17 @@ def test_reply_claiming_workflow_without_update_fires_nudge() -> None:
assert ctx.no_workflow_nudge_count == 1
+def test_initial_part_workflow_claim_without_update_fires_nudge() -> None:
+ ctx = _Ctx()
+ parsed = {
+ "type": "REPLY",
+ "user_response": "In the meantime, I've drafted the initial part of your workflow with placeholders.",
+ }
+
+ assert _response_coverage_nudge(ctx, parsed) == POST_NO_WORKFLOW_DELIVERY_NUDGE
+ assert ctx.no_workflow_nudge_count == 1
+
+
def test_no_workflow_delivery_nudge_respects_counter_cap() -> None:
ctx = _Ctx()
parsed = {"type": "REPLY", "user_response": "I created a workflow for this."}