Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix(SKY-9658): prevent unbacked copilot workflow delivery replies (#5890)
Andrew Neilsonf8144c4f7aae
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.pyindex 9ef355430..829a021af 100644--- a/skyvern/forge/sdk/copilot/agent.py+++ b/skyvern/forge/sdk/copilot/agent.py@@ -27,7 +27,11 @@ from skyvern.forge import appfrom skyvern.forge.prompts import prompt_enginefrom skyvern.forge.sdk.copilot.block_goal_wrapping import wrap_block_goalsfrom skyvern.forge.sdk.copilot.context import COPILOT_RESPONSE_TYPES, AgentResult, CopilotContext, StructuredContext-from skyvern.forge.sdk.copilot.output_utils import extract_final_text, parse_final_response+from skyvern.forge.sdk.copilot.output_utils import (+ extract_final_text,+ looks_like_workflow_delivery_claim,+ parse_final_response,+)from skyvern.forge.sdk.copilot.tracing_setup import _copilot_model_name, ensure_tracing_initialized, is_tracing_enabledfrom skyvern.forge.sdk.schemas.persistent_browser_sessions import is_final_statusfrom skyvern.forge.sdk.schemas.workflow_copilot import (@@ -515,6 +519,10 @@ _CANCEL_REPLY_UNVALIDATED = ("(note: it hasn't been verified end-to-end), or discard.")_CANCEL_REPLY_TESTED = "Cancelled. I have a tested draft for you. Accept it to save, or discard."+_UNBACKED_WORKFLOW_DELIVERY_REPLY = (+ "I wasn't able to produce a workflow proposal in this turn. Please try again, or provide the missing details "+ "so I can build and test it."+)def _build_wip_exit_result(@@ -749,6 +757,17 @@ def _translate_to_agent_result(last_workflow = Nonelast_workflow_yaml = None+ last_update_block_count = ctx.last_update_block_count+ last_test_ok = ctx.last_test_ok+ if (+ resp_type == "REPLY"+ and last_workflow is None+ and last_update_block_count is None+ and last_test_ok is None+ and looks_like_workflow_delivery_claim(user_response)+ ):+ user_response = _UNBACKED_WORKFLOW_DELIVERY_REPLY+llm_context_raw = action_data.get("global_llm_context")structured = StructuredContext.from_json_str(global_llm_context)if isinstance(llm_context_raw, dict):diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.pyindex b89cc860c..d50e30a1a 100644--- a/skyvern/forge/sdk/copilot/context.py+++ b/skyvern/forge/sdk/copilot/context.py@@ -160,6 +160,7 @@ class CopilotContext(AgentContext):post_update_nudge_count: int = 0coverage_nudge_count: int = 0format_nudge_count: int = 0+ no_workflow_nudge_count: int = 0copilot_total_timeout_exceeded: bool = Falseuser_message: str = ""block_goal_main_goal: str = ""diff --git a/skyvern/forge/sdk/copilot/enforcement.py b/skyvern/forge/sdk/copilot/enforcement.pyindex b4ff5792e..bb306ea57 100644--- a/skyvern/forge/sdk/copilot/enforcement.py+++ b/skyvern/forge/sdk/copilot/enforcement.py@@ -14,7 +14,11 @@ from agents.run import Runnerfrom skyvern.forge.sdk.copilot.failure_tracking import PER_TOOL_BUDGET_FAILURE_CATEGORY, normalize_failure_reasonfrom skyvern.forge.sdk.copilot.narration import TransitionKind-from skyvern.forge.sdk.copilot.output_utils import extract_final_text, parse_final_response+from skyvern.forge.sdk.copilot.output_utils import (+ extract_final_text,+ looks_like_workflow_delivery_claim,+ parse_final_response,+)from skyvern.forge.sdk.copilot.screenshot_utils import ScreenshotEntryfrom skyvern.forge.sdk.copilot.tracing_setup import copilot_spanfrom skyvern.utils.token_counter import count_tokens@@ -31,6 +35,7 @@ MAX_POST_UPDATE_NUDGES = 2MAX_INTERMEDIATE_NUDGES = 8MAX_FAILED_TEST_NUDGES = 2MAX_FORMAT_NUDGES = 2+MAX_NO_WORKFLOW_NUDGES = 2MAX_EXPLORE_WITHOUT_WORKFLOW_NUDGES = 2# Stops the suspicious-success nudge from re-firing forever when the agent has# correctly diagnosed an unrecoverable block (anti-bot, paywall) and is no@@ -257,6 +262,15 @@ POST_PER_TOOL_BUDGET_NUDGE = ("navigation_goal scope, or reply with a blocker explanation.")+POST_NO_WORKFLOW_DELIVERY_NUDGE = (+ "STOP — you are telling the user you created or are showing a workflow, "+ "but no workflow update tool has succeeded in this turn. The user will see "+ "an empty proposal. You MUST either call update_and_run_blocks with a real "+ "workflow and test it, or respond with ASK_QUESTION if required input is "+ 'missing. Do NOT say "Here\'s the workflow" until there is an actual '+ "workflow proposal behind the response."+)+_PROBABLE_SITE_BLOCK_STOP_NUDGE_PREFIX = ("STOP — the target site has failed to scrape on every attempt across ""multiple workflow shapes. Every run navigated successfully but the "@@ -481,6 +495,16 @@ def _response_coverage_nudge(ctx: Any, parsed: dict[str, Any]) -> str | None:if response_type not in ("REPLY", "REPLACE_WORKFLOW"):return None+ if (+ response_type == "REPLY"+ and not getattr(ctx, "update_workflow_called", False)+ and looks_like_workflow_delivery_claim(parsed.get("user_response"))+ ):+ nudge_count = getattr(ctx, "no_workflow_nudge_count", 0)+ if nudge_count < MAX_NO_WORKFLOW_NUDGES:+ ctx.no_workflow_nudge_count = nudge_count + 1+ return POST_NO_WORKFLOW_DELIVERY_NUDGE+workflow_tested_ok = (getattr(ctx, "last_test_ok", None) is Trueand getattr(ctx, "update_workflow_called", False)@@ -1013,6 +1037,7 @@ _NUDGE_TYPE_BY_MESSAGE: dict[str, str] = {POST_ANTI_BOT_FAILED_TEST_NUDGE: "anti_bot_block",POST_PROBABLE_SITE_BLOCK_STOP_NUDGE: "probable_site_block_stop",POST_PER_TOOL_BUDGET_NUDGE: "per_tool_budget_split",+ POST_NO_WORKFLOW_DELIVERY_NUDGE: "no_workflow_delivery",POST_FAILED_TEST_NUDGE: "post_failed_test",SCREENSHOT_DROPPED_NUDGE: "screenshot_dropped_on_recovery",}diff --git a/skyvern/forge/sdk/copilot/output_utils.py b/skyvern/forge/sdk/copilot/output_utils.pyindex 2b0bb021d..9431c14cb 100644--- a/skyvern/forge/sdk/copilot/output_utils.py+++ b/skyvern/forge/sdk/copilot/output_utils.py@@ -47,6 +47,12 @@ _TYPE_ALTERNATION = "|".join(COPILOT_RESPONSE_TYPES)_LEADING_LABEL_RE = re.compile(rf"^\s*({_TYPE_ALTERNATION})\s*[:,]?\s+", re.IGNORECASE)_USER_RESPONSE_VALUE_RE = re.compile(r'"user_response"\s*:\s*"((?:[^"\\]|\\.)*)"')_TYPE_VALUE_RE = re.compile(rf'"type"\s*:\s*"({_TYPE_ALTERNATION})"')+_WORKFLOW_DELIVERY_CLAIM_PATTERNS = [+ re.compile(r"\bhere(?:'|’)?s\s+(?:the|a)\s+workflow\b", re.IGNORECASE),+ re.compile(r"\b(?:created|built|drafted|generated)\s+(?:a|the)\s+(?:draft\s+)?workflow\b", re.IGNORECASE),+ re.compile(r"\byour\s+workflow\s+(?:is\s+)?(?:ready|complete|completed|set\s+up)\b", re.IGNORECASE),+ re.compile(r"\bworkflow\s+(?:is\s+)?(?:ready|complete|completed)\b", re.IGNORECASE),+]def _try_loads_dict(text: str) -> dict[str, Any] | None:@@ -130,6 +136,12 @@ def parse_final_response(text: str) -> dict[str, Any]:return {"type": "REPLY", "user_response": text}+def looks_like_workflow_delivery_claim(text: Any) -> bool:+ if not isinstance(text, str) or not text.strip():+ return False+ return any(pattern.search(text) for pattern in _WORKFLOW_DELIVERY_CLAIM_PATTERNS)++def extract_screenshot_b64(result: dict[str, Any]) -> str | None:"""Extract screenshot_base64 from a copilot result dict, if present."""data = result.get("data")diff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.pyindex 95301d4fd..978ce0410 100644--- a/tests/unit/test_copilot_agent_helpers.py+++ b/tests/unit/test_copilot_agent_helpers.py@@ -648,6 +648,29 @@ workflow_definition:assert agent_result.workflow_yaml is Noneassert agent_result.unvalidated is False+ def test_unbacked_workflow_claim_is_rewritten_without_proposal(self) -> None:+ ctx = _ctx(last_test_ok=None)+ result = _fake_run_result({"type": "REPLY", "user_response": "Here's the workflow."})+ agent_result = agent_module._translate_to_agent_result(+ result, ctx, global_llm_context=None, chat_request=_chat_request(), organization_id="org-1"+ )++ assert "here's the workflow" not in agent_result.user_response.lower()+ assert "wasn't able to produce a workflow proposal" in agent_result.user_response+ assert agent_result.updated_workflow is None+ assert agent_result.workflow_yaml is None++ def test_unbacked_workflow_claim_not_rewritten_when_proposal_exists(self) -> None:+ wf = SimpleNamespace(name="drafted")+ ctx = _ctx(last_workflow=wf, last_workflow_yaml="title: drafted", last_test_ok=True)+ result = _fake_run_result({"type": "REPLY", "user_response": "Here's the workflow."})+ agent_result = agent_module._translate_to_agent_result(+ result, ctx, global_llm_context=None, chat_request=_chat_request(), organization_id="org-1"+ )++ assert agent_result.user_response == "Here's the workflow."+ assert agent_result.updated_workflow is wf+def test_goal_reached_false_on_failed_test_does_not_double_unvalidate(self) -> None:# Failed-test path already routes to unvalidated WIP. A redundant# ``goal_reached: false`` from the agent must not change the outcomediff --git a/tests/unit/test_copilot_response_coverage_gate.py b/tests/unit/test_copilot_response_coverage_gate.pyindex fc8750efb..9049a629c 100644--- a/tests/unit/test_copilot_response_coverage_gate.py+++ b/tests/unit/test_copilot_response_coverage_gate.py@@ -17,8 +17,10 @@ from typing import Anyfrom skyvern.forge.sdk.copilot.enforcement import (MAX_FORMAT_NUDGES,MAX_INTERMEDIATE_NUDGES,+ MAX_NO_WORKFLOW_NUDGES,POST_FORMAT_NUDGE,POST_INTERMEDIATE_SUCCESS_NUDGE,+ POST_NO_WORKFLOW_DELIVERY_NUDGE,_check_enforcement,_is_progress_narration,_response_coverage_nudge,@@ -37,6 +39,7 @@ class _Ctx:self.post_update_nudge_count = 0self.coverage_nudge_count = 0self.format_nudge_count = 0+ self.no_workflow_nudge_count = 0self.user_message = ""self.last_update_block_count = Noneself.last_test_ok = None@@ -126,6 +129,31 @@ def test_reply_before_any_successful_test_passes_through() -> None:assert _response_coverage_nudge(ctx, parsed) is None+def test_reply_claiming_workflow_without_update_fires_nudge() -> None:+ ctx = _Ctx()+ parsed = {"type": "REPLY", "user_response": "Here's the workflow."}++ assert _response_coverage_nudge(ctx, parsed) == POST_NO_WORKFLOW_DELIVERY_NUDGE+ assert ctx.no_workflow_nudge_count == 1+++def test_no_workflow_delivery_nudge_respects_counter_cap() -> None:+ ctx = _Ctx()+ parsed = {"type": "REPLY", "user_response": "I created a workflow for this."}++ for _ in range(MAX_NO_WORKFLOW_NUDGES):+ assert _response_coverage_nudge(ctx, parsed) == POST_NO_WORKFLOW_DELIVERY_NUDGE+ assert _response_coverage_nudge(ctx, parsed) is None+++def test_no_workflow_delivery_nudge_ignores_existing_update_path() -> None:+ ctx = _Ctx()+ ctx.update_workflow_called = True+ parsed = {"type": "REPLY", "user_response": "Here's the workflow."}++ assert _response_coverage_nudge(ctx, parsed) is None++def test_reply_after_failed_test_passes_through() -> None:ctx = _post_success_ctx("go to X and download Y")ctx.last_test_ok = False # test failed