Skip to content

Automate browser based workflows with AI

fix(SKY-9658): prevent unbacked copilot workflow delivery replies (#5890)

Andrew Neilsonf8144c4f7aae
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.py
index 9ef355430..829a021af 100644
--- a/skyvern/forge/sdk/copilot/agent.py
+++ b/skyvern/forge/sdk/copilot/agent.py
@@ -27,7 +27,11 @@ from skyvern.forge import app
from skyvern.forge.prompts import prompt_engine
from skyvern.forge.sdk.copilot.block_goal_wrapping import wrap_block_goals
from skyvern.forge.sdk.copilot.context import COPILOT_RESPONSE_TYPES, AgentResult, CopilotContext, StructuredContext
-from skyvern.forge.sdk.copilot.output_utils import extract_final_text, parse_final_response
+from skyvern.forge.sdk.copilot.output_utils import (
+ extract_final_text,
+ looks_like_workflow_delivery_claim,
+ parse_final_response,
+)
from skyvern.forge.sdk.copilot.tracing_setup import _copilot_model_name, ensure_tracing_initialized, is_tracing_enabled
from skyvern.forge.sdk.schemas.persistent_browser_sessions import is_final_status
from skyvern.forge.sdk.schemas.workflow_copilot import (
@@ -515,6 +519,10 @@ _CANCEL_REPLY_UNVALIDATED = (
"(note: it hasn't been verified end-to-end), or discard."
)
_CANCEL_REPLY_TESTED = "Cancelled. I have a tested draft for you. Accept it to save, or discard."
+_UNBACKED_WORKFLOW_DELIVERY_REPLY = (
+ "I wasn't able to produce a workflow proposal in this turn. Please try again, or provide the missing details "
+ "so I can build and test it."
+)
def _build_wip_exit_result(
@@ -749,6 +757,17 @@ def _translate_to_agent_result(
last_workflow = None
last_workflow_yaml = None
+ last_update_block_count = ctx.last_update_block_count
+ last_test_ok = ctx.last_test_ok
+ if (
+ resp_type == "REPLY"
+ and last_workflow is None
+ and last_update_block_count is None
+ and last_test_ok is None
+ and looks_like_workflow_delivery_claim(user_response)
+ ):
+ user_response = _UNBACKED_WORKFLOW_DELIVERY_REPLY
+
llm_context_raw = action_data.get("global_llm_context")
structured = StructuredContext.from_json_str(global_llm_context)
if isinstance(llm_context_raw, dict):
diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.py
index b89cc860c..d50e30a1a 100644
--- a/skyvern/forge/sdk/copilot/context.py
+++ b/skyvern/forge/sdk/copilot/context.py
@@ -160,6 +160,7 @@ class CopilotContext(AgentContext):
post_update_nudge_count: int = 0
coverage_nudge_count: int = 0
format_nudge_count: int = 0
+ no_workflow_nudge_count: int = 0
copilot_total_timeout_exceeded: bool = False
user_message: str = ""
block_goal_main_goal: str = ""
diff --git a/skyvern/forge/sdk/copilot/enforcement.py b/skyvern/forge/sdk/copilot/enforcement.py
index b4ff5792e..bb306ea57 100644
--- a/skyvern/forge/sdk/copilot/enforcement.py
+++ b/skyvern/forge/sdk/copilot/enforcement.py
@@ -14,7 +14,11 @@ from agents.run import Runner
from skyvern.forge.sdk.copilot.failure_tracking import PER_TOOL_BUDGET_FAILURE_CATEGORY, normalize_failure_reason
from skyvern.forge.sdk.copilot.narration import TransitionKind
-from skyvern.forge.sdk.copilot.output_utils import extract_final_text, parse_final_response
+from skyvern.forge.sdk.copilot.output_utils import (
+ extract_final_text,
+ looks_like_workflow_delivery_claim,
+ parse_final_response,
+)
from skyvern.forge.sdk.copilot.screenshot_utils import ScreenshotEntry
from skyvern.forge.sdk.copilot.tracing_setup import copilot_span
from skyvern.utils.token_counter import count_tokens
@@ -31,6 +35,7 @@ MAX_POST_UPDATE_NUDGES = 2
MAX_INTERMEDIATE_NUDGES = 8
MAX_FAILED_TEST_NUDGES = 2
MAX_FORMAT_NUDGES = 2
+MAX_NO_WORKFLOW_NUDGES = 2
MAX_EXPLORE_WITHOUT_WORKFLOW_NUDGES = 2
# Stops the suspicious-success nudge from re-firing forever when the agent has
# correctly diagnosed an unrecoverable block (anti-bot, paywall) and is no
@@ -257,6 +262,15 @@ POST_PER_TOOL_BUDGET_NUDGE = (
"navigation_goal scope, or reply with a blocker explanation."
)
+POST_NO_WORKFLOW_DELIVERY_NUDGE = (
+ "STOP — you are telling the user you created or are showing a workflow, "
+ "but no workflow update tool has succeeded in this turn. The user will see "
+ "an empty proposal. You MUST either call update_and_run_blocks with a real "
+ "workflow and test it, or respond with ASK_QUESTION if required input is "
+ 'missing. Do NOT say "Here\'s the workflow" until there is an actual '
+ "workflow proposal behind the response."
+)
+
_PROBABLE_SITE_BLOCK_STOP_NUDGE_PREFIX = (
"STOP — the target site has failed to scrape on every attempt across "
"multiple workflow shapes. Every run navigated successfully but the "
@@ -481,6 +495,16 @@ def _response_coverage_nudge(ctx: Any, parsed: dict[str, Any]) -> str | None:
if response_type not in ("REPLY", "REPLACE_WORKFLOW"):
return None
+ if (
+ response_type == "REPLY"
+ and not getattr(ctx, "update_workflow_called", False)
+ and looks_like_workflow_delivery_claim(parsed.get("user_response"))
+ ):
+ nudge_count = getattr(ctx, "no_workflow_nudge_count", 0)
+ if nudge_count < MAX_NO_WORKFLOW_NUDGES:
+ ctx.no_workflow_nudge_count = nudge_count + 1
+ return POST_NO_WORKFLOW_DELIVERY_NUDGE
+
workflow_tested_ok = (
getattr(ctx, "last_test_ok", None) is True
and getattr(ctx, "update_workflow_called", False)
@@ -1013,6 +1037,7 @@ _NUDGE_TYPE_BY_MESSAGE: dict[str, str] = {
POST_ANTI_BOT_FAILED_TEST_NUDGE: "anti_bot_block",
POST_PROBABLE_SITE_BLOCK_STOP_NUDGE: "probable_site_block_stop",
POST_PER_TOOL_BUDGET_NUDGE: "per_tool_budget_split",
+ POST_NO_WORKFLOW_DELIVERY_NUDGE: "no_workflow_delivery",
POST_FAILED_TEST_NUDGE: "post_failed_test",
SCREENSHOT_DROPPED_NUDGE: "screenshot_dropped_on_recovery",
}
diff --git a/skyvern/forge/sdk/copilot/output_utils.py b/skyvern/forge/sdk/copilot/output_utils.py
index 2b0bb021d..9431c14cb 100644
--- a/skyvern/forge/sdk/copilot/output_utils.py
+++ b/skyvern/forge/sdk/copilot/output_utils.py
@@ -47,6 +47,12 @@ _TYPE_ALTERNATION = "|".join(COPILOT_RESPONSE_TYPES)
_LEADING_LABEL_RE = re.compile(rf"^\s*({_TYPE_ALTERNATION})\s*[:,]?\s+", re.IGNORECASE)
_USER_RESPONSE_VALUE_RE = re.compile(r'"user_response"\s*:\s*"((?:[^"\\]|\\.)*)"')
_TYPE_VALUE_RE = re.compile(rf'"type"\s*:\s*"({_TYPE_ALTERNATION})"')
+_WORKFLOW_DELIVERY_CLAIM_PATTERNS = [
+ re.compile(r"\bhere(?:'|’)?s\s+(?:the|a)\s+workflow\b", re.IGNORECASE),
+ re.compile(r"\b(?:created|built|drafted|generated)\s+(?:a|the)\s+(?:draft\s+)?workflow\b", re.IGNORECASE),
+ re.compile(r"\byour\s+workflow\s+(?:is\s+)?(?:ready|complete|completed|set\s+up)\b", re.IGNORECASE),
+ re.compile(r"\bworkflow\s+(?:is\s+)?(?:ready|complete|completed)\b", re.IGNORECASE),
+]
def _try_loads_dict(text: str) -> dict[str, Any] | None:
@@ -130,6 +136,12 @@ def parse_final_response(text: str) -> dict[str, Any]:
return {"type": "REPLY", "user_response": text}
+def looks_like_workflow_delivery_claim(text: Any) -> bool:
+ if not isinstance(text, str) or not text.strip():
+ return False
+ return any(pattern.search(text) for pattern in _WORKFLOW_DELIVERY_CLAIM_PATTERNS)
+
+
def extract_screenshot_b64(result: dict[str, Any]) -> str | None:
"""Extract screenshot_base64 from a copilot result dict, if present."""
data = result.get("data")
diff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.py
index 95301d4fd..978ce0410 100644
--- a/tests/unit/test_copilot_agent_helpers.py
+++ b/tests/unit/test_copilot_agent_helpers.py
@@ -648,6 +648,29 @@ workflow_definition:
assert agent_result.workflow_yaml is None
assert agent_result.unvalidated is False
+ def test_unbacked_workflow_claim_is_rewritten_without_proposal(self) -> None:
+ ctx = _ctx(last_test_ok=None)
+ result = _fake_run_result({"type": "REPLY", "user_response": "Here's the workflow."})
+ agent_result = agent_module._translate_to_agent_result(
+ result, ctx, global_llm_context=None, chat_request=_chat_request(), organization_id="org-1"
+ )
+
+ assert "here's the workflow" not in agent_result.user_response.lower()
+ assert "wasn't able to produce a workflow proposal" in agent_result.user_response
+ assert agent_result.updated_workflow is None
+ assert agent_result.workflow_yaml is None
+
+ def test_unbacked_workflow_claim_not_rewritten_when_proposal_exists(self) -> None:
+ wf = SimpleNamespace(name="drafted")
+ ctx = _ctx(last_workflow=wf, last_workflow_yaml="title: drafted", last_test_ok=True)
+ result = _fake_run_result({"type": "REPLY", "user_response": "Here's the workflow."})
+ agent_result = agent_module._translate_to_agent_result(
+ result, ctx, global_llm_context=None, chat_request=_chat_request(), organization_id="org-1"
+ )
+
+ assert agent_result.user_response == "Here's the workflow."
+ assert agent_result.updated_workflow is wf
+
def test_goal_reached_false_on_failed_test_does_not_double_unvalidate(self) -> None:
# Failed-test path already routes to unvalidated WIP. A redundant
# ``goal_reached: false`` from the agent must not change the outcome
diff --git a/tests/unit/test_copilot_response_coverage_gate.py b/tests/unit/test_copilot_response_coverage_gate.py
index fc8750efb..9049a629c 100644
--- a/tests/unit/test_copilot_response_coverage_gate.py
+++ b/tests/unit/test_copilot_response_coverage_gate.py
@@ -17,8 +17,10 @@ from typing import Any
from skyvern.forge.sdk.copilot.enforcement import (
MAX_FORMAT_NUDGES,
MAX_INTERMEDIATE_NUDGES,
+ MAX_NO_WORKFLOW_NUDGES,
POST_FORMAT_NUDGE,
POST_INTERMEDIATE_SUCCESS_NUDGE,
+ POST_NO_WORKFLOW_DELIVERY_NUDGE,
_check_enforcement,
_is_progress_narration,
_response_coverage_nudge,
@@ -37,6 +39,7 @@ class _Ctx:
self.post_update_nudge_count = 0
self.coverage_nudge_count = 0
self.format_nudge_count = 0
+ self.no_workflow_nudge_count = 0
self.user_message = ""
self.last_update_block_count = None
self.last_test_ok = None
@@ -126,6 +129,31 @@ def test_reply_before_any_successful_test_passes_through() -> None:
assert _response_coverage_nudge(ctx, parsed) is None
+def test_reply_claiming_workflow_without_update_fires_nudge() -> None:
+ ctx = _Ctx()
+ parsed = {"type": "REPLY", "user_response": "Here's the workflow."}
+
+ assert _response_coverage_nudge(ctx, parsed) == POST_NO_WORKFLOW_DELIVERY_NUDGE
+ assert ctx.no_workflow_nudge_count == 1
+
+
+def test_no_workflow_delivery_nudge_respects_counter_cap() -> None:
+ ctx = _Ctx()
+ parsed = {"type": "REPLY", "user_response": "I created a workflow for this."}
+
+ for _ in range(MAX_NO_WORKFLOW_NUDGES):
+ assert _response_coverage_nudge(ctx, parsed) == POST_NO_WORKFLOW_DELIVERY_NUDGE
+ assert _response_coverage_nudge(ctx, parsed) is None
+
+
+def test_no_workflow_delivery_nudge_ignores_existing_update_path() -> None:
+ ctx = _Ctx()
+ ctx.update_workflow_called = True
+ parsed = {"type": "REPLY", "user_response": "Here's the workflow."}
+
+ assert _response_coverage_nudge(ctx, parsed) is None
+
+
def test_reply_after_failed_test_passes_through() -> None:
ctx = _post_success_ctx("go to X and download Y")
ctx.last_test_ok = False # test failed