Skip to content

Automate browser based workflows with AI

SKY-9645 Prevent challenge wait blocks (#5881)

Andrew Neilson802122cc0bc5
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.py
index b748b819d..0ab6c4122 100644
--- a/skyvern/forge/sdk/copilot/agent.py
+++ b/skyvern/forge/sdk/copilot/agent.py
@@ -452,8 +452,14 @@ def _translate_to_agent_result(
_detect_stale_block_metadata,
_record_banned_block_reject_span,
_stale_block_metadata_message,
+ _timing_only_challenge_wait_reject_message,
)
+ wait_block_error = _timing_only_challenge_wait_reject_message(ctx, workflow_yaml)
+ if wait_block_error:
+ user_response = f"{user_response}\n\n(Note: {wait_block_error})"
+ ctx.last_test_ok = None
+ workflow_yaml = ""
banned_items = _detect_new_banned_blocks(workflow_yaml, ctx.last_workflow_yaml)
if banned_items:
_record_banned_block_reject_span("replace_workflow_inline", banned_items)
diff --git a/skyvern/forge/sdk/copilot/tools.py b/skyvern/forge/sdk/copilot/tools.py
index cee10ce0d..c20b971b5 100644
--- a/skyvern/forge/sdk/copilot/tools.py
+++ b/skyvern/forge/sdk/copilot/tools.py
@@ -24,7 +24,7 @@ from skyvern.forge.sdk.artifact.models import ArtifactType
from skyvern.forge.sdk.copilot.attribution import resolve_copilot_created_by_stamp
from skyvern.forge.sdk.copilot.block_goal_wrapping import wrap_block_goals
from skyvern.forge.sdk.copilot.context import CopilotContext
-from skyvern.forge.sdk.copilot.enforcement import TOTAL_TIMEOUT_SECONDS
+from skyvern.forge.sdk.copilot.enforcement import PROBABLE_SITE_BLOCK_STREAK_STOP_AT, TOTAL_TIMEOUT_SECONDS
from skyvern.forge.sdk.copilot.failure_tracking import (
PER_TOOL_BUDGET_FAILURE_CATEGORY,
_canonical_block_config,
@@ -592,6 +592,10 @@ async def _update_workflow(params: dict[str, Any], ctx: AgentContext) -> dict[st
if stale_metadata:
return {"ok": False, "error": _stale_block_metadata_message(stale_metadata)}
+ wait_block_error = _timing_only_challenge_wait_reject_message(ctx, workflow_yaml)
+ if wait_block_error:
+ return {"ok": False, "error": wait_block_error}
+
# Post-emission reject of copilot-v2 writes that introduce a banned
# block type. The schema pre_hook only fires when the LLM consults the
# schema; this safety net fires regardless of emission path. Label-based
@@ -2093,6 +2097,65 @@ def _detect_new_banned_blocks(
return [(label, block_type) for label, block_type in submitted_items if label not in prior_labels]
+_CHALLENGE_WAIT_PATTERN = re.compile(
+ r"\b(anti[-_\s]?bot|bot[-_\s]?block|captcha|challenge|cloudflare|ip[-_\s]?block|waf)\b",
+ re.IGNORECASE,
+)
+
+
+def _has_confirmed_waf_or_site_block(ctx: Any) -> bool:
+ if getattr(ctx, "last_test_anti_bot", None):
+ return True
+ return _get_int_attr(ctx, "probable_site_block_streak_count") >= PROBABLE_SITE_BLOCK_STREAK_STOP_AT
+
+
+def _get_int_attr(ctx: Any, name: str, default: int = 0) -> int:
+ value = getattr(ctx, name, default)
+ return value if isinstance(value, int) else default
+
+
+def _block_challenge_wait_text(block: dict[str, Any]) -> str:
+ values = []
+ for key in ("label", "title", "description", "navigation_goal", "complete_criterion"):
+ value = block.get(key)
+ if isinstance(value, str):
+ values.append(value)
+ return " ".join(values)
+
+
+def _detect_timing_only_challenge_wait_blocks(submitted_yaml: str | None) -> list[str]:
+ submitted_blocks = _parse_workflow_blocks(submitted_yaml)
+ if submitted_blocks is None:
+ return []
+ labels: list[str] = []
+ for block in _iter_yaml_blocks(submitted_blocks):
+ raw_type = block.get("block_type")
+ if not isinstance(raw_type, str) or raw_type.strip().lower() != "wait":
+ continue
+ label = block.get("label")
+ if not isinstance(label, str):
+ continue
+ if _CHALLENGE_WAIT_PATTERN.search(_block_challenge_wait_text(block)):
+ labels.append(label)
+ return labels
+
+
+def _timing_only_challenge_wait_reject_message(ctx: Any, submitted_yaml: str | None) -> str | None:
+ if not _has_confirmed_waf_or_site_block(ctx):
+ return None
+ labels = _detect_timing_only_challenge_wait_blocks(submitted_yaml)
+ if not labels:
+ return None
+ labels_text = ", ".join(sorted(set(labels)))
+ return (
+ "Workflow validation failed: timing-only challenge wait blocks are not allowed after confirmed "
+ "anti-bot/WAF or repeated site-block evidence. "
+ f"Offending labels: [{labels_text}]. "
+ "Do not add wait/delay-only blocks for this blocker; use a conditional challenge check that takes a "
+ "real action, try a materially different proxy/source if allowed, or stop and explain the blocker."
+ )
+
+
async def _get_block_schema_pre_hook(
params: dict[str, Any],
ctx: AgentContext,
diff --git a/tests/unit/test_copilot_probable_site_block.py b/tests/unit/test_copilot_probable_site_block.py
index 9dcf864f6..cdcf3aa81 100644
--- a/tests/unit/test_copilot_probable_site_block.py
+++ b/tests/unit/test_copilot_probable_site_block.py
@@ -19,7 +19,10 @@ from skyvern.forge.sdk.copilot.enforcement import (
)
from skyvern.forge.sdk.copilot.tools import (
_detect_probable_site_block_wall,
+ _detect_timing_only_challenge_wait_blocks,
_record_run_blocks_result,
+ _timing_only_challenge_wait_reject_message,
+ _update_workflow,
)
_SCRAPE_WALL_REASON = (
@@ -27,6 +30,48 @@ _SCRAPE_WALL_REASON = (
"unexpectedly or become unresponsive during analysis."
)
+_CHALLENGE_WAIT_WORKFLOW = """
+workflow_definition:
+ blocks:
+ - label: open_page
+ block_type: goto_url
+ url: https://example.com
+ next_block_label: wait_challenge
+ - label: wait_challenge
+ title: Wait for challenge
+ block_type: wait
+ wait_sec: 10
+"""
+
+_GENERIC_WAIT_WORKFLOW = """
+workflow_definition:
+ blocks:
+ - label: wait_for_download
+ title: Wait for download
+ block_type: wait
+ wait_sec: 10
+"""
+
+_CONDITIONAL_ACTION_WORKFLOW = """
+workflow_definition:
+ blocks:
+ - label: check_for_challenge
+ block_type: conditional
+ branch_conditions:
+ - condition_type: prompt
+ condition: If a challenge is visible on the page
+ next_block_label: handle_visible_challenge
+ next_block_label: extract_data
+ - label: handle_visible_challenge
+ title: Handle visible challenge
+ block_type: navigation
+ navigation_goal: Click the visible verification control if present.
+ next_block_label: extract_data
+ - label: extract_data
+ block_type: extraction
+ data_extraction_goal: Extract the requested data.
+"""
+
def _fresh_context() -> CopilotContext:
return CopilotContext(
@@ -298,3 +343,54 @@ def test_nudge_text_is_stop_oriented() -> None:
# Sanity-check the stop nudge tells the agent not to retry.
assert "STOP" in POST_PROBABLE_SITE_BLOCK_STOP_NUDGE
assert "Do NOT" in POST_PROBABLE_SITE_BLOCK_STOP_NUDGE
+
+
+def test_detects_challenge_named_wait_block() -> None:
+ assert _detect_timing_only_challenge_wait_blocks(_CHALLENGE_WAIT_WORKFLOW) == ["wait_challenge"]
+
+
+def test_rejects_challenge_wait_after_explicit_anti_bot_evidence() -> None:
+ ctx = _fresh_context()
+ ctx.last_test_anti_bot = "Cloudflare challenge page detected"
+
+ message = _timing_only_challenge_wait_reject_message(ctx, _CHALLENGE_WAIT_WORKFLOW)
+
+ assert message is not None
+ assert "wait_challenge" in message
+ assert "timing-only challenge wait" in message
+
+
+async def test_update_workflow_rejects_challenge_wait_after_explicit_anti_bot_evidence() -> None:
+ ctx = _fresh_context()
+ ctx.last_test_anti_bot = "Cloudflare challenge page detected"
+
+ result = await _update_workflow({"workflow_yaml": _CHALLENGE_WAIT_WORKFLOW}, ctx)
+
+ assert result["ok"] is False
+ assert "wait_challenge" in str(result["error"])
+
+
+def test_rejects_challenge_wait_after_repeated_scrape_wall() -> None:
+ ctx = _fresh_context()
+ _record_run_blocks_result(ctx, _scrape_wall_result())
+ _record_run_blocks_result(ctx, _scrape_wall_result())
+
+ message = _timing_only_challenge_wait_reject_message(ctx, _CHALLENGE_WAIT_WORKFLOW)
+
+ assert message is not None
+ assert "wait_challenge" in message
+
+
+def test_allows_generic_wait_after_block_evidence() -> None:
+ ctx = _fresh_context()
+ ctx.last_test_anti_bot = "challenge page detected"
+
+ assert _timing_only_challenge_wait_reject_message(ctx, _GENERIC_WAIT_WORKFLOW) is None
+
+
+def test_allows_conditional_challenge_action_after_block_evidence() -> None:
+ ctx = _fresh_context()
+ ctx.last_test_anti_bot = "challenge page detected"
+
+ assert _timing_only_challenge_wait_reject_message(ctx, _CONDITIONAL_ACTION_WORKFLOW) is None