Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
SKY-9645 Prevent challenge wait blocks (#5881)
Andrew Neilson802122cc0bc5
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.pyindex b748b819d..0ab6c4122 100644--- a/skyvern/forge/sdk/copilot/agent.py+++ b/skyvern/forge/sdk/copilot/agent.py@@ -452,8 +452,14 @@ def _translate_to_agent_result(_detect_stale_block_metadata,_record_banned_block_reject_span,_stale_block_metadata_message,+ _timing_only_challenge_wait_reject_message,)+ wait_block_error = _timing_only_challenge_wait_reject_message(ctx, workflow_yaml)+ if wait_block_error:+ user_response = f"{user_response}\n\n(Note: {wait_block_error})"+ ctx.last_test_ok = None+ workflow_yaml = ""banned_items = _detect_new_banned_blocks(workflow_yaml, ctx.last_workflow_yaml)if banned_items:_record_banned_block_reject_span("replace_workflow_inline", banned_items)diff --git a/skyvern/forge/sdk/copilot/tools.py b/skyvern/forge/sdk/copilot/tools.pyindex cee10ce0d..c20b971b5 100644--- a/skyvern/forge/sdk/copilot/tools.py+++ b/skyvern/forge/sdk/copilot/tools.py@@ -24,7 +24,7 @@ from skyvern.forge.sdk.artifact.models import ArtifactTypefrom skyvern.forge.sdk.copilot.attribution import resolve_copilot_created_by_stampfrom skyvern.forge.sdk.copilot.block_goal_wrapping import wrap_block_goalsfrom skyvern.forge.sdk.copilot.context import CopilotContext-from skyvern.forge.sdk.copilot.enforcement import TOTAL_TIMEOUT_SECONDS+from skyvern.forge.sdk.copilot.enforcement import PROBABLE_SITE_BLOCK_STREAK_STOP_AT, TOTAL_TIMEOUT_SECONDSfrom skyvern.forge.sdk.copilot.failure_tracking import (PER_TOOL_BUDGET_FAILURE_CATEGORY,_canonical_block_config,@@ -592,6 +592,10 @@ async def _update_workflow(params: dict[str, Any], ctx: AgentContext) -> dict[stif stale_metadata:return {"ok": False, "error": _stale_block_metadata_message(stale_metadata)}+ wait_block_error = _timing_only_challenge_wait_reject_message(ctx, workflow_yaml)+ if wait_block_error:+ return {"ok": False, "error": wait_block_error}+# Post-emission reject of copilot-v2 writes that introduce a banned# block type. The schema pre_hook only fires when the LLM consults the# schema; this safety net fires regardless of emission path. Label-based@@ -2093,6 +2097,65 @@ def _detect_new_banned_blocks(return [(label, block_type) for label, block_type in submitted_items if label not in prior_labels]+_CHALLENGE_WAIT_PATTERN = re.compile(+ r"\b(anti[-_\s]?bot|bot[-_\s]?block|captcha|challenge|cloudflare|ip[-_\s]?block|waf)\b",+ re.IGNORECASE,+)+++def _has_confirmed_waf_or_site_block(ctx: Any) -> bool:+ if getattr(ctx, "last_test_anti_bot", None):+ return True+ return _get_int_attr(ctx, "probable_site_block_streak_count") >= PROBABLE_SITE_BLOCK_STREAK_STOP_AT+++def _get_int_attr(ctx: Any, name: str, default: int = 0) -> int:+ value = getattr(ctx, name, default)+ return value if isinstance(value, int) else default+++def _block_challenge_wait_text(block: dict[str, Any]) -> str:+ values = []+ for key in ("label", "title", "description", "navigation_goal", "complete_criterion"):+ value = block.get(key)+ if isinstance(value, str):+ values.append(value)+ return " ".join(values)+++def _detect_timing_only_challenge_wait_blocks(submitted_yaml: str | None) -> list[str]:+ submitted_blocks = _parse_workflow_blocks(submitted_yaml)+ if submitted_blocks is None:+ return []+ labels: list[str] = []+ for block in _iter_yaml_blocks(submitted_blocks):+ raw_type = block.get("block_type")+ if not isinstance(raw_type, str) or raw_type.strip().lower() != "wait":+ continue+ label = block.get("label")+ if not isinstance(label, str):+ continue+ if _CHALLENGE_WAIT_PATTERN.search(_block_challenge_wait_text(block)):+ labels.append(label)+ return labels+++def _timing_only_challenge_wait_reject_message(ctx: Any, submitted_yaml: str | None) -> str | None:+ if not _has_confirmed_waf_or_site_block(ctx):+ return None+ labels = _detect_timing_only_challenge_wait_blocks(submitted_yaml)+ if not labels:+ return None+ labels_text = ", ".join(sorted(set(labels)))+ return (+ "Workflow validation failed: timing-only challenge wait blocks are not allowed after confirmed "+ "anti-bot/WAF or repeated site-block evidence. "+ f"Offending labels: [{labels_text}]. "+ "Do not add wait/delay-only blocks for this blocker; use a conditional challenge check that takes a "+ "real action, try a materially different proxy/source if allowed, or stop and explain the blocker."+ )++async def _get_block_schema_pre_hook(params: dict[str, Any],ctx: AgentContext,diff --git a/tests/unit/test_copilot_probable_site_block.py b/tests/unit/test_copilot_probable_site_block.pyindex 9dcf864f6..cdcf3aa81 100644--- a/tests/unit/test_copilot_probable_site_block.py+++ b/tests/unit/test_copilot_probable_site_block.py@@ -19,7 +19,10 @@ from skyvern.forge.sdk.copilot.enforcement import ()from skyvern.forge.sdk.copilot.tools import (_detect_probable_site_block_wall,+ _detect_timing_only_challenge_wait_blocks,_record_run_blocks_result,+ _timing_only_challenge_wait_reject_message,+ _update_workflow,)_SCRAPE_WALL_REASON = (@@ -27,6 +30,48 @@ _SCRAPE_WALL_REASON = ("unexpectedly or become unresponsive during analysis.")+_CHALLENGE_WAIT_WORKFLOW = """+workflow_definition:+ blocks:+ - label: open_page+ block_type: goto_url+ url: https://example.com+ next_block_label: wait_challenge+ - label: wait_challenge+ title: Wait for challenge+ block_type: wait+ wait_sec: 10+"""++_GENERIC_WAIT_WORKFLOW = """+workflow_definition:+ blocks:+ - label: wait_for_download+ title: Wait for download+ block_type: wait+ wait_sec: 10+"""++_CONDITIONAL_ACTION_WORKFLOW = """+workflow_definition:+ blocks:+ - label: check_for_challenge+ block_type: conditional+ branch_conditions:+ - condition_type: prompt+ condition: If a challenge is visible on the page+ next_block_label: handle_visible_challenge+ next_block_label: extract_data+ - label: handle_visible_challenge+ title: Handle visible challenge+ block_type: navigation+ navigation_goal: Click the visible verification control if present.+ next_block_label: extract_data+ - label: extract_data+ block_type: extraction+ data_extraction_goal: Extract the requested data.+"""+def _fresh_context() -> CopilotContext:return CopilotContext(@@ -298,3 +343,54 @@ def test_nudge_text_is_stop_oriented() -> None:# Sanity-check the stop nudge tells the agent not to retry.assert "STOP" in POST_PROBABLE_SITE_BLOCK_STOP_NUDGEassert "Do NOT" in POST_PROBABLE_SITE_BLOCK_STOP_NUDGE+++def test_detects_challenge_named_wait_block() -> None:+ assert _detect_timing_only_challenge_wait_blocks(_CHALLENGE_WAIT_WORKFLOW) == ["wait_challenge"]+++def test_rejects_challenge_wait_after_explicit_anti_bot_evidence() -> None:+ ctx = _fresh_context()+ ctx.last_test_anti_bot = "Cloudflare challenge page detected"++ message = _timing_only_challenge_wait_reject_message(ctx, _CHALLENGE_WAIT_WORKFLOW)++ assert message is not None+ assert "wait_challenge" in message+ assert "timing-only challenge wait" in message+++async def test_update_workflow_rejects_challenge_wait_after_explicit_anti_bot_evidence() -> None:+ ctx = _fresh_context()+ ctx.last_test_anti_bot = "Cloudflare challenge page detected"++ result = await _update_workflow({"workflow_yaml": _CHALLENGE_WAIT_WORKFLOW}, ctx)++ assert result["ok"] is False+ assert "wait_challenge" in str(result["error"])+++def test_rejects_challenge_wait_after_repeated_scrape_wall() -> None:+ ctx = _fresh_context()+ _record_run_blocks_result(ctx, _scrape_wall_result())+ _record_run_blocks_result(ctx, _scrape_wall_result())++ message = _timing_only_challenge_wait_reject_message(ctx, _CHALLENGE_WAIT_WORKFLOW)++ assert message is not None+ assert "wait_challenge" in message+++def test_allows_generic_wait_after_block_evidence() -> None:+ ctx = _fresh_context()+ ctx.last_test_anti_bot = "challenge page detected"++ assert _timing_only_challenge_wait_reject_message(ctx, _GENERIC_WAIT_WORKFLOW) is None+++def test_allows_conditional_challenge_action_after_block_evidence() -> None:+ ctx = _fresh_context()+ ctx.last_test_anti_bot = "challenge page detected"++ assert _timing_only_challenge_wait_reject_message(ctx, _CONDITIONAL_ACTION_WORKFLOW) is None