Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Carry obstruction controls through build-test results (SKY-14614) (#8293)
Andrew Neilsone8dd2d2121f6
diff --git a/skyvern/forge/sdk/copilot/build_test_outcome.py b/skyvern/forge/sdk/copilot/build_test_outcome.pyindex 5403423d5..e8d6877fb 100644--- a/skyvern/forge/sdk/copilot/build_test_outcome.py+++ b/skyvern/forge/sdk/copilot/build_test_outcome.py@@ -23,7 +23,7 @@ from skyvern.forge.sdk.copilot.completion_verification import (only_degraded_blocking,)from skyvern.forge.sdk.copilot.composition_evidence import page_evidence_source_matches_run, workflow_target_url-from skyvern.forge.sdk.copilot.context import CodeAuthoringRepairContext+from skyvern.forge.sdk.copilot.context import CodeAuthoringRepairContext, PageObstructionfrom skyvern.forge.sdk.copilot.failure_tracking import selector_identities_in_text, selector_identity_from_failurefrom skyvern.forge.sdk.copilot.request_policy import redact_raw_secrets_for_promptfrom skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcome@@ -127,6 +127,7 @@ class BuildTestPacketPageState(BaseModel):action_summaries: list[str] = Field(default_factory=list)challenge_summaries: list[str] = Field(default_factory=list)obstruction_summaries: list[str] = Field(default_factory=list)+ obstructions: list[PageObstruction] = Field(default_factory=list)class BuildTestPacketFailure(BaseModel):diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.pyindex 8cc3642ee..a0563d331 100644--- a/skyvern/forge/sdk/copilot/context.py+++ b/skyvern/forge/sdk/copilot/context.py@@ -199,6 +199,49 @@ _RETIRED_INTERACTION_FIELDS = frozenset({"typed_value"})OUTPUT_OWNER_AMBIGUITY_REASON_CODE = "output_owner_ambiguous"+class PageObstructionSelectorCandidate(BaseModel):+ model_config = ConfigDict(extra="forbid", frozen=True)++ selector: str+ source: str+++class PageObstructionIdentity(BaseModel):+ model_config = ConfigDict(extra="forbid", frozen=True)++ tag: str+ role: str+ label_context: str+++class PageObstructionControl(BaseModel):+ """A producer-bounded control plus scalar capture extensions not known to this consumer."""++ model_config = ConfigDict(extra="allow", frozen=True)++ __pydantic_extra__: dict[str, str | bool | int | float] = Field(init=False)+ tag: str | None = None+ text: str | None = None+ aria_label: str | None = None+ title: str | None = None+ selector: str | None = None+ type: str | None = None+ selector_candidates: list[PageObstructionSelectorCandidate] = Field(default_factory=list)+ identity: PageObstructionIdentity | None = None+++class PageObstruction(BaseModel):+ model_config = ConfigDict(extra="forbid", frozen=True)++ kind: str | None = None+ source: str | None = None+ selector: str | None = None+ text: str | None = None+ visual_location: str | None = None+ underlying_page_blocked: bool | None = None+ visible_controls: list[PageObstructionControl] = Field(default_factory=list)++class CodeAuthoringRepairContext(BaseModel):block_label: strreason_code: str@@ -233,6 +276,8 @@ class CodeAuthoringRepairContext(BaseModel):page_action_summaries: list[str] = Field(default_factory=list)page_challenge_summaries: list[str] = Field(default_factory=list)page_obstruction_summaries: list[str] = Field(default_factory=list)+ page_obstructions: list[PageObstruction] = Field(default_factory=list)+ page_obstruction_omission_notices: list[str] = Field(default_factory=list)required_block_structure: str = ""spine_stage_count: int | None = Nonespine_split_blockers: list[str] = Field(default_factory=list)diff --git a/skyvern/forge/sdk/copilot/output_utils.py b/skyvern/forge/sdk/copilot/output_utils.pyindex 77166046e..1a6ac8e7a 100644--- a/skyvern/forge/sdk/copilot/output_utils.py+++ b/skyvern/forge/sdk/copilot/output_utils.py@@ -14,7 +14,13 @@ import structlogfrom skyvern.forge.sdk.agents.context import sanitize_agent_tool_result_for_llm as sanitize_generic_tool_result_for_llmfrom skyvern.forge.sdk.copilot.blocker_signal import CopilotToolBlockerSignal, assert_clean_user_facing_text-from skyvern.forge.sdk.copilot.context import COPILOT_RESPONSE_TYPES+from skyvern.forge.sdk.copilot.context import (+ COPILOT_RESPONSE_TYPES,+ PageObstruction,+ PageObstructionControl,+ PageObstructionIdentity,+ PageObstructionSelectorCandidate,+)from skyvern.forge.sdk.copilot.failure_tracking import (PER_TOOL_BUDGET_FAILURE_CATEGORY,)@@ -44,6 +50,11 @@ _BUILD_TEST_UNFINISHED_MAX_ITEMS = 24_BUILD_TEST_ACTION_TRACE_MAX_ITEMS = 6_BUILD_TEST_PAGE_SUMMARY_MAX_ITEMS = 8_BUILD_TEST_PAGE_SUMMARY_MAX_CHARS = 300+_BUILD_TEST_OBSTRUCTION_MAX_ITEMS = 5+_BUILD_TEST_OBSTRUCTION_CONTROL_MAX_ITEMS = 6+_BUILD_TEST_SELECTOR_CANDIDATE_MAX_ITEMS = 8+_BUILD_TEST_OBSTRUCTION_VALUE_MAX_CHARS = 240+_BUILD_TEST_IDENTITY_LABEL_MAX_CHARS = 2_048_PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n"_JPEG_PREFIX = b"\xff\xd8\xff"@@ -312,6 +323,125 @@ def _bounded_packet_strings(return rendered+def _bounded_obstruction_control(+ control: PageObstructionControl,+ *,+ obstruction_index: int,+ control_index: int,+ notices: list[str],+) -> PageObstructionControl:+ field_prefix = f"failure.page_state.obstructions[{obstruction_index}].visible_controls[{control_index}]"+ candidates = control.selector_candidates[:_BUILD_TEST_SELECTOR_CANDIDATE_MAX_ITEMS]+ if len(control.selector_candidates) > _BUILD_TEST_SELECTOR_CANDIDATE_MAX_ITEMS:+ _append_omission(+ notices,+ f"{field_prefix}.selector_candidates shortened: "+ f"{len(control.selector_candidates) - _BUILD_TEST_SELECTOR_CANDIDATE_MAX_ITEMS} item(s) omitted.",+ )+ bounded_candidates = [+ PageObstructionSelectorCandidate(+ selector=_bounded_packet_string(+ candidate.selector,+ field_name=f"{field_prefix}.selector_candidates[].selector",+ max_chars=_BUILD_TEST_IDENTIFIER_MAX_CHARS,+ notices=notices,+ )+ or "",+ source=_bounded_packet_string(+ candidate.source,+ field_name=f"{field_prefix}.selector_candidates[].source",+ max_chars=40,+ notices=notices,+ )+ or "",+ )+ for candidate in candidates+ ]+ identity = control.identity+ if identity is not None:+ identity = PageObstructionIdentity(+ tag=_bounded_packet_string(+ identity.tag,+ field_name=f"{field_prefix}.identity.tag",+ max_chars=40,+ notices=notices,+ )+ or "",+ role=_bounded_packet_string(+ identity.role,+ field_name=f"{field_prefix}.identity.role",+ max_chars=40,+ notices=notices,+ )+ or "",+ label_context=_bounded_packet_string(+ identity.label_context,+ field_name=f"{field_prefix}.identity.label_context",+ max_chars=_BUILD_TEST_IDENTITY_LABEL_MAX_CHARS,+ notices=notices,+ )+ or "",+ )+ values = control.model_dump(mode="json", exclude_none=True)+ for key, value in tuple(values.items()):+ if key in {"selector_candidates", "identity"} or not isinstance(value, str):+ continue+ max_chars = _BUILD_TEST_IDENTIFIER_MAX_CHARS if key == "selector" else _BUILD_TEST_OBSTRUCTION_VALUE_MAX_CHARS+ values[key] = _bounded_packet_string(+ value,+ field_name=f"{field_prefix}.{key}",+ max_chars=max_chars,+ notices=notices,+ )+ values["selector_candidates"] = bounded_candidates+ values["identity"] = identity+ return PageObstructionControl.model_validate(values)+++def _bounded_page_obstructions(obstructions: list[PageObstruction], notices: list[str]) -> list[PageObstruction]:+ bounded_obstructions = obstructions[:_BUILD_TEST_OBSTRUCTION_MAX_ITEMS]+ if len(obstructions) > _BUILD_TEST_OBSTRUCTION_MAX_ITEMS:+ _append_omission(+ notices,+ "failure.page_state.obstructions shortened: "+ f"{len(obstructions) - _BUILD_TEST_OBSTRUCTION_MAX_ITEMS} item(s) omitted.",+ )+ projected: list[PageObstruction] = []+ for obstruction_index, obstruction in enumerate(bounded_obstructions):+ field_prefix = f"failure.page_state.obstructions[{obstruction_index}]"+ controls = obstruction.visible_controls[:_BUILD_TEST_OBSTRUCTION_CONTROL_MAX_ITEMS]+ if len(obstruction.visible_controls) > _BUILD_TEST_OBSTRUCTION_CONTROL_MAX_ITEMS:+ _append_omission(+ notices,+ f"{field_prefix}.visible_controls shortened: "+ f"{len(obstruction.visible_controls) - _BUILD_TEST_OBSTRUCTION_CONTROL_MAX_ITEMS} item(s) omitted.",+ )+ values = obstruction.model_dump(mode="json", exclude_none=True)+ for key, value in tuple(values.items()):+ if key in {"visible_controls", "underlying_page_blocked"} or not isinstance(value, str):+ continue+ max_chars = (+ _BUILD_TEST_IDENTIFIER_MAX_CHARS if key == "selector" else _BUILD_TEST_OBSTRUCTION_VALUE_MAX_CHARS+ )+ values[key] = _bounded_packet_string(+ value,+ field_name=f"{field_prefix}.{key}",+ max_chars=max_chars,+ notices=notices,+ )+ values["visible_controls"] = [+ _bounded_obstruction_control(+ control,+ obstruction_index=obstruction_index,+ control_index=control_index,+ notices=notices,+ )+ for control_index, control in enumerate(controls)+ ]+ projected.append(PageObstruction.model_validate(values))+ return projected++def _bounded_packet_page_state(page_state: BuildTestPacketPageState | None, notices: list[str]) -> BuildTestPacketPageState | None:@@ -377,6 +507,7 @@ def _bounded_packet_page_state(max_chars=_BUILD_TEST_PAGE_SUMMARY_MAX_CHARS,notices=notices,),+ "obstructions": _bounded_page_obstructions(page_state.obstructions, notices),}return page_state.model_copy(update=updates)@@ -406,6 +537,7 @@ def _compact_packet_for_aggregate_limit(if failure is not None:page_state = failure.page_stateif page_state is not None:+ had_obstructions = bool(page_state.obstructions)def compact_summaries(values: list[str]) -> list[str]:return [value[:117] + "..." if len(value) > 120 else value for value in values[:2]]@@ -417,8 +549,14 @@ def _compact_packet_for_aggregate_limit("action_summaries": compact_summaries(page_state.action_summaries),"challenge_summaries": compact_summaries(page_state.challenge_summaries),"obstruction_summaries": compact_summaries(page_state.obstruction_summaries),+ "obstructions": [],})+ if had_obstructions:+ _append_omission(+ notices,+ "failure.page_state.obstructions omitted at the aggregate packet limit.",+ )failure = failure.model_copy(update={"action_trace": [@@ -749,6 +887,12 @@ def sanitize_tool_result_for_llm(tool_name: str, result: dict[str, Any]) -> dictelse blockfor block in blocks]+ repair_context = data.get("authoring_repair_context")+ if isinstance(repair_context, dict):+ repair_context = dict(repair_context)+ repair_context.pop("page_obstructions", None)+ repair_context.pop("page_obstruction_omission_notices", None)+ data["authoring_repair_context"] = repair_contextraw_packet = data.get(BUILD_TEST_PACKET_KEY)if isinstance(raw_packet, dict):from skyvern.forge.sdk.copilot.build_test_outcome import BuildTestEvidencePacketdiff --git a/skyvern/forge/sdk/copilot/runtime_authoring_repair.py b/skyvern/forge/sdk/copilot/runtime_authoring_repair.pyindex 2c8bf1812..4c3168909 100644--- a/skyvern/forge/sdk/copilot/runtime_authoring_repair.py+++ b/skyvern/forge/sdk/copilot/runtime_authoring_repair.py@@ -5,6 +5,7 @@ from typing import Any, TypeGuardfrom urllib.parse import urlsplit, urlunsplitimport structlog+from pydantic import ValidationErrorfrom skyvern.forge.sdk.copilot.challenge_evidence import (RUNTIME_SOLVABLE_CHALLENGE_KINDS,@@ -15,7 +16,7 @@ from skyvern.forge.sdk.copilot.challenge_evidence import ()from skyvern.forge.sdk.copilot.composition_evidence import has_bounded_page_schemafrom skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy, normalize_block_authoring_policy-from skyvern.forge.sdk.copilot.context import CodeAuthoringRepairContext+from skyvern.forge.sdk.copilot.context import CodeAuthoringRepairContext, PageObstructionfrom skyvern.forge.sdk.copilot.output_contracts import code_block_available_contracts_by_labelfrom skyvern.forge.sdk.copilot.request_policy import redact_raw_secrets_for_promptfrom skyvern.forge.sdk.copilot.run_outcome import trusted_terminal_challenge_category_name@@ -198,24 +199,47 @@ def _runtime_result_summaries(value: Any) -> list[str]:return summaries[:_RUNTIME_SUMMARY_MAX_ITEMS]-def _obstruction_entries(evidence: dict[str, Any]) -> list[dict[str, Any]]:- page_obstructions = evidence.get("page_obstructions")- if isinstance(page_obstructions, list):- entries = [entry for entry in page_obstructions if isinstance(entry, dict)]- if entries:- return entries+def _raw_obstruction_entries(evidence: dict[str, Any]) -> tuple[list[Any], list[str]]:+ if "page_obstructions" in evidence:+ page_obstructions = evidence.get("page_obstructions")+ if isinstance(page_obstructions, list):+ return page_obstructions, []+ return [], ["failure.page_state.obstructions omitted: canonical page_obstructions was malformed."]modal_overlays = evidence.get("modal_overlays")if not isinstance(modal_overlays, list):- return []- return [- {"selector": overlay.get("selector"), "visible_controls": overlay.get("dismiss_controls")}- for overlay in modal_overlays- if isinstance(overlay, dict)- ]+ return [], []+ return (+ [+ {"selector": overlay.get("selector"), "visible_controls": overlay.get("dismiss_controls") or []}+ for overlay in modal_overlays+ if isinstance(overlay, dict)+ ],+ [],+ )+++def _typed_runtime_page_obstructions(evidence: Any) -> tuple[list[PageObstruction], list[str]]:+ if not isinstance(evidence, dict):+ return [], []+ raw_entries, notices = _raw_obstruction_entries(evidence)+ obstructions: list[PageObstruction] = []+ malformed = 0+ for entry in raw_entries:+ if not isinstance(entry, dict):+ malformed += 1+ continue+ try:+ obstructions.append(PageObstruction.model_validate(entry))+ except ValidationError:+ malformed += 1+ if malformed:+ notices.append(f"failure.page_state.obstructions omitted: {malformed} malformed item(s).")+ return obstructions, noticesdef _has_page_obstruction(evidence: dict[str, Any]) -> bool:- return bool(_obstruction_entries(evidence))+ obstructions, _ = _typed_runtime_page_obstructions(evidence)+ return bool(obstructions)def repair_page_evidence_is_admissible(evidence: dict[str, Any]) -> bool:@@ -235,11 +259,10 @@ def _joined_obstruction_summary(obstruction: str, control: str) -> str:return f"{obstruction[:budget]} {control}".strip()-def _runtime_obstruction_summaries(evidence: Any) -> list[str]:- if not isinstance(evidence, dict):- return []+def _runtime_obstruction_summaries(obstructions: list[PageObstruction]) -> list[str]:summaries: list[str] = []- for entry in _obstruction_entries(evidence)[:_RUNTIME_SUMMARY_MAX_ITEMS]:+ for obstruction_entry in obstructions[:_RUNTIME_SUMMARY_MAX_ITEMS]:+ entry = obstruction_entry.model_dump(mode="json", exclude_none=True)obstruction = _runtime_summary_entry(entry, _OBSTRUCTION_KEYS, _OBSTRUCTION_FIELD_MAX_CHARS, OBSTRUCTION_SUMMARY_MAX_CHARS)@@ -517,7 +540,8 @@ def finalize_runtime_authoring_repair_context_from_page_observation(page_challenge_summaries = _runtime_summary_list(evidence.get("challenge_controls"), ("text", "selector", "disabled"))- page_obstruction_summaries = _runtime_obstruction_summaries(evidence)+ page_obstructions, page_obstruction_omission_notices = _typed_runtime_page_obstructions(evidence)+ page_obstruction_summaries = _runtime_obstruction_summaries(page_obstructions)finalized = pending.model_copy(update={"current_origin": _origin_from_runtime_url(current_url),@@ -529,13 +553,15 @@ def finalize_runtime_authoring_repair_context_from_page_observation(or page_result_summariesor page_action_summariesor page_challenge_summaries- or page_obstruction_summaries+ or page_obstructions),"page_form_summaries": page_form_summaries,"page_result_summaries": page_result_summaries,"page_action_summaries": page_action_summaries,"page_challenge_summaries": page_challenge_summaries,"page_obstruction_summaries": page_obstruction_summaries,+ "page_obstructions": page_obstructions,+ "page_obstruction_omission_notices": page_obstruction_omission_notices,})copilot_ctx.last_code_authoring_repair_context = finalized@@ -572,5 +598,6 @@ def inject_runtime_authoring_repair_context(copilot_ctx: Any, result: dict[str,page_result_summary_count=len(repair_context.page_result_summaries),page_action_summary_count=len(repair_context.page_action_summaries),page_obstruction_summary_count=len(repair_context.page_obstruction_summaries),+ page_obstruction_count=len(repair_context.page_obstructions),)data["authoring_repair_context"] = repair_context.model_dump(mode="json")diff --git a/skyvern/forge/sdk/copilot/tools/run_execution.py b/skyvern/forge/sdk/copilot/tools/run_execution.pyindex 4ec3cf305..97fc06904 100644--- a/skyvern/forge/sdk/copilot/tools/run_execution.py+++ b/skyvern/forge/sdk/copilot/tools/run_execution.py@@ -63,7 +63,7 @@ from skyvern.forge.sdk.copilot.completion_verification import ()from skyvern.forge.sdk.copilot.composition_evidence import has_bounded_page_schema, parse_composition_htmlfrom skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy-from skyvern.forge.sdk.copilot.context import CopilotContext+from skyvern.forge.sdk.copilot.context import CopilotContext, PageObstructionfrom skyvern.forge.sdk.copilot.diagnosis_repair_contract import (DiagnosisRepairContract,build_diagnosis_repair_contract,@@ -3700,6 +3700,26 @@ def _packet_string_list(value: Any) -> list[str]:return [item for item in value if isinstance(item, str) and item] if isinstance(value, list) else []+def _packet_page_obstructions(value: Any, omission_notices: list[str]) -> list[PageObstruction]:+ if value is None:+ return []+ if not isinstance(value, list):+ omission_notices.append("failure.page_state.obstructions omitted: repair-context value was malformed.")+ return []+ obstructions: list[PageObstruction] = []+ malformed = 0+ for item in value:+ try:+ obstructions.append(PageObstruction.model_validate(item))+ except ValueError:+ malformed += 1+ if malformed:+ omission_notices.append(+ f"failure.page_state.obstructions omitted: {malformed} malformed repair-context item(s)."+ )+ return obstructions++def _packet_workflow_readback(copilot_ctx: CopilotContext) -> tuple[str | None, str]:accepted = copilot_ctx.last_workflow_yamlif isinstance(accepted, str) and accepted.strip():@@ -3723,6 +3743,7 @@ def _packet_page_state(data: Mapping[str, Any], omission_notices: list[str]) ->if isinstance(repair_context, Mapping) and repair_run_id != run_id:omission_notices.append("failure.page_state omitted repair-context fields belonging to another or unknown run.")repair = repair_context if isinstance(repair_context, Mapping) and repair_run_id == run_id else {}+ omission_notices.extend(_packet_string_list(repair.get("page_obstruction_omission_notices")))current_url = _packet_string(repair.get("current_url")) or _packet_string(data.get("current_url"))title = _packet_string(repair.get("current_title")) or _packet_string(data.get("page_title"))page_state = BuildTestPacketPageState(@@ -3736,6 +3757,7 @@ def _packet_page_state(data: Mapping[str, Any], omission_notices: list[str]) ->action_summaries=_packet_string_list(repair.get("page_action_summaries")),challenge_summaries=_packet_string_list(repair.get("page_challenge_summaries")),obstruction_summaries=_packet_string_list(repair.get("page_obstruction_summaries")),+ obstructions=_packet_page_obstructions(repair.get("page_obstructions"), omission_notices),)return (page_state@@ -3750,6 +3772,7 @@ def _packet_page_state(data: Mapping[str, Any], omission_notices: list[str]) ->page_state.action_summaries,page_state.challenge_summaries,page_state.obstruction_summaries,+ page_state.obstructions,))else Nonediff --git a/skyvern/forge/taskv3/captcha_tools.py b/skyvern/forge/taskv3/captcha_tools.pyindex 322843648..9a4d5aecd 100644--- a/skyvern/forge/taskv3/captcha_tools.py+++ b/skyvern/forge/taskv3/captcha_tools.py@@ -22,10 +22,11 @@ from skyvern.webeye.utils.captcha_solver import CaptchaChallengeUnsolvedError, sLOG = structlog.get_logger()-# The ladder's own arms are already bounded (anchor 5s + extension 12s + token 90s). This is a safety-# ceiling above their worst-case sum, well under the run deadline, so a wedged solver cannot outlive the-# tool. NB: should_cancel is checked by the loop BETWEEN tool calls, not inside a handler, so a solve in-# flight blocks cancellation for up to this ceiling (accepted; v1's cascade blocks up to 600s).+# The ladder clamps its bounded arms to _LADDER_BUDGET_SECONDS (110s), but its DOM-checkbox arm relies on+# Playwright's own click timeouts rather than that budget; this ceiling is the hard stop above both, well+# under the run deadline, so a wedged solver cannot outlive the tool. NB: should_cancel is checked by the+# loop BETWEEN tool calls, not inside a handler, so a solve in flight blocks cancellation for up to this+# ceiling (accepted; v1's cascade blocks up to 600s)._SOLVE_CAPTCHA_CEILING_SECONDS = 120# A pathological loop must not keep running the solver on an unsolvable gate. Bounds CONSECUTIVE FAILED@@ -105,8 +106,8 @@ def build_captcha_tools(tool = ToolSpec(name="solve_captcha",description=(- "Detect and solve a captcha / anti-bot challenge (Cloudflare Turnstile or reCAPTCHA) blocking "- "the page, including one rendered inside an iframe you cannot otherwise interact with. Call "+ "Detect and solve a captcha / anti-bot challenge (Cloudflare Turnstile, reCAPTCHA, or hCaptcha) "+ "blocking the page, including one rendered inside an iframe you cannot otherwise interact with. Call ""this when a submit does not advance or a 'verify you are human' challenge is present, then ""re-observe. Returns whether a captcha was solved, was absent, or could not be solved."),diff --git a/skyvern/webeye/scraper/scraped_page.py b/skyvern/webeye/scraper/scraped_page.pyindex 123ff99cc..fa6c357fa 100644--- a/skyvern/webeye/scraper/scraped_page.py+++ b/skyvern/webeye/scraper/scraped_page.py@@ -65,7 +65,7 @@ def json_to_html(element: dict, need_skyvern_attrs: bool = True) -> str:if element is flagged as dropped, the html format is empty"""tag = element["tagName"]- attributes: dict[str, Any] = copy.deepcopy(element.get("attributes", {}))+ attributes: dict[str, Any] = dict(element.get("attributes", {}))interactable = element.get("interactable", False)if element.get("isDropped", False):diff --git a/skyvern/webeye/utils/captcha_solver.py b/skyvern/webeye/utils/captcha_solver.pyindex 76bf01096..ae0b71089 100644--- a/skyvern/webeye/utils/captcha_solver.py+++ b/skyvern/webeye/utils/captcha_solver.py@@ -13,6 +13,7 @@ so this module stays OSS-clean: the OSS bases return False and the cloud overridfrom __future__ import annotationsimport asyncio+import timefrom typing import TYPE_CHECKING, Anyfrom urllib.parse import urlparse@@ -47,12 +48,16 @@ _CAPTCHA_MARKER_SELECTOR = ", ".join(".cf-turnstile",".g-recaptcha[data-sitekey]",".cf-turnstile[data-sitekey]",+ ".h-captcha",+ ".h-captcha[data-sitekey]",'iframe[src*="recaptcha" i]','iframe[src*="turnstile" i]',+ 'iframe[src*="hcaptcha" i]','iframe[title*="recaptcha" i]','iframe[title*="challenge" i]',))+_HCAPTCHA_MARKER_SELECTOR = ", ".join((".h-captcha", ".h-captcha[data-sitekey]", 'iframe[src*="hcaptcha" i]'))_RECAPTCHA_MARKER_SELECTOR = ", ".join((".g-recaptcha",@@ -67,10 +72,18 @@ _RECAPTCHA_ANCHOR_PATHS = ("/recaptcha/api2/anchor", "/recaptcha/enterprise/anch_RECAPTCHA_ANCHOR_ARM_TIMEOUT_SECONDS = 5# The extension arm polls a solver over the network; the scout caller has no enclosing bound._EXTENSION_ARM_TIMEOUT_SECONDS = 12+# hCaptcha is solved by the extension through an image challenge that takes tens of seconds, so the+# Turnstile-sized bound would cut nearly every solve.+_HCAPTCHA_ARM_TIMEOUT_SECONDS = 90# A correct solver task returns in ~25s, so this covers one with headroom while cutting the losing task# of the pair, which only ends at its own 180s timeout._TOKEN_ARM_TIMEOUT_SECONDS = 90_WIDGET_RESET_TIMEOUT_SECONDS = 3+# Sum of the bounded arms on a non-hCaptcha page (anchor 5 + reset 3 + extension 12 + token 90); the token+# arm is clamped to what remains so a 90s hCaptcha extension arm cannot push the bounded arms past the v3+# tool's 120s ceiling. On a page carrying both hCaptcha and reCAPTCHA markers the token arm may therefore+# get less than a full solve needs; hCaptcha is the visible gate there.+_LADDER_BUDGET_SECONDS = 110# Google's widget flips aria-checked after its own animation; a shorter wait reads as unsolved._RECAPTCHA_ANCHOR_SETTLE_MS = 2_000_CAPTCHA_CONTINUE_SELECTOR = ", ".join(@@ -130,6 +143,7 @@ async def solve_challenge_ladder(callers do not re-perceive a page nothing touched. Raises CaptchaChallengeUnsolvedError when achallenge was present but no arm resolved it."""+ start = time.monotonic()checkbox = page.locator(_CAPTCHA_CHECKBOX_SELECTOR)checkbox_count = await _bounded_locator_count(checkbox)marker = page.locator(_CAPTCHA_MARKER_SELECTOR)@@ -223,8 +237,10 @@ async def solve_challenge_ladder(except Exception:LOG.info("CAPTCHA widget reset did not run", arm="recaptcha_anchor_frame")+ hcaptcha_present = await _bounded_locator_count(page.locator(_HCAPTCHA_MARKER_SELECTOR)) > 0+ extension_timeout = _HCAPTCHA_ARM_TIMEOUT_SECONDS if hcaptcha_present else _EXTENSION_ARM_TIMEOUT_SECONDStry:- async with asyncio.timeout(_EXTENSION_ARM_TIMEOUT_SECONDS):+ async with asyncio.timeout(extension_timeout):if await app.AGENT_FUNCTION.auto_solve_captchas(page):return Trueexcept Exception:@@ -232,16 +248,21 @@ async def solve_challenge_ladder(recaptcha = page.locator(_RECAPTCHA_MARKER_SELECTOR)if await _bounded_locator_count(recaptcha) > 0:- try:- async with asyncio.timeout(_TOKEN_ARM_TIMEOUT_SECONDS):- if await app.AGENT_FUNCTION.solve_recaptcha_token(- page,- organization_id=organization_id,- workflow_run_id=workflow_run_id,- browser_session_id=browser_session_id,- ):- return True- except Exception:- LOG.info("CAPTCHA token arm did not solve", arm="token")+ remaining = _LADDER_BUDGET_SECONDS - (time.monotonic() - start)+ token_timeout = min(_TOKEN_ARM_TIMEOUT_SECONDS, remaining)+ if token_timeout <= 0:+ LOG.info("CAPTCHA token arm skipped: ladder budget exhausted", arm="token")+ else:+ try:+ async with asyncio.timeout(token_timeout):+ if await app.AGENT_FUNCTION.solve_recaptcha_token(+ page,+ organization_id=organization_id,+ workflow_run_id=workflow_run_id,+ browser_session_id=browser_session_id,+ ):+ return True+ except Exception:+ LOG.info("CAPTCHA token arm did not solve", arm="token")raise CaptchaChallengeUnsolvedError("CAPTCHA could not be solved.")diff --git a/tests/unit/test_code_block_captcha.py b/tests/unit/test_code_block_captcha.pyindex 282a45e2f..a2509fb95 100644--- a/tests/unit/test_code_block_captcha.py+++ b/tests/unit/test_code_block_captcha.py@@ -11,6 +11,7 @@ from skyvern.forge.agent_functions import AgentFunctionfrom skyvern.forge.sdk.workflow.models import block as block_modulefrom skyvern.forge.sdk.workflow.models.block import CodeBlock, CodeBlockCaptchaErrorfrom skyvern.webeye.utils import captcha_solver as captcha_solver_module+from skyvern.webeye.utils.captcha_solver import CaptchaChallengeUnsolvedError, solve_challenge_ladderclass FakeLocator:@@ -86,6 +87,8 @@ class FakePage:*,checkbox: bool = False,recaptcha: bool = False,+ hcaptcha: bool = False,+ turnstile: bool = False,token_values: list[str] | None = None,frames: list[FakeFrame] | None = None,url: str = "https://app.example/login",@@ -95,6 +98,8 @@ class FakePage:self.recaptcha_token = FakeLocator(count=1 if token_values is not None else 0, input_values=token_values)self.continue_button = FakeLocator(count=1 if checkbox else 0)self.continue_button.click = AsyncMock(side_effect=self._continue)+ self.hcaptcha_marker = FakeLocator(count=1 if hcaptcha else 0)+ self.marker = FakeLocator(count=1 if (recaptcha or hcaptcha or turnstile) else 0)self.frames = list(frames or [])self.url = urlself.evaluated: list[str] = []@@ -110,6 +115,10 @@ class FakePage:return self.checkboxif "button" in selector:return self.continue_button+ if selector == captcha_solver_module._HCAPTCHA_MARKER_SELECTOR:+ return self.hcaptcha_marker+ if selector == captcha_solver_module._CAPTCHA_MARKER_SELECTOR:+ return self.markerreturn self.challengeasync def wait_for_timeout(self, _milliseconds: int) -> None:@@ -643,3 +652,114 @@ async def test_an_anchor_click_that_left_a_token_is_not_reset_away(monkeypatch:await block_module._code_block_solve_captcha_builtin(page, organization_id="org-1")assert not any("grecaptcha" in expression for expression in page.evaluated)+++async def test_hcaptcha_marker_reaches_extension_arm(monkeypatch: pytest.MonkeyPatch) -> None:+ """A page whose only marker is an hCaptcha iframe must still reach the extension arm.++ Before hCaptcha markers were recognized, the initial probe saw no known marker and the ladder+ returned False without ever awaiting the solver.+ """+ agent_function = type(+ "AgentFunctionStub",+ (AgentFunction,),+ {+ "auto_solve_captchas": AsyncMock(return_value=True),+ "solve_recaptcha_token": AsyncMock(return_value=False),+ },+ )()+ monkeypatch.setattr(app, "AGENT_FUNCTION", agent_function)+ page = FakePage(hcaptcha=True)++ assert await solve_challenge_ladder(page) is True+ agent_function.auto_solve_captchas.assert_awaited_once_with(page)+++async def test_hcaptcha_arm_bound_cuts_extension_before_default_timeout(monkeypatch: pytest.MonkeyPatch) -> None:+ """The extension arm's timeout must switch on the hCaptcha marker, not stay at the Turnstile-sized+ default: a slow hCaptcha solve inside the hCaptcha bound must survive, but the same slowness on a+ non-hCaptcha page must be cut by the shorter default."""+ monkeypatch.setattr(captcha_solver_module, "_HCAPTCHA_ARM_TIMEOUT_SECONDS", 0.05)+ monkeypatch.setattr(captcha_solver_module, "_EXTENSION_ARM_TIMEOUT_SECONDS", 5)++ async def slow_solve(_page: FakePage) -> bool:+ await asyncio.sleep(1.0)+ return True++ agent_function = type(+ "AgentFunctionStub",+ (AgentFunction,),+ {+ "auto_solve_captchas": AsyncMock(side_effect=slow_solve),+ "solve_recaptcha_token": AsyncMock(return_value=False),+ },+ )()+ monkeypatch.setattr(app, "AGENT_FUNCTION", agent_function)++ hcaptcha_page = FakePage(hcaptcha=True)+ with pytest.raises(CaptchaChallengeUnsolvedError):+ await solve_challenge_ladder(hcaptcha_page)++ turnstile_page = FakePage(turnstile=True)+ assert await solve_challenge_ladder(turnstile_page) is True+++async def test_extension_arm_bound_falls_back_to_default_when_no_hcaptcha_marker(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """Mirror of the above with the timeouts swapped: the hCaptcha marker must still be what selects+ the longer bound, not incidental ordering."""+ monkeypatch.setattr(captcha_solver_module, "_HCAPTCHA_ARM_TIMEOUT_SECONDS", 5)+ monkeypatch.setattr(captcha_solver_module, "_EXTENSION_ARM_TIMEOUT_SECONDS", 0.05)++ async def slow_solve(_page: FakePage) -> bool:+ await asyncio.sleep(1.0)+ return True++ agent_function = type(+ "AgentFunctionStub",+ (AgentFunction,),+ {+ "auto_solve_captchas": AsyncMock(side_effect=slow_solve),+ "solve_recaptcha_token": AsyncMock(return_value=False),+ },+ )()+ monkeypatch.setattr(app, "AGENT_FUNCTION", agent_function)++ hcaptcha_page = FakePage(hcaptcha=True)+ assert await solve_challenge_ladder(hcaptcha_page) is True++ turnstile_page = FakePage(turnstile=True)+ with pytest.raises(CaptchaChallengeUnsolvedError):+ await solve_challenge_ladder(turnstile_page)+++async def test_token_arm_skipped_when_ladder_budget_exhausted(monkeypatch: pytest.MonkeyPatch) -> None:+ """A slow hCaptcha extension arm on a mixed-vendor page must not leave enough of the ladder's fixed+ budget for the token arm to still run at its full timeout."""+ monkeypatch.setattr(captcha_solver_module, "_LADDER_BUDGET_SECONDS", 0.1)+ monkeypatch.setattr(captcha_solver_module, "_HCAPTCHA_ARM_TIMEOUT_SECONDS", 0.2)++ async def slow_solve(_page: FakePage) -> bool:+ await asyncio.sleep(0.25)+ return False++ agent_function = type(+ "AgentFunctionStub",+ (AgentFunction,),+ {+ "auto_solve_captchas": AsyncMock(side_effect=slow_solve),+ "solve_recaptcha_token": AsyncMock(return_value=True),+ },+ )()+ monkeypatch.setattr(app, "AGENT_FUNCTION", agent_function)+ page = FakePage(hcaptcha=True, recaptcha=True)++ with pytest.raises(CaptchaChallengeUnsolvedError):+ await solve_challenge_ladder(page)++ agent_function.solve_recaptcha_token.assert_not_awaited()diff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.pyindex 77be5c801..658832712 100644--- a/tests/unit/test_copilot_agent_helpers.py+++ b/tests/unit/test_copilot_agent_helpers.py@@ -42,6 +42,10 @@ from skyvern.forge.sdk.copilot.completion_verification import (CriterionVerdict,gradeable_completion_criteria,)+from skyvern.forge.sdk.copilot.composition_evidence import (+ _page_obstructions_from_visual_summary,+ parse_composition_structured,+)from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy, CopilotConfigfrom skyvern.forge.sdk.copilot.context import CodeAuthoringRepairContext, CopilotContextfrom skyvern.forge.sdk.copilot.diagnosis_repair_contract import (@@ -2000,6 +2004,29 @@ workflow_definition:async def test_same_saved_workflow_and_failed_run_share_one_packet_across_three_paths(self, monkeypatch: pytest.MonkeyPatch) -> None:+ structured_fixture = json.loads(+ (Path(__file__).parent / "data/click_overlay_named_dismiss.structured.json").read_text()+ )+ producer_packet = parse_composition_structured(+ structured_fixture,+ inspected_url="https://example.test/statements",+ current_url="https://example.test/statements",+ )+ assert producer_packet is not None+ dom_obstructions = producer_packet["page_obstructions"]+ screenshot_obstruction = _page_obstructions_from_visual_summary(+ {+ "page_obstruction_detected": True,+ "obstruction_kind": "floating_notice",+ "obstruction_location": "Lower-right corner",+ "underlying_page_blocked": False,+ "visible_dismiss_controls": ["Dismiss notice"],+ }+ )[0]+ source_obstructions = [dom_obstructions[0], screenshot_obstruction, *dom_obstructions, *dom_obstructions]+ failed_run = json.loads(json.dumps(self._FAILED_RUN))+ failed_run["data"]["authoring_repair_context"]["page_obstructions"] = source_obstructions+async def fake_update_workflow(payload, ctx, **_kwargs):assert payload["workflow_yaml"] == self._SAVED_WORKFLOWctx.workflow_yaml = self._SAVED_WORKFLOW@@ -2008,7 +2035,7 @@ workflow_definition:return {"ok": True, "data": {"block_count": 1}, "_workflow": ctx.last_workflow}async def fake_run_blocks(_params, _ctx, **_kwargs):- return json.loads(json.dumps(self._FAILED_RUN))+ return json.loads(json.dumps(failed_run))async def fake_verify(ctx, _result, _handler_start):ctx.last_unverified_block_labels = ["read_total"]@@ -2088,6 +2115,46 @@ workflow_definition:assert packet["failure"]["page_state"]["result_summaries"] == ["result container #summary; text=Total unavailable"]+ carried_obstructions = packet["failure"]["page_state"]["obstructions"]+ assert len(carried_obstructions) == 5+ assert [+ control["selector_candidates"]+ for obstruction in carried_obstructions+ for control in obstruction["visible_controls"]+ if obstruction.get("source") != "vision_summary"+ ] == [+ control["selector_candidates"]+ for obstruction in source_obstructions[:5]+ for control in obstruction["visible_controls"]+ if obstruction.get("source") != "vision_summary"+ ]+ assert [+ control["identity"]+ for obstruction in carried_obstructions+ for control in obstruction["visible_controls"]+ if "identity" in control+ ] == [+ control["identity"]+ for obstruction in source_obstructions[:5]+ for control in obstruction["visible_controls"]+ if "identity" in control+ ]+ screenshot_control = carried_obstructions[1]["visible_controls"][0]+ assert carried_obstructions[1]["source"] == "vision_summary"+ assert carried_obstructions[1]["visual_location"] == "Lower-right corner"+ assert carried_obstructions[1]["underlying_page_blocked"] is False+ assert screenshot_control["text"] == "Dismiss notice"+ assert screenshot_control["selector_candidates"] == []+ assert "identity" not in screenshot_control+ assert any(+ notice+ == f"failure.page_state.obstructions shortened: {len(source_obstructions) - len(carried_obstructions)} item(s) omitted."+ for notice in packet["omission_notices"]+ )+ for result in (update_then_run, combined, edited):+ model_data = json.loads(result)["data"]+ assert "page_obstructions" not in model_data["authoring_repair_context"]+ assert "page_obstruction_omission_notices" not in model_data["authoring_repair_context"]assert packet["registered_outputs"][0]["output_parameter_key"] == "total"assert packet["downloads"] == [{"artifact_id": "artifact_1"}]assert packet["screenshot"] == {"present": True, "provenance": "data.screenshot_base64"}@@ -2138,6 +2205,14 @@ workflow_definition:"authoring_repair_context": {"workflow_run_id": "wr_old","page_result_summaries": ["stale page result"],+ "page_obstructions": [+ {+ "kind": "modal_overlay",+ "source": "dom_html",+ "selector": "#stale-overlay",+ "visible_controls": [],+ }+ ],},},}@@ -2153,6 +2228,7 @@ workflow_definition:assert packet["unfinished_items"] == []assert packet["failure"]["page_state"]["current_url"] == "https://example.test/current"assert packet["failure"]["page_state"]["result_summaries"] == []+ assert packet["failure"]["page_state"]["obstructions"] == []assert any("another or unknown run" in notice for notice in packet["omission_notices"])def test_packet_omits_failed_block_screenshot_without_a_recorded_run(self) -> None:diff --git a/tests/unit/test_copilot_diagnosis_repair_contract.py b/tests/unit/test_copilot_diagnosis_repair_contract.pyindex 6bb67ba5a..233b99c5c 100644--- a/tests/unit/test_copilot_diagnosis_repair_contract.py+++ b/tests/unit/test_copilot_diagnosis_repair_contract.py@@ -3298,6 +3298,30 @@ def test_overlay_dismiss_controls_reach_the_repair_prompt_beside_the_runtime_faiassert repair_context is not Nonesummaries = repair_context.page_obstruction_summaries+ source_obstructions = ctx.composition_page_evidence["page_obstructions"]+ assert [obstruction.model_dump(exclude_none=True) for obstruction in repair_context.page_obstructions] == (+ source_obstructions+ )+ assert [+ [candidate.model_dump() for candidate in control.selector_candidates]+ for obstruction in repair_context.page_obstructions+ for control in obstruction.visible_controls+ ] == [+ control["selector_candidates"]+ for obstruction in source_obstructions+ for control in obstruction["visible_controls"]+ ]+ assert [+ control.identity.model_dump()+ for obstruction in repair_context.page_obstructions+ for control in obstruction.visible_controls+ if control.identity is not None+ ] == [+ control["identity"]+ for obstruction in source_obstructions+ for control in obstruction["visible_controls"]+ if "identity" in control+ ]assert [summary.split()[1] for summary in summaries] == ["#terms-overlay", "#terms-modal"]for summary in summaries:assert "#accept-terms" in summary@@ -3375,13 +3399,14 @@ def test_dismiss_controls_are_reported_without_an_action_or_a_preference() -> No"page_action_summaries","page_challenge_summaries","page_obstruction_summaries",+ "page_obstructions",+ "page_obstruction_omission_notices",}-def test_modal_overlay_dismiss_controls_are_read_only_without_page_obstructions() -> None:+def test_modal_overlay_dismiss_controls_are_read_only_when_page_obstructions_is_absent() -> None:ctx = _overlay_repair_ctx({- "page_obstructions": [],"modal_overlays": [{"selector": "#gate-modal",@@ -3399,6 +3424,18 @@ def test_modal_overlay_dismiss_controls_are_read_only_without_page_obstructions(assert repair_context is not Noneassert repair_context.page_obstruction_summaries == ["#gate-modal Accept All button.accept"]+ assert [obstruction.model_dump(exclude_none=True) for obstruction in repair_context.page_obstructions] == [+ {+ "selector": "#gate-modal",+ "visible_controls": [+ {+ "text": "Accept All",+ "selector": "button.accept",+ "selector_candidates": [],+ }+ ],+ }+ ]def test_page_free_of_obstructions_renders_no_obstruction_line() -> None:@@ -3417,9 +3454,24 @@ def test_page_free_of_obstructions_renders_no_obstruction_line() -> None:assert repair_context is not Noneassert repair_context.page_obstruction_summaries == []+ assert repair_context.page_obstructions == []assert "page_obstructions:" not in _code_authoring_repair_context_prompt(ctx)+def test_malformed_canonical_obstruction_is_omitted_with_an_exact_notice() -> None:+ evidence = _obstruction_only_page_evidence()+ evidence["page_obstructions"].append("malformed")+ ctx = _overlay_repair_ctx(evidence)++ repair_context = finalize_runtime_authoring_repair_context_from_page_observation(ctx)++ assert repair_context is not None+ assert len(repair_context.page_obstructions) == 1+ assert repair_context.page_obstruction_omission_notices == [+ "failure.page_state.obstructions omitted: 1 malformed item(s)."+ ]++def test_page_obstruction_summaries_separate_runtime_repair_root_cause_signatures() -> None:def signature(summaries: list[str]) -> str | None:repair_context = CodeAuthoringRepairContext(diff --git a/tests/unit/test_copilot_output_utils.py b/tests/unit/test_copilot_output_utils.pyindex 82cd2af38..4026e63b9 100644--- a/tests/unit/test_copilot_output_utils.py+++ b/tests/unit/test_copilot_output_utils.py@@ -298,6 +298,11 @@ def test_sanitize_build_test_packet_projection_failure_keeps_tool_result_availab"workflow_run_id": "wr_1","overall_status": "failed","build_test_packet": packet,+ "authoring_repair_context": {+ "page_obstruction_summaries": ["modal_overlay #gate"],+ "page_obstructions": [{"kind": "modal_overlay", "visible_controls": []}],+ "page_obstruction_omission_notices": ["structured facts shortened"],+ },},},)@@ -306,6 +311,30 @@ def test_sanitize_build_test_packet_projection_failure_keeps_tool_result_availabassert sanitized["data"]["overall_status"] == "failed"assert "build_test_packet" not in sanitized["data"]assert sanitized["data"]["build_test_packet_omitted"] == "The internal packet projection failed."+ assert sanitized["data"]["authoring_repair_context"] == {"page_obstruction_summaries": ["modal_overlay #gate"]}+++def test_sanitize_build_test_packet_validation_failure_does_not_expose_structured_repair_copy() -> None:+ sanitized = sanitize_tool_result_for_llm(+ "run_blocks_and_collect_debug",+ {+ "ok": False,+ "data": {+ "workflow_run_id": "wr_1",+ "overall_status": "failed",+ "build_test_packet": {"contract_version": "unknown"},+ "authoring_repair_context": {+ "page_obstruction_summaries": ["modal_overlay #gate"],+ "page_obstructions": [{"kind": "modal_overlay", "visible_controls": []}],+ "page_obstruction_omission_notices": ["structured facts shortened"],+ },+ },+ },+ )++ assert "build_test_packet" not in sanitized["data"]+ assert sanitized["data"]["build_test_packet_omitted"] == "The internal packet failed typed validation."+ assert sanitized["data"]["authoring_repair_context"] == {"page_obstruction_summaries": ["modal_overlay #gate"]}def test_sanitize_run_blocks_debug_preserves_post_run_page_evidence() -> None: