Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
SKY-10562 Ground Copilot block authoring in observations (#6363)
Andrew Neilsonabf5e7660e24
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.pyindex e2acf7d44..64e9a1970 100644--- a/skyvern/forge/sdk/copilot/agent.py+++ b/skyvern/forge/sdk/copilot/agent.py@@ -51,6 +51,7 @@ from skyvern.forge.sdk.copilot.context import (TurnNarrativePayload,finalize_discovery_counter_in_global_llm_context,)+from skyvern.forge.sdk.copilot.enforcement import outcome_fully_verifiedfrom skyvern.forge.sdk.copilot.outcome_verification_trace import (finalize_outcome_verification_trace,record_gate_decision,@@ -685,31 +686,40 @@ def _partial_verification_response(ctx: CopilotContext) -> str | None:def _rewrite_failed_test_response(user_response: str, ctx: CopilotContext) -> str:has_keepable_draft = ctx.last_workflow is not None and bool(ctx.last_workflow_yaml)keep_draft_affordance = " Keep the draft to iterate on, or discard." if has_keepable_draft else ""+ block_count = ctx.last_update_block_count if isinstance(ctx.last_update_block_count, int) else None+ positive_block_count = block_count if block_count is not None and block_count > 0 else None++ if outcome_fully_verified(ctx) and has_keepable_draft and positive_block_count is not None:+ block_word = "block" if positive_block_count == 1 else "blocks"+ return (+ f"I created a workflow with {positive_block_count} {block_word} and verified the requested "+ "outcome from the current browser page after the run. The workflow is ready to review."+ )policy = ctx.request_policy if isinstance(ctx.request_policy, RequestPolicy) else Noneif (policy is not Noneand policy.clarification_reason == "workflow_credential_inputs_unbound"and ctx.last_workflow is not None- and ctx.last_update_block_count is not None+ and block_count is not None):- if ctx.last_update_block_count <= 0:+ if positive_block_count is None:draft_phrase = "a draft workflow"else:- block_word = "block" if ctx.last_update_block_count == 1 else "blocks"- draft_phrase = f"a draft workflow with {ctx.last_update_block_count} {block_word}"+ block_word = "block" if positive_block_count == 1 else "blocks"+ draft_phrase = f"a draft workflow with {positive_block_count} {block_word}"return (f"I applied your requested change as {draft_phrase}. "f"I couldn't test the modified workflow because I couldn't find the required credentials — "f"please add them via the Credentials UI, then I can try again.{keep_draft_affordance}")- if ctx.last_test_ok is False and ctx.last_update_block_count is not None:- if ctx.last_update_block_count <= 0:+ if ctx.last_test_ok is False and block_count is not None:+ if positive_block_count is None:draft_phrase = "a draft workflow"else:- block_word = "block" if ctx.last_update_block_count == 1 else "blocks"- draft_phrase = f"a draft workflow with {ctx.last_update_block_count} {block_word}"+ block_word = "block" if positive_block_count == 1 else "blocks"+ draft_phrase = f"a draft workflow with {positive_block_count} {block_word}"failure_summary = _normalize_failure_reason(ctx.last_test_failure_reason)follow_up = _FAILURE_FOLLOW_UP.get(ctx.last_failure_category_top or "", "")@@ -723,7 +733,7 @@ def _rewrite_failed_test_response(user_response: str, ctx: CopilotContext) -> stif partial_reply is not None:return partial_reply- if ctx.last_test_ok is None and ctx.last_update_block_count is not None and ctx.last_workflow is not None:+ if ctx.last_test_ok is None and block_count is not None and ctx.last_workflow is not None:if policy is not None and policy.raw_secret_handling == "redacted_draft":return ("I drafted the workflow with the pasted secret redacted. "@@ -770,8 +780,6 @@ def _completion_contract_not_violated(ctx: CopilotContext) -> bool:def _verified_workflow_or_none(ctx: CopilotContext) -> tuple[Any, str | None]:"""Surface a proposal when it passed a test this turn, or when the outcome judgeconfirmed the goal from evidence even though the run did not finish cleanly."""- from skyvern.forge.sdk.copilot.enforcement import outcome_fully_verified-run_status_clean = ctx.last_test_ok is True and ctx.last_full_workflow_test_ok is Trueif (ctx.last_workflow is not Nonediff --git a/skyvern/forge/sdk/copilot/completion_verification.py b/skyvern/forge/sdk/copilot/completion_verification.pyindex 2eb05121a..bc31dd915 100644--- a/skyvern/forge/sdk/copilot/completion_verification.py+++ b/skyvern/forge/sdk/copilot/completion_verification.py@@ -70,6 +70,7 @@ class RunEvidenceSnapshot:current_url: str | None = Nonepage_title: str | None = Noneexecuted_block_labels: list[str] = field(default_factory=list)+ verified_context_block_labels: list[str] = field(default_factory=list)def has_evidence(self) -> bool:return bool(self.block_outputs or self.current_url or self.page_title)@@ -82,6 +83,10 @@ class RunEvidenceSnapshot:lines.append(f"observed_end_state_url: {self.current_url}")if self.page_title:lines.append(f"observed_end_state_page_title: {self.page_title}")+ if self.verified_context_block_labels:+ lines.append(+ "verified_context_block_labels: " + ", ".join(self.verified_context_block_labels[:_MAX_BLOCK_OUTPUTS])+ )if self.executed_block_labels:lines.append("executed_block_labels: " + ", ".join(self.executed_block_labels[:_MAX_BLOCK_OUTPUTS]))if self.block_outputs:diff --git a/skyvern/forge/sdk/copilot/composition_evidence.py b/skyvern/forge/sdk/copilot/composition_evidence.pyindex 997a5b27d..58a0f991a 100644--- a/skyvern/forge/sdk/copilot/composition_evidence.py+++ b/skyvern/forge/sdk/copilot/composition_evidence.py@@ -43,6 +43,24 @@ _MAX_RESULT_CONTAINERS = 8_MAX_NAVIGATION_TARGETS = 20_MAX_SELECT_OPTIONS = 30_MAX_CHALLENGE_CONTROLS = 8+_MAX_MODAL_OVERLAYS = 5+_MAX_MODAL_DISMISS_CONTROLS = 6+_MAX_PAGE_OBSTRUCTIONS = 5+_MAX_VISIBLE_CONTROLS = 6+_MODAL_IDENTITY_PATTERNS: frozenset[str] = frozenset({"modal", "popup", "overlay", "dialog", "drawer", "lightbox"})+_MODAL_ROLE_VALUES: frozenset[str] = frozenset({"dialog", "alertdialog"})+_MODAL_DISMISS_HINTS: frozenset[str] = frozenset(+ {+ "cancel",+ "close",+ "dismiss",+ "got it",+ "no thanks",+ "not now",+ "skip",+ }+)+_MODAL_DISMISS_SYMBOLS: frozenset[str] = frozenset({"x", "\u00d7"})DOM_EVIDENCE_SOURCE = "dom_html"SCREENSHOT_EVIDENCE_SOURCE = "screenshot"VISION_EVIDENCE_SOURCE = "vision_summary"@@ -50,6 +68,8 @@ _ANTI_BOT_PATTERNS = ("just a moment","captcha","challenge",+ "turnstile",+ "cf-turnstile","human-verification","human verification","verify you are human",@@ -157,6 +177,37 @@ def _evidence_metadata(}+def _bounded_visual_controls(values: Iterable[Any]) -> list[dict[str, Any]]:+ controls: list[dict[str, Any]] = []+ for value in values:+ if len(controls) >= _MAX_VISIBLE_CONTROLS:+ break+ text = _bounded_string(value, 120)+ if text:+ controls.append({"text": text})+ return controls+++def _page_obstructions_from_visual_summary(visual_summary: dict[str, Any]) -> list[dict[str, Any]]:+ if visual_summary.get("page_obstruction_detected") is not True:+ return []+ obstruction: dict[str, Any] = {+ "kind": _bounded_string(visual_summary.get("obstruction_kind"), 80) or "visual_obstruction",+ "source": VISION_EVIDENCE_SOURCE,+ "visual_location": _bounded_string(visual_summary.get("obstruction_location"), 180),+ "visible_controls": _bounded_visual_controls(visual_summary.get("visible_dismiss_controls") or []),+ }+ if isinstance(visual_summary.get("underlying_page_blocked"), bool):+ obstruction["underlying_page_blocked"] = visual_summary["underlying_page_blocked"]+ return [+ {+ key: value+ for key, value in obstruction.items()+ if value or key in {"visible_controls", "underlying_page_blocked"}+ }+ ]++def page_evidence_needs_visual_fallback(evidence: dict[str, Any]) -> bool:"""Return True when DOM evidence should be augmented with visual evidence."""@@ -232,6 +283,10 @@ def merge_visual_composition_evidence(]challenge_state["gated_submit_controls"] = (existing_controls + visual_blocked_controls)[:5]merged["challenge_state"] = challenge_state+ visual_obstructions = _page_obstructions_from_visual_summary(visual_summary)+ if visual_obstructions:+ existing_obstructions = [item for item in merged.get("page_obstructions") or [] if isinstance(item, dict)]+ merged["page_obstructions"] = (existing_obstructions + visual_obstructions)[:_MAX_PAGE_OBSTRUCTIONS]# Empty-page classification is visual-only: DOM parsing can tell that a page# is schema-empty, but only the screenshot summary distinguishes settled empty# pages from loading shells.@@ -495,6 +550,22 @@ def has_bounded_page_schema(evidence: dict[str, Any]) -> bool:value = evidence.get(key)if isinstance(value, list) and value:return True+ modal_overlays = evidence.get("modal_overlays")+ if isinstance(modal_overlays, list):+ for overlay in modal_overlays:+ if not isinstance(overlay, dict):+ continue+ dismiss_controls = overlay.get("dismiss_controls")+ if isinstance(dismiss_controls, list) and dismiss_controls:+ return True+ page_obstructions = evidence.get("page_obstructions")+ if isinstance(page_obstructions, list):+ for obstruction in page_obstructions:+ if not isinstance(obstruction, dict):+ continue+ visible_controls = obstruction.get("visible_controls")+ if isinstance(visible_controls, list) and visible_controls:+ return Truechallenge_state = evidence.get("challenge_state")if isinstance(challenge_state, dict) and challenge_state.get("detected") is True:return True@@ -691,6 +762,27 @@ def _associated_observation_satisfies_block(return _evidence_matches_target(evidence, observed_url, allow_post_run_browser_observation=allow_post_run)+def _current_page_evidence_has_reached_page_credit(+ evidence: dict[str, Any],+ reached_via: str,+ *,+ flow_evidence_by_step: dict[int, tuple[dict[str, Any], str]],+) -> bool:+ if reached_via != "current_page" or not has_bounded_page_schema(evidence):+ return False+ observed_url = _evidence_observed_url(evidence)+ if not observed_url:+ return False+ for prior_evidence, prior_reached_via in flow_evidence_by_step.values():+ if prior_reached_via not in {"interaction", "post_run"}:+ continue+ if not has_bounded_page_schema(prior_evidence):+ continue+ if _same_page(observed_url, _evidence_observed_url(prior_evidence)):+ return True+ return False++def _block_has_observed_page(ctx: Any,block: dict[str, Any],@@ -705,11 +797,20 @@ def _block_has_observed_page(if evidence_entry is None:return Falseevidence, reached_via = evidence_entry+ effective_reached_via = (+ "interaction"+ if _current_page_evidence_has_reached_page_credit(+ evidence,+ reached_via,+ flow_evidence_by_step=flow_evidence_by_step,+ )+ else reached_via+ )return _associated_observation_satisfies_block(evidence,block.get("target_url"),acts_via=str(block.get("acts_via") or ""),- reached_via=reached_via,+ reached_via=effective_reached_via,requires_observation_ref=block.get("requires_observation_ref") is True,allow_post_run=allow_post_run,)@@ -772,9 +873,15 @@ def _wrong_reached_via_observation_ref(evidence_entry = flow_evidence_by_step.get(step)if evidence_entry is None:return None- _, reached_via = evidence_entry+ evidence, reached_via = evidence_entryif reached_via in {"interaction", "post_run"}:return None+ if _current_page_evidence_has_reached_page_credit(+ evidence,+ reached_via,+ flow_evidence_by_step=flow_evidence_by_step,+ ):+ return Nonereturn step, reached_via or "<missing>"@@ -840,7 +947,7 @@ def composition_page_evidence_error(return ("Workflow validation failed: a block_observation_refs entry uses observation_step "f"{string_step!r} as a string. Pass the integer observation_step returned by "- "inspect_page_for_composition for click-reached blocks. "+ "inspect_page_for_composition or evaluate for click-reached blocks. "f"Offending blocks: {_format_page_block_findings([block])}")if _required_observation_ref_missing(block, block_observation_refs):@@ -868,8 +975,8 @@ def composition_page_evidence_error(return ("Workflow validation failed: a block references observation_step "f"{missing_step}, but {missing_reason}. "- "Call inspect_page_for_composition again for the reached page and pass the new "- "observation_step in block_observation_refs before composing page-dependent blocks. "+ "Inspect or evaluate the reached page again and pass the new observation_step in "+ "block_observation_refs before composing page-dependent blocks. "f"Offending blocks: {_format_page_block_findings([block])}")return (@@ -896,6 +1003,8 @@ def _empty_evidence(inspected_url: str, current_url: str) -> dict[str, Any]:"result_containers": [],"anti_bot_indicators": [],"challenge_controls": [],+ "modal_overlays": [],+ "page_obstructions": [],"schema_empty_page": False,"observed_empty_page": False,"empty_page_visual_state": None,@@ -1122,6 +1231,145 @@ def _challenge_controls(soup: Any) -> list[dict[str, Any]]:return controls+def _modal_identity(node: Any) -> str:+ values = (+ getattr(node, "name", ""),+ _attr_value(node, "id"),+ " ".join(_classes_for(node)),+ _attr_value(node, "role"),+ _attr_value(node, "aria-label"),+ _attr_value(node, "title"),+ _attr_value(node, "data-testid"),+ _attr_value(node, "data-test"),+ _attr_value(node, "data-dismiss"),+ )+ return " ".join(str(value or "") for value in values).lower()+++def _is_modal_overlay_candidate(node: Any) -> bool:+ role = _attr_value(node, "role").strip().lower()+ if role in _MODAL_ROLE_VALUES:+ return True+ if _attr_value(node, "aria-modal").strip().lower() == "true":+ return True+ return any(pattern in _modal_identity(node) for pattern in _MODAL_IDENTITY_PATTERNS)+++def _is_hidden_modal_candidate(node: Any) -> bool:+ current = node+ while current is not None:+ if _attr_value(current, "aria-hidden").strip().lower() == "true":+ return True+ if hasattr(current, "has_attr") and current.has_attr("hidden"):+ return True+ style = _attr_value(current, "style").replace(" ", "").lower()+ if "display:none" in style or "visibility:hidden" in style:+ return True+ current = getattr(current, "parent", None)+ return False+++def _modal_dismiss_controls(node: Any) -> list[dict[str, Any]]:+ controls: list[dict[str, Any]] = []+ seen_selectors: set[str] = set()+ for control in node.find_all(["button", "a", "input"]):+ if len(controls) >= _MAX_MODAL_DISMISS_CONTROLS:+ break+ selector = _selector_for(control)[:160]+ if selector in seen_selectors:+ continue+ text = _schema_text(_node_text(control) or _attr_value(control, "value"), 120)+ aria_label = _schema_text(_attr_value(control, "aria-label"), 120)+ title = _schema_text(_attr_value(control, "title"), 120)+ explicit_values = {text.strip().lower(), aria_label.strip().lower(), title.strip().lower()}+ identity = f"{text} {aria_label} {title} {_modal_identity(control)}".lower()+ # Controls are typed as Any because BeautifulSoup is optional at import time.+ has_data_dismiss = hasattr(control, "has_attr") and control.has_attr("data-dismiss")+ has_symbol_hint = bool(explicit_values & _MODAL_DISMISS_SYMBOLS)+ has_text_hint = any(hint in identity for hint in _MODAL_DISMISS_HINTS)+ if not (has_data_dismiss or has_symbol_hint or has_text_hint):+ continue+ seen_selectors.add(selector)+ controls.append(+ {+ "tag": str(getattr(control, "name", "") or "").lower()[:40],+ "text": text,+ "aria_label": aria_label,+ "title": title,+ "selector": selector,+ "type": _attr_value(control, "type")[:40],+ }+ )+ return [{key: value for key, value in entry.items() if value} for entry in controls]+++def _modal_overlay_entry(node: Any) -> dict[str, Any]:+ return {+ "role": _attr_value(node, "role")[:80],+ "aria_modal": _attr_value(node, "aria-modal").strip().lower() == "true",+ "id": _attr_value(node, "id")[:120],+ "class": " ".join(_classes_for(node)[:5])[:160],+ "selector": _selector_for(node)[:160],+ "text": _schema_text(_node_text(node), 240),+ "dismiss_controls": _modal_dismiss_controls(node),+ }+++def _modal_overlays(nodes: Iterable[Any]) -> list[dict[str, Any]]:+ overlays: list[dict[str, Any]] = []+ seen_selectors: set[str] = set()+ for node in nodes:+ if len(overlays) >= _MAX_MODAL_OVERLAYS:+ break+ if not _is_modal_overlay_candidate(node):+ continue+ if _is_hidden_modal_candidate(node):+ continue+ selector = _selector_for(node)[:160]+ if selector in seen_selectors:+ continue+ entry = _modal_overlay_entry(node)+ dismiss_controls = entry.get("dismiss_controls")+ has_dismiss_controls = isinstance(dismiss_controls, list) and bool(dismiss_controls)+ has_explicit_modal_semantics = bool(entry.get("role") or entry.get("aria_modal") is True)+ if not (has_explicit_modal_semantics or has_dismiss_controls):+ continue+ seen_selectors.add(selector)+ # Omit aria_modal=False from compact evidence: only affirmative modal+ # semantics matter for obstruction handoff.+ compact_entry = {key: value for key, value in entry.items() if value or key == "dismiss_controls"}+ overlays.append(compact_entry)+ return overlays+++def _page_obstructions_from_modal_overlays(modal_overlays: list[dict[str, Any]]) -> list[dict[str, Any]]:+ obstructions: list[dict[str, Any]] = []+ for overlay in modal_overlays[:_MAX_PAGE_OBSTRUCTIONS]:+ if not isinstance(overlay, dict):+ continue+ visible_controls: list[dict[str, Any]] = []+ for control in overlay.get("dismiss_controls") or []:+ if len(visible_controls) >= _MAX_VISIBLE_CONTROLS:+ break+ if not isinstance(control, dict):+ continue+ text = _bounded_string(control.get("text") or control.get("aria_label") or control.get("title"), 120)+ selector = _bounded_string(control.get("selector"), 160)+ if text or selector:+ visible_controls.append(+ {key: value for key, value in {"text": text, "selector": selector}.items() if value}+ )+ entry = {+ "kind": "modal_overlay",+ "source": DOM_EVIDENCE_SOURCE,+ "selector": _bounded_string(overlay.get("selector"), 160),+ "text": _bounded_string(overlay.get("text"), 240),+ "visible_controls": visible_controls,+ }+ obstructions.append({key: value for key, value in entry.items() if value or key == "visible_controls"})+ return obstructions++def _anti_bot_indicators(html: str, page_title: str) -> list[str]:haystack = f"{page_title}\n{html[:_ANTI_BOT_SCAN_BYTES]}".lower()return [pattern for pattern in _ANTI_BOT_PATTERNS if pattern in haystack]@@ -1144,6 +1392,8 @@ def parse_composition_html(html: str, *, inspected_url: str, current_url: str) -node.decompose()all_nodes = soup.find_all(True)+ modal_overlays = _modal_overlays(all_nodes)+ page_obstructions = _page_obstructions_from_modal_overlays(modal_overlays)forms: list[dict[str, Any]] = []for form in soup.find_all("form")[:_MAX_FORMS]:@@ -1235,7 +1485,12 @@ def parse_composition_html(html: str, *, inspected_url: str, current_url: str) -body_text = _node_text(soup.body if soup.body is not None else soup).strip()# Schema-empty means the page had content, but no bounded form/link/result/challenge structure.schema_empty_page = bool((html or "").strip() or page_title or body_text) and not (- forms or navigation_targets or result_containers or challenge_controls or anti_bot_indicators+ forms+ or navigation_targets+ or result_containers+ or challenge_controls+ or page_obstructions+ or anti_bot_indicators)# Higher confidence means the parser saw a more complete form surface.confidence = 0.85 if field_count and control_count else 0.6 if field_count else 0.3 if forms else 0.1@@ -1248,6 +1503,8 @@ def parse_composition_html(html: str, *, inspected_url: str, current_url: str) -"result_containers": result_containers,"anti_bot_indicators": anti_bot_indicators,"challenge_controls": challenge_controls,+ "modal_overlays": modal_overlays,+ "page_obstructions": page_obstructions,"schema_empty_page": schema_empty_page,"observed_empty_page": False,"empty_page_visual_state": None,diff --git a/skyvern/forge/sdk/copilot/runtime.py b/skyvern/forge/sdk/copilot/runtime.pyindex ff0feac62..1b68ab171 100644--- a/skyvern/forge/sdk/copilot/runtime.py+++ b/skyvern/forge/sdk/copilot/runtime.py@@ -95,6 +95,12 @@ async def _get_persistent_browser_session(session_id: str, organization_id: str)return cast("PersistentBrowserSession | None", persistent_result)+@dataclass+class PendingBrowserInteractionObservation:+ tool_name: str+ url: str = ""++@dataclassclass AgentContext:organization_id: str@@ -178,6 +184,7 @@ class AgentContext:# while scouting the goal path, each tagged with how that state was reached.# Feeds the per-acted-page composition gate; never persisted into workflow YAML.flow_evidence: list[dict[str, Any]] = field(default_factory=list)+ pending_browser_interaction_observation: PendingBrowserInteractionObservation | None = None# In-turn side channel from workflow mutation calls: block label -> flow_evidence# observation step used to ground the newly authored page-acting block.block_observation_refs: dict[str, int] = field(default_factory=dict)diff --git a/skyvern/forge/sdk/copilot/tools.py b/skyvern/forge/sdk/copilot/tools.pyindex 0ef835816..02660250b 100644--- a/skyvern/forge/sdk/copilot/tools.py+++ b/skyvern/forge/sdk/copilot/tools.py@@ -106,7 +106,11 @@ from skyvern.forge.sdk.copilot.output_utils import (truncate_output,)from skyvern.forge.sdk.copilot.request_policy import CREDENTIAL_DEFERRED_DRAFT_REASONS, RequestPolicy-from skyvern.forge.sdk.copilot.runtime import AgentContext, ensure_browser_session+from skyvern.forge.sdk.copilot.runtime import (+ AgentContext,+ PendingBrowserInteractionObservation,+ ensure_browser_session,+)from skyvern.forge.sdk.copilot.screenshot_utils import enqueue_screenshot_from_resultfrom skyvern.forge.sdk.copilot.streaming_adapter import emit_workflow_draft, maybe_emit_design_endfrom skyvern.forge.sdk.copilot.tracing_setup import copilot_span@@ -1360,6 +1364,57 @@ def _mark_page_inspected(ctx: AgentContext) -> None:ctx.post_budget_page_inspection_run_id = None+def _same_page_ignoring_fragment(left: str | None, right: str | None) -> bool:+ if not left or not right:+ return False+ try:+ left_parsed = urlparse(left)+ right_parsed = urlparse(right)+ except Exception as exc:+ LOG.debug("copilot_same_page_url_parse_failed", left=left, right=right, error=str(exc))+ return False+ left_url = left_parsed._replace(fragment="").geturl().rstrip("/")+ right_url = right_parsed._replace(fragment="").geturl().rstrip("/")+ return left_url == right_url+++def _clear_pending_browser_interaction_observation(ctx: AgentContext) -> None:+ ctx.pending_browser_interaction_observation = None+++def _mark_pending_browser_interaction_observation(ctx: AgentContext, *, tool_name: str, url: str) -> None:+ if not url.strip():+ _clear_pending_browser_interaction_observation(ctx)+ return+ ctx.pending_browser_interaction_observation = PendingBrowserInteractionObservation(+ tool_name=tool_name,+ url=url.strip(),+ )+++def _consume_pending_browser_interaction_observation(+ ctx: AgentContext,+ *,+ current_url: str,+ evidence: dict[str, Any],+) -> bool:+ pending = ctx.pending_browser_interaction_observation+ if pending is None:+ return False+ _clear_pending_browser_interaction_observation(ctx)+ if not has_bounded_page_schema(evidence):+ return False+ if not _same_page_ignoring_fragment(pending.url, current_url):+ LOG.warning(+ "copilot_pending_browser_interaction_observation_page_mismatch",+ tool_name=pending.tool_name,+ pending_url=pending.url,+ current_url=current_url,+ )+ return False+ return True++def _mark_post_run_page_observed(ctx: AgentContext, *, source_tool: str, url: str) -> None:run_id = getattr(ctx, "last_run_blocks_workflow_run_id", None)if not isinstance(run_id, str) or not run_id:@@ -4262,6 +4317,47 @@ def _normalize_observed_forms(observed_data: dict[str, Any]) -> list[dict[str, Areturn [{"id": "", "name": "", "action": "", "method": "", "fields": fields, "submit_controls": []}]+def _evaluate_observed_payload(observed_data: object) -> dict[str, Any]:+ if not isinstance(observed_data, dict):+ return {}+ raw_result = observed_data.get("result")+ if isinstance(raw_result, dict):+ payload: dict[str, Any] = dict(raw_result)+ elif isinstance(raw_result, list):+ payload = {"rows": raw_result}+ else:+ payload = {}+ for key, value in observed_data.items():+ if key == "result":+ continue+ if key not in payload or payload.get(key) in (None, "", [], {}):+ payload[key] = value+ return payload or observed_data+++def _observed_row_text(row: object) -> str:+ if isinstance(row, str):+ return _bounded_observation_text(row, 500)+ if not isinstance(row, dict):+ return ""+ parts: list[str] = []+ for key in ("text", "label", "name", "title"):+ value = row.get(key)+ if isinstance(value, str):+ parts.append(value)+ cells = row.get("cells")+ if isinstance(cells, list):+ for cell in cells[:12]:+ if isinstance(cell, dict):+ for key in ("text", "value"):+ value = cell.get(key)+ if isinstance(value, str):+ parts.append(value)+ elif isinstance(cell, str):+ parts.append(cell)+ return _bounded_observation_text(" ".join(parts), 500)++def _normalize_observed_result_containers(observed_data: dict[str, Any]) -> list[dict[str, Any]]:raw_results = observed_data.get("result_containers") or observed_data.get("tables")containers: list[dict[str, Any]] = []@@ -4277,14 +4373,74 @@ def _normalize_observed_result_containers(observed_data: dict[str, Any]) -> list)if containers:return containers+ raw_rows = observed_data.get("rows")+ if isinstance(raw_rows, list) and raw_rows:+ sample_rows = [text for row in raw_rows[:5] if (text := _observed_row_text(row))]+ return [+ {+ "tag": "table",+ "id": _bounded_observation_text(observed_data.get("id"), 120),+ "selector": _bounded_observation_text(observed_data.get("selector"), 160),+ "row_count": len(raw_rows),+ "sample_rows": sample_rows,+ }+ ]table = observed_data.get("table")if table:return [{"tag": "table", "id": "", "selector": ""}]return []+def _normalize_evaluate_challenge_state(+ observed_data: dict[str, Any],+ anti_bot_indicators: list[str],+) -> dict[str, Any]:+ detected = bool(anti_bot_indicators)+ gated_controls: list[dict[str, Any]] = []+ for key, value in observed_data.items():+ normalized_key = str(key).strip().lower().replace("_", " ").replace("-", " ")+ if "disabled" not in normalized_key or value is not True:+ continue+ if any(token in normalized_key for token in ("submit", "search", "button", "btn")):+ gated_controls.append(+ {+ "text": _bounded_observation_text(str(key), 120),+ "id": "",+ "name": _bounded_observation_text(str(key), 120),+ "selector": "",+ "disabled": True,+ }+ )+ indicator_text = " ".join(anti_bot_indicators).lower()+ if "access denied" in indicator_text:+ kind = "access_denied"+ elif "turnstile" in indicator_text or "captcha" in indicator_text or "are you a robot" in indicator_text:+ kind = "captcha"+ elif detected:+ kind = "human_verification"+ else:+ kind = "none"+ return {+ "detected": detected,+ "kind": kind,+ "source": "mcp_evaluate" if detected else "",+ "indicators": anti_bot_indicators[:8],+ "requires_human_verification": detected,+ "visual_location": "",+ "gates_submit_controls": bool(detected and gated_controls),+ "gated_submit_controls": gated_controls[:5] if detected else [],+ }+++def _evaluate_anti_bot_indicators(observed_data: dict[str, Any], text: str) -> list[str]:+ key_text = " ".join(str(key) for key in observed_data.keys()).lower()+ combined = f"{text} {key_text}"+ return [pattern for pattern in _DISCOVERY_ANTI_BOT_PATTERNS if pattern in combined]++def _normalize_evaluate_page_schema(observed_data: object) -> dict[str, Any]:- if not isinstance(observed_data, dict):+ observed_data = _evaluate_observed_payload(observed_data)+ if not observed_data:return {}text_parts = [observed_data.get("body"),@@ -4293,7 +4449,7 @@ def _normalize_evaluate_page_schema(observed_data: object) -> dict[str, Any]:observed_data.get("html"),]text = " ".join(part for part in text_parts if isinstance(part, str)).lower()[:4096]- anti_bot = [pattern for pattern in _DISCOVERY_ANTI_BOT_PATTERNS if pattern in text]+ anti_bot = _evaluate_anti_bot_indicators(observed_data, text)forms = _normalize_observed_forms(observed_data)result_containers = _normalize_observed_result_containers(observed_data)if not forms and not result_containers and not anti_bot:@@ -4302,6 +4458,7 @@ def _normalize_evaluate_page_schema(observed_data: object) -> dict[str, Any]:"forms": forms,"result_containers": result_containers,"anti_bot_indicators": anti_bot,+ "challenge_state": _normalize_evaluate_challenge_state(observed_data, anti_bot),"evidence_sources": ["mcp_evaluate"],"evidence_confidence": (_EVALUATE_EVIDENCE_CONFIDENCE_WITH_SCHEMA@@ -4318,15 +4475,15 @@ def _record_composition_page_observation(url: str,title: str = "",observed_data: object | None = None,-) -> None:+ append_to_flow: bool = False,+ reached_via: str = "current_page",+) -> int | None:if not url:- return+ return None_mark_post_run_page_observed(ctx, source_tool=source_tool, url=url)if title:_workflow_verification_evidence(ctx).page_title = title[:160]existing = ctx.composition_page_evidence- if isinstance(existing, dict) and existing.get("source_tool") == "inspect_page_for_composition":- returnevidence: dict[str, Any] = {"inspected_url": url,@@ -4357,7 +4514,157 @@ def _record_composition_page_observation(if isinstance(run_id, str) and run_id:evidence["workflow_run_id"] = run_idevidence["observed_after_workflow_run"] = True- ctx.composition_page_evidence = evidence++ observation_step: int | None = None+ if append_to_flow and has_bounded_page_schema(evidence):+ actual_reached_via = reached_via+ if reached_via == "auto":+ if isinstance(run_id, str) and run_id:+ actual_reached_via = "post_run"+ elif _consume_pending_browser_interaction_observation(ctx, current_url=url, evidence=evidence):+ actual_reached_via = "interaction"+ else:+ actual_reached_via = "current_page"+ observation_step = _append_flow_evidence(ctx, evidence, reached_via=actual_reached_via)++ if not _should_keep_existing_composition_page_evidence(existing, evidence):+ ctx.composition_page_evidence = evidence+ return observation_step+++def _composition_evidence_page_url(evidence: dict[str, Any] | None) -> str | None:+ if not isinstance(evidence, dict):+ return None+ for key in ("current_url", "inspected_url"):+ value = evidence.get(key)+ if isinstance(value, str) and value.strip() and value != "current_page":+ return value.strip()+ return None+++def _should_keep_existing_composition_page_evidence(+ existing: dict[str, Any] | None,+ incoming: dict[str, Any],+) -> bool:+ if not isinstance(existing, dict) or existing.get("source_tool") != "inspect_page_for_composition":+ return False+ if incoming.get("source_tool") != "evaluate":+ return True+ if not has_bounded_page_schema(incoming):+ return True+ if not has_bounded_page_schema(existing):+ return False+ existing_url = _composition_evidence_page_url(existing)+ incoming_url = _composition_evidence_page_url(incoming)+ if existing_url and incoming_url and not _same_page_ignoring_fragment(existing_url, incoming_url):+ return False+ return True+++def _completion_verification_criteria(copilot_ctx: Any) -> list[Any]:+ policy = getattr(copilot_ctx, "request_policy", None)+ # A method-mandated criterion asserts HOW the goal was reached; the outcome+ # judge sees only end-state evidence.+ return [c for c in (policy.completion_criteria if policy is not None else []) if not c.method_mandated]+++def _verified_context_block_labels_for_snapshot(+ copilot_ctx: Any, current_labels: list[str], executed_labels: list[str]+) -> list[str]:+ current_label_set = set(current_labels)+ executed_label_set = set(executed_labels)+ candidates: set[str] = set()+ for raw_values in (+ getattr(copilot_ctx, "verified_prefix_labels", None),+ getattr(getattr(copilot_ctx, "workflow_verification_evidence", None), "block_verified", None),+ ):+ if not isinstance(raw_values, list):+ continue+ candidates.update(str(label) for label in raw_values if isinstance(label, str) and label in current_label_set)+ return [label for label in current_labels if label in candidates and label not in executed_label_set]+++def _build_page_observation_evidence_snapshot(+ copilot_ctx: Any,+ *,+ url: str,+ title: str = "",+ observed_data: object | None = None,+) -> RunEvidenceSnapshot:+ run_id = getattr(copilot_ctx, "last_run_blocks_workflow_run_id", None)+ block_outputs: dict[str, Any] = {}+ if isinstance(observed_data, dict) and observed_data:+ block_outputs["current_page_observation"] = observed_data+ elif observed_data is not None:+ block_outputs["current_page_observation"] = str(observed_data)+ return RunEvidenceSnapshot(+ workflow_run_id=run_id if isinstance(run_id, str) else None,+ block_outputs=block_outputs,+ current_url=_valid_runtime_anchor_url(url),+ page_title=title if isinstance(title, str) and title.strip() else None,+ )+++async def _maybe_run_completion_verification_from_page_observation(+ copilot_ctx: Any,+ *,+ url: str,+ title: str = "",+ observed_data: object | None = None,+) -> CompletionVerificationResult | None:+ """Verify completion only for post-run page observations after failed tests."""++ if not settings.COPILOT_OUTCOME_VERIFICATION_ENABLED:+ return None+ existing = getattr(copilot_ctx, "completion_verification_result", None)+ if isinstance(existing, CompletionVerificationResult) and existing.is_fully_satisfied():+ return existing+ if getattr(copilot_ctx, "post_run_page_observation_after_failed_test", False) is not True:+ return None+ criteria = _completion_verification_criteria(copilot_ctx)+ if not criteria:+ return None+ handler = await _completion_verification_handler(copilot_ctx)+ if handler is None:+ return None+ remaining = _copilot_seconds_remaining(copilot_ctx)+ if (+ remaining is not None+ and remaining+ <= settings.COPILOT_FEASIBILITY_GATE_TIMEOUT_SECONDS + _COMPLETION_VERIFICATION_BUDGET_MARGIN_SECONDS+ ):+ verification = CompletionVerificationResult(status="unavailable")+ else:+ snapshot = _build_page_observation_evidence_snapshot(+ copilot_ctx,+ url=url,+ title=title,+ observed_data=observed_data,+ )+ if not snapshot.has_evidence():+ verification = CompletionVerificationResult(+ status="evaluated",+ criterion_ids=[criterion.id for criterion in criteria],+ verdicts=[+ CriterionVerdict(criterion_id=criterion.id, satisfied=False, reason_code="no_evidence")+ for criterion in criteria+ ],+ )+ else:+ verification = await evaluate_completion_criteria(criteria, snapshot, handler)++ if (+ isinstance(existing, CompletionVerificationResult)+ and not verification.is_fully_satisfied()+ and not (existing.status == "unavailable" and verification.status == "evaluated")+ ):+ return existing++ copilot_ctx.completion_verification_result = verification+ record_completion_verification(copilot_ctx, verification)+ if verification.status == "evaluated":+ _emit_completion_verification_trace(copilot_ctx, verification)+ return verification# Block types the copilot must never emit. They delegate the entire goal to@@ -4709,6 +5016,7 @@ async def _navigate_post_hook(raw: dict[str, Any],ctx: AgentContext,) -> dict[str, Any]:+ _clear_pending_browser_interaction_observation(ctx)if result.get("ok"):data = result.pop("data", {})result["url"] = data.get("url", "")@@ -4743,9 +5051,11 @@ async def _click_post_hook(raw: dict[str, Any],ctx: AgentContext,) -> dict[str, Any]:+ _clear_pending_browser_interaction_observation(ctx)if result.get("ok") and result.get("data"):data = result["data"]url, title = await _resolve_url_title(raw, ctx)+ _mark_pending_browser_interaction_observation(ctx, tool_name="click", url=url)result["data"] = {"selector": data.get("selector", ""),"url": url,@@ -4818,6 +5128,7 @@ async def _type_text_post_hook(raw: dict[str, Any],ctx: AgentContext,) -> dict[str, Any]:+ _clear_pending_browser_interaction_observation(ctx)if result.get("ok") and result.get("data"):data = result["data"]selector = data.get("selector", "")@@ -4831,6 +5142,7 @@ async def _type_text_post_hook(landing_failure = await _verify_scout_type_landed(ctx, selector=selector, typed_length=typed_length)if landing_failure is not None:return landing_failure+ _mark_pending_browser_interaction_observation(ctx, tool_name="type_text", url=url)return result@@ -4850,12 +5162,23 @@ async def _evaluate_post_hook(title = str(result["data"].get("title") or "")if not title:_, title = await _resolve_url_title(raw, ctx)- _record_composition_page_observation(+ observation_step = _record_composition_page_observation(ctx,source_tool="evaluate",url=url,title=title,observed_data=result["data"],+ append_to_flow=True,+ reached_via="auto",+ )+ if observation_step is not None:+ result["observation_step"] = observation_step+ result["data"]["observation_step"] = observation_step+ await _maybe_run_completion_verification_from_page_observation(+ ctx,+ url=url,+ title=title,+ observed_data=result["data"],)return result@@ -4881,9 +5204,11 @@ async def _select_option_post_hook(raw: dict[str, Any],ctx: AgentContext,) -> dict[str, Any]:+ _clear_pending_browser_interaction_observation(ctx)if result.get("ok") and result.get("data"):data = result["data"]url, _ = await _resolve_url_title(raw, ctx)+ _mark_pending_browser_interaction_observation(ctx, tool_name="select_option", url=url)result["data"] = {"selector": data.get("selector", ""),"value": data.get("value", ""),@@ -4897,9 +5222,11 @@ async def _press_key_post_hook(raw: dict[str, Any],ctx: AgentContext,) -> dict[str, Any]:+ _clear_pending_browser_interaction_observation(ctx)if result.get("ok") and result.get("data"):data = result["data"]url, _ = await _resolve_url_title(raw, ctx)+ _mark_pending_browser_interaction_observation(ctx, tool_name="press_key", url=url)result["data"] = {"key": data.get("key", ""),"selector": data.get("selector", ""),@@ -5384,7 +5711,8 @@ def _is_unfinished_run_verification_candidate(copilot_ctx: Any, result: dict[strdef _build_run_evidence_snapshot(copilot_ctx: Any, result: dict[str, Any]) -> RunEvidenceSnapshot:data = result.get("data")data = data if isinstance(data, dict) else {}- current_labels = set(_current_workflow_block_labels(copilot_ctx))+ current_label_order = _current_workflow_block_labels(copilot_ctx)+ current_labels = set(current_label_order)# Evidence must be what THIS run produced. ``verified_block_outputs`` accumulates# across incremental runs, so sourcing from it would let an output from a prior# run satisfy a criterion the current run never re-produced.@@ -5399,6 +5727,7 @@ def _build_run_evidence_snapshot(copilot_ctx: Any, result: dict[str, Any]) -> Ruif isinstance(label, str) and label in current_labels and _is_meaningful_extracted_data(output):block_outputs[label] = outputexecuted = data.get("executed_block_labels")+ executed_block_labels = [str(label) for label in executed] if isinstance(executed, list) else []page_title = data.get("page_title")run_id = data.get("workflow_run_id")return RunEvidenceSnapshot(@@ -5406,7 +5735,12 @@ def _build_run_evidence_snapshot(copilot_ctx: Any, result: dict[str, Any]) -> Rublock_outputs=block_outputs,current_url=_valid_runtime_anchor_url(data.get("current_url")),page_title=page_title if isinstance(page_title, str) and page_title.strip() else None,- executed_block_labels=[str(label) for label in executed] if isinstance(executed, list) else [],+ executed_block_labels=executed_block_labels,+ verified_context_block_labels=_verified_context_block_labels_for_snapshot(+ copilot_ctx,+ current_label_order,+ executed_block_labels,+ ),)@@ -5417,11 +5751,7 @@ async def _maybe_run_completion_verification(return Noneif getattr(copilot_ctx, "copilot_total_timeout_exceeded", False):return None- policy = getattr(copilot_ctx, "request_policy", None)- # A method-mandated criterion asserts HOW the goal was reached; this judge sees- # only end-state evidence and would always return no_evidence, false-blocking a- # legitimate success. Verifying method adherence needs an action trace (separate).- criteria = [c for c in (policy.completion_criteria if policy is not None else []) if not c.method_mandated]+ criteria = _completion_verification_criteria(copilot_ctx)if not criteria:return Noneif not (@@ -5495,6 +5825,42 @@ def _outcome_failure_warrants_repair(return _current_workflow_has_evidence_block(copilot_ctx)+def _tool_visible_result_after_completion_verification(+ copilot_ctx: Any,+ result: dict[str, Any],+ completion_verification: CompletionVerificationResult | None,+) -> dict[str, Any]:+ outcome_unverified_reason = _outcome_unverified_reason(copilot_ctx, completion_verification)+ if outcome_unverified_reason is None:+ return result+ if not _outcome_failure_warrants_repair(copilot_ctx, completion_verification):+ return result++ data = result.get("data")+ copied_data = dict(data) if isinstance(data, dict) else {}+ copied_data["failure_reason"] = outcome_unverified_reason+ copied_data["completion_verification"] = (+ completion_verification.to_trace_data() if completion_verification is not None else None+ )+ categories = copied_data.get("failure_categories")+ copied_categories = list(categories) if isinstance(categories, list) else []+ copied_categories.insert(+ 0,+ {+ "category": "OUTCOME_UNVERIFIED",+ "confidence_float": 1.0,+ "reasoning": outcome_unverified_reason,+ },+ )+ copied_data["failure_categories"] = copied_categories+ return {+ **result,+ "ok": False,+ "error": outcome_unverified_reason,+ "data": copied_data,+ }++def _emit_completion_verification_trace(copilot_ctx: Any, completion_verification: CompletionVerificationResult) -> None:@@ -5769,6 +6135,27 @@ async def update_workflow_tool(record_tool_step_result_for_ctx(copilot_ctx, "update_workflow", arguments, result)return json.dumps(result)+ with copilot_span(+ "composition_evidence_precheck",+ data={**_COMPOSITION_EVIDENCE_PRECHECK_TRACE_DATA, "tool_name": "update_workflow"},+ ):+ composition_evidence_error = _update_and_run_blocks_composition_evidence_precheck(+ copilot_ctx,+ workflow_yaml,+ normalized_block_observation_refs,+ block_observation_refs,+ )+ if composition_evidence_error:+ result = {"ok": False, "error": composition_evidence_error}+ record_tool_step_result_for_ctx(copilot_ctx, "update_workflow", arguments, result)+ _record_diagnosis_repair_contract(+ copilot_ctx,+ source_tool="update_workflow",+ result=result,+ )+ sanitized = sanitize_tool_result_for_llm("update_workflow", result)+ return json.dumps(sanitized)+with copilot_span("update_workflow", data={"yaml_length": len(workflow_yaml)}):result = await _update_workflow({**arguments, "raw_block_observation_refs": block_observation_refs},@@ -5835,6 +6222,10 @@ def _frontier_run_size_result(block_labels: list[str],labels_to_execute: list[str],) -> dict[str, Any]:+ suggested_labels = list(labels_to_execute[:_MAX_INCREMENTAL_PAGE_FRONTIER_LABELS])+ user_facing_summary = (+ "Workflow draft saved; I still need to test the next smaller browser frontier before continuing."+ )return {"ok": False,"error": error,@@ -5845,8 +6236,16 @@ def _frontier_run_size_result("requested_block_labels": list(block_labels),"executed_block_labels": [],"planned_block_labels": list(labels_to_execute),- "suggested_block_labels": list(labels_to_execute[:_MAX_INCREMENTAL_PAGE_FRONTIER_LABELS]),+ "suggested_block_labels": suggested_labels,"deferred_block_labels": list(labels_to_execute[_MAX_INCREMENTAL_PAGE_FRONTIER_LABELS:]),+ "control_signal": {+ "kind": "intermediate_success",+ "user_facing_summary": user_facing_summary,+ "next_tool": "run_blocks_and_collect_debug",+ "next_block_labels": suggested_labels,+ "preserve_workflow_yaml": True,+ },+ "user_facing_summary": user_facing_summary,},}@@ -5934,15 +6333,20 @@ async def run_blocks_tool()completion_verification = await _maybe_run_completion_verification(copilot_ctx, result, handler_start)_record_run_blocks_result(copilot_ctx, result, completion_verification=completion_verification)- record_tool_step_result_for_ctx(copilot_ctx, "run_blocks_and_collect_debug", arguments, result)+ tool_visible_result = _tool_visible_result_after_completion_verification(+ copilot_ctx,+ result,+ completion_verification,+ )+ record_tool_step_result_for_ctx(copilot_ctx, "run_blocks_and_collect_debug", arguments, tool_visible_result)_record_diagnosis_repair_contract(copilot_ctx,source_tool="run_blocks_and_collect_debug",- result=result,+ result=tool_visible_result,)enqueue_screenshot_from_result(copilot_ctx, result)- sanitized = sanitize_tool_result_for_llm("run_blocks_and_collect_debug", result)+ sanitized = sanitize_tool_result_for_llm("run_blocks_and_collect_debug", tool_visible_result)return json.dumps(sanitized)@@ -6029,8 +6433,8 @@ async def update_and_run_blocks_tool(for the reusable actions/checks the workflow actually needs.When you compose no-url blocks from a page reached by prior clicks, include`block_observation_refs` entries with each block label and the- `observation_step` returned by inspect_page_for_composition for the page- that block acts on.+ `observation_step` returned by inspect_page_for_composition or evaluate for+ the page that block acts on.When inspected evidence shows an anti-bot challenge gating a disabledsubmit/search control, account for challenge resolution before submit;do not compose a click against a control observed as disabled.@@ -6076,6 +6480,8 @@ async def update_and_run_blocks_tool(sanitized = sanitize_tool_result_for_llm("update_and_run_blocks", result)return json.dumps(sanitized)+ _clear_pending_browser_interaction_observation(copilot_ctx)+# Snapshot the prior workflow definition BEFORE _update_workflow saves# the new one — we need the pre-update state to diff against.prior_definition = await _get_prior_workflow_definition(copilot_ctx)@@ -6217,18 +6623,23 @@ async def update_and_run_blocks_tool()completion_verification = await _maybe_run_completion_verification(copilot_ctx, run_result, handler_start)_record_run_blocks_result(copilot_ctx, run_result, completion_verification=completion_verification)- record_tool_step_result_for_ctx(copilot_ctx, "update_and_run_blocks", arguments, run_result)+ tool_visible_result = _tool_visible_result_after_completion_verification(+ copilot_ctx,+ run_result,+ completion_verification,+ )+ record_tool_step_result_for_ctx(copilot_ctx, "update_and_run_blocks", arguments, tool_visible_result)_record_diagnosis_repair_contract(copilot_ctx,source_tool="update_and_run_blocks",- result=run_result,+ result=tool_visible_result,workflow_updated=True,)enqueue_screenshot_from_result(copilot_ctx, run_result)if run_result.get("ok"):advance_to_testing(copilot_ctx)- sanitized = sanitize_tool_result_for_llm("run_blocks_and_collect_debug", run_result)+ sanitized = sanitize_tool_result_for_llm("run_blocks_and_collect_debug", tool_visible_result)return json.dumps(sanitized)@@ -6268,6 +6679,8 @@ _DISCOVERY_ANTI_BOT_PATTERNS = ("just a moment","captcha","challenge",+ "turnstile",+ "cf-turnstile","human-verification","human verification","verify you are human",@@ -6652,17 +7065,21 @@ def _composition_visual_prompt(evidence: dict[str, Any]) -> str:"challenge_state": evidence.get("challenge_state") or {},"form_count": len(evidence.get("forms") or []),"result_container_count": len(evidence.get("result_containers") or []),+ "page_obstruction_count": len(evidence.get("page_obstructions") or []),"schema_empty_page": evidence.get("schema_empty_page") is True,}return ("Summarize this screenshot for Workflow Copilot build-time page evidence. ""Return JSON only with keys: summary, challenge_detected, challenge_kind, ""challenge_location, submit_blocked, blocked_submit_controls, empty_page_visible, "- "loading_state_visible, omissions. Focus on "+ "loading_state_visible, page_obstruction_detected, obstruction_kind, "+ "obstruction_location, underlying_page_blocked, visible_dismiss_controls, omissions. Focus on ""visible anti-bot or human-verification state, whether it appears to gate a submit/search "- "control, and where it appears relative to the page controls. Do not include raw DOM, "+ "control, whether any visible artificial barrier mechanically blocks the underlying page, "+ "and where it appears relative to the page controls. Do not include raw DOM, ""code, selectors, personal data, or workflow instructions. If no challenge is visible, "- "set challenge_detected to false and submit_blocked to false. If DOM context shows a "+ "set challenge_detected to false and submit_blocked to false. If no page obstruction is visible, "+ "set page_obstruction_detected to false. If DOM context shows a ""schema-empty page, set empty_page_visible to true only when the screenshot shows a ""settled page with no visible forms, controls, result data, challenge, or loading/progress ""state; set loading_state_visible to true when the page appears to be waiting, loading, "@@ -6689,6 +7106,11 @@ def _normalize_visual_summary(value: Any) -> dict[str, Any] | None:blocked_submit_controls = value.get("blocked_submit_controls")empty_page_visible = value.get("empty_page_visible")loading_state_visible = value.get("loading_state_visible")+ page_obstruction_detected = value.get("page_obstruction_detected")+ obstruction_kind = value.get("obstruction_kind")+ obstruction_location = value.get("obstruction_location")+ underlying_page_blocked = value.get("underlying_page_blocked")+ visible_dismiss_controls = value.get("visible_dismiss_controls")omissions = value.get("omissions")return {"summary": summary if isinstance(summary, str) else "",@@ -6698,9 +7120,16 @@ def _normalize_visual_summary(value: Any) -> dict[str, Any] | None:"submit_blocked": submit_blocked if isinstance(submit_blocked, bool) else None,"empty_page_visible": empty_page_visible if isinstance(empty_page_visible, bool) else None,"loading_state_visible": loading_state_visible if isinstance(loading_state_visible, bool) else None,+ "page_obstruction_detected": page_obstruction_detected if isinstance(page_obstruction_detected, bool) else None,+ "obstruction_kind": obstruction_kind if isinstance(obstruction_kind, str) else "",+ "obstruction_location": obstruction_location if isinstance(obstruction_location, str) else "",+ "underlying_page_blocked": underlying_page_blocked if isinstance(underlying_page_blocked, bool) else None,"blocked_submit_controls": [item for item in blocked_submit_controls if isinstance(item, str)]if isinstance(blocked_submit_controls, list)else [],+ "visible_dismiss_controls": [item for item in visible_dismiss_controls if isinstance(item, str)]+ if isinstance(visible_dismiss_controls, list)+ else [],"omissions": [item for item in omissions if isinstance(item, str)] if isinstance(omissions, list) else [],}@@ -6820,18 +7249,19 @@ async def _composition_evidence_after_navigation_failure(_MAX_FLOW_EVIDENCE = 64-def _inspection_reached_via(*, use_current_page: bool, post_run: bool) -> str:+def _inspection_reached_via(*, use_current_page: bool, post_run: bool, earned_interaction: bool) -> str:"""How the just-inspected state was reached, for the flow-evidence trajectory.A target_url inspection navigates there itself ("navigate"); a post-run- current-page inspection observes the page the run left behind ("post_run"); any- other current-page inspection follows an in-turn interaction on that page.+ current-page inspection observes the page the run left behind ("post_run"); a+ normal current-page inspection counts as an interaction only when a successful+ browser action immediately earned that credit."""if not use_current_page:return "navigate"if post_run:return "post_run"- return "interaction"+ return "interaction" if earned_interaction else "current_page"def _append_flow_evidence(copilot_ctx: Any, evidence: dict[str, Any], *, reached_via: str) -> int | None:@@ -6869,6 +7299,61 @@ def _append_flow_evidence(copilot_ctx: Any, evidence: dict[str, Any], *, reachedreturn step+def _latest_interaction_reached_flow_evidence(copilot_ctx: Any) -> tuple[int, str, dict[str, Any]] | None:+ trajectory = getattr(copilot_ctx, "flow_evidence", None)+ if not isinstance(trajectory, list):+ return None+ for entry in reversed(trajectory):+ if not isinstance(entry, dict):+ continue+ reached_via = str(entry.get("reached_via") or "")+ if reached_via not in {"interaction", "post_run"}:+ continue+ evidence = entry.get("evidence")+ step = entry.get("step")+ if isinstance(step, bool) or not isinstance(step, int) or not isinstance(evidence, dict):+ continue+ if not has_bounded_page_schema(evidence):+ continue+ observed_url = _composition_evidence_page_url(evidence)+ if observed_url:+ return step, observed_url, evidence+ return None+++def _non_current_inspection_regression_error(copilot_ctx: Any, *, entry_url: str) -> dict[str, Any] | None:+ latest = _latest_interaction_reached_flow_evidence(copilot_ctx)+ if latest is None:+ return None+ observation_step, observed_url, _ = latest+ if _same_page_ignoring_fragment(observed_url, entry_url):+ return None+ return {+ "ok": False,+ "data": {+ "current_url": observed_url,+ "observation_step": observation_step,+ },+ "error": (+ "inspect_page_for_composition would navigate away from the latest interaction-reached page "+ f'({observed_url}). Use inspect_page_for_composition(target_url="current_page") to inspect '+ "the live page, or compose from the existing page evidence and pass observation_step "+ f"{observation_step} in block_observation_refs for blocks that act on that reached page."+ ),+ }+++def _page_inspection_budget_error(copilot_ctx: Any, *, scope: Literal["turn", "chat"]) -> str:+ scope_label = "turn" if scope == "turn" else "chat"+ return (+ f"inspect_page_for_composition reached the page-inspection budget for this {scope_label}. "+ "This is not evidence that scouting is complete. Use evaluate, get_browser_screenshot, or a browser "+ "action on the current page to determine whether the goal is already satisfied, whether progress is still "+ "possible, or whether a real blocker exists. Do not author downstream result, extraction, or confirmation "+ "blocks unless the existing evidence already shows the page state those blocks will act on."+ )++_COMPOSITION_HOLLOW_RECAPTURE_RETRIES = 2_COMPOSITION_HOLLOW_RECAPTURE_DELAY_SECONDS = 2.5# The composition inspect navigates with `domcontentloaded`, so a heavier cap than@@ -6998,6 +7483,8 @@ async def _inspect_page_for_composition_impl(return resultuse_current_page = (target_url or "").strip().lower() in _CURRENT_PAGE_INSPECTION_TARGETS+ if not use_current_page:+ _clear_pending_browser_interaction_observation(copilot_ctx)bypass_budget_for_post_run_current_page = _allows_post_run_current_page_inspection_budget_bypass(copilot_ctx,use_current_page=use_current_page,@@ -7010,11 +7497,7 @@ async def _inspect_page_for_composition_impl(result = {"ok": False,"data": None,- "error": (- "inspect_page_for_composition reached the page-inspection budget for this turn. "- "Compose from existing evidence or ask for clarification if inspection still cannot determine "- "required page inputs."- ),+ "error": _page_inspection_budget_error(copilot_ctx, scope="turn"),}record_tool_step_result_for_ctx(copilot_ctx, "inspect_page_for_composition", arguments, result)return result@@ -7024,7 +7507,7 @@ async def _inspect_page_for_composition_impl(result = {"ok": False,"data": None,- "error": "inspect_page_for_composition has reached the page-inspection budget for this chat. Compose from existing evidence or ask for clarification.",+ "error": _page_inspection_budget_error(copilot_ctx, scope="chat"),}record_tool_step_result_for_ctx(copilot_ctx, "inspect_page_for_composition", arguments, result)return result@@ -7046,6 +7529,10 @@ async def _inspect_page_for_composition_impl(record_tool_step_result_for_ctx(copilot_ctx, "inspect_page_for_composition", arguments, result)return resultentry_url = resolved_entry_url+ regression_error = _non_current_inspection_regression_error(copilot_ctx, entry_url=entry_url)+ if regression_error is not None:+ record_tool_step_result_for_ctx(copilot_ctx, "inspect_page_for_composition", arguments, regression_error)+ return regression_errorwith copilot_span("inspect_page_for_composition",@@ -7128,7 +7615,30 @@ async def _inspect_page_for_composition_impl(if bypass_budget_for_post_run_current_page:copilot_ctx.post_run_current_page_inspection_workflow_run_id = run_idcopilot_ctx.composition_page_evidence = evidence- reached_via = _inspection_reached_via(use_current_page=use_current_page, post_run=bool(run_id))+ if (+ isinstance(run_id, str)+ and run_id+ and getattr(copilot_ctx, "post_run_page_observation_after_failed_test", False)+ ):+ page_title = evidence.get("page_title")+ await _maybe_run_completion_verification_from_page_observation(+ copilot_ctx,+ url=str(evidence.get("current_url") or current_url or ""),+ title=page_title if isinstance(page_title, str) else "",+ observed_data=evidence,+ )+ earned_interaction = False+ if use_current_page and not run_id:+ earned_interaction = _consume_pending_browser_interaction_observation(+ copilot_ctx,+ current_url=str(evidence.get("current_url") or current_url or ""),+ evidence=evidence,+ )+ reached_via = _inspection_reached_via(+ use_current_page=use_current_page,+ post_run=bool(run_id),+ earned_interaction=earned_interaction,+ )observation_step = _append_flow_evidence(copilot_ctx, evidence, reached_via=reached_via)if observation_step is None:LOG.warning("copilot_flow_evidence_append_failed_no_trajectory")diff --git a/skyvern/forge/sdk/copilot/turn_intent.py b/skyvern/forge/sdk/copilot/turn_intent.pyindex 6b5fbec00..ca69ee976 100644--- a/skyvern/forge/sdk/copilot/turn_intent.py+++ b/skyvern/forge/sdk/copilot/turn_intent.py@@ -570,7 +570,6 @@ def build_turn_intent(required_context.append(RequiredContextKey.LATEST_RUN_RESULT)reason_codes.append(TurnIntentReasonCode.RUN_CONTEXT_PRESENT)if browser_session_id:- required_context.append(RequiredContextKey.BROWSER_STATE)reason_codes.append(TurnIntentReasonCode.BROWSER_CONTEXT_PRESENT)if request_policy.credential_input_kind != "none" or request_policy.resolved_credentials:required_context.append(RequiredContextKey.CREDENTIAL_METADATA)diff --git a/skyvern/forge/sdk/routes/__init__.py b/skyvern/forge/sdk/routes/__init__.pyindex dd58df058..3cca8d97d 100644--- a/skyvern/forge/sdk/routes/__init__.py+++ b/skyvern/forge/sdk/routes/__init__.py@@ -1,3 +1,5 @@+import importlib.util+from skyvern.forge.sdk.routes import agent_protocol # noqa: F401from skyvern.forge.sdk.routes import browser_profiles # noqa: F401from skyvern.forge.sdk.routes import browser_sessions # noqa: F401@@ -10,9 +12,15 @@ from skyvern.forge.sdk.routes import runtime_config # noqa: F401from skyvern.forge.sdk.routes import scripts # noqa: F401from skyvern.forge.sdk.routes import sdk # noqa: F401from skyvern.forge.sdk.routes import webhooks # noqa: F401-from skyvern.forge.sdk.routes import workflow_copilot # noqa: F401from skyvern.forge.sdk.routes import workflow_schedules # noqa: F401from skyvern.forge.sdk.routes.streaming import cdp_input # noqa: F401from skyvern.forge.sdk.routes.streaming import messages # noqa: F401from skyvern.forge.sdk.routes.streaming import screenshot # noqa: F401from skyvern.forge.sdk.routes.streaming import vnc # noqa: F401++# Workflow Copilot depends on openai-agents, imported as `agents`, which is+# intentionally server-only.+# skyvern[local] starts the embedded API app without that extra, so skip these+# private routes unless the SDK package is installed.+if importlib.util.find_spec("agents") is not None:+ from skyvern.forge.sdk.routes import workflow_copilot # noqa: F401diff --git a/skyvern/webeye/real_browser_state.py b/skyvern/webeye/real_browser_state.pyindex 93a5aba29..a62ecd3ea 100644--- a/skyvern/webeye/real_browser_state.py+++ b/skyvern/webeye/real_browser_state.py@@ -36,6 +36,19 @@ SETTLE_TIME_MS = 750SETTLE_JITTER_MS = 500+def _same_page_ignoring_fragment(left: str | None, right: str | None) -> bool:+ if not left or not right:+ return False+ try:+ left_parsed = urlparse(left)+ right_parsed = urlparse(right)+ except Exception:+ return False+ left_url = left_parsed._replace(fragment="").geturl().rstrip("/")+ right_url = right_parsed._replace(fragment="").geturl().rstrip("/")+ return left_url == right_url++class RealBrowserState(BrowserState):def __init__(self,@@ -118,7 +131,7 @@ class RealBrowserState(BrowserState):use_existing_page = Falseif browser_address and len(self.browser_context.pages) > 0:pages = await self.list_valid_pages()- if len(pages) > 0:+ if pages:page = pages[-1]use_existing_page = Trueif page is None:@@ -128,7 +141,7 @@ class RealBrowserState(BrowserState):if not use_existing_page:await self._close_all_other_pages()- if url and page.url.rstrip("/") != url.rstrip("/"):+ if url and not _same_page_ignoring_fragment(page.url, url):await self.navigate_to_url(page=page, url=url)async def _wait_for_settle(self) -> None:diff --git a/tests/unit/test_copilot_completion_verification.py b/tests/unit/test_copilot_completion_verification.pyindex 1dfde1f07..1239f6eaf 100644--- a/tests/unit/test_copilot_completion_verification.py+++ b/tests/unit/test_copilot_completion_verification.py@@ -7,7 +7,11 @@ from types import SimpleNamespaceimport pytestfrom skyvern.config import settings-from skyvern.forge.sdk.copilot.agent import _completion_contract_not_violated, _verified_workflow_or_none+from skyvern.forge.sdk.copilot.agent import (+ _completion_contract_not_violated,+ _rewrite_failed_test_response,+ _verified_workflow_or_none,+)from skyvern.forge.sdk.copilot.completion_verification import (CompletionVerificationResult,CriterionVerdict,@@ -39,9 +43,12 @@ from skyvern.forge.sdk.copilot.tools import (_is_outcome_evidence_candidate,_is_unfinished_run_verification_candidate,_maybe_run_completion_verification,+ _maybe_run_completion_verification_from_page_observation,_outcome_failure_warrants_repair,_outcome_unverified_reason,+ _record_composition_page_observation,_record_run_blocks_result,+ _tool_visible_result_after_completion_verification,)@@ -320,6 +327,33 @@ def test_record_run_blocks_downgrades_when_confirmation_block_present_but_unmet(assert "item in cart" in (ctx.last_test_failure_reason or "")+def test_tool_visible_result_fails_when_confirmation_block_outcome_unmet() -> None:+ ctx = _ctx_with_blocks("extraction")+ result = _clean_success_result()+ verification = _evaluated(("c0", False))++ visible = _tool_visible_result_after_completion_verification(ctx, result, verification)++ assert visible["ok"] is False+ assert "item in cart" in visible["error"]+ assert result["ok"] is True+ assert visible["data"]["overall_status"] == "completed"+ assert visible["data"]["completion_verification"]["fully_satisfied"] is False+ assert visible["data"]["failure_categories"][0]["category"] == "OUTCOME_UNVERIFIED"+++def test_tool_visible_result_keeps_mid_build_run_visible_success() -> None:+ ctx = _ctx_with_blocks("goto_url", "navigation")++ visible = _tool_visible_result_after_completion_verification(+ ctx,+ _clean_success_result(),+ _evaluated(("c0", False)),+ )++ assert visible["ok"] is True++def test_record_run_blocks_keeps_building_on_mid_build_no_evidence() -> None:ctx = _ctx_with_blocks("goto_url", "navigation")_record_run_blocks_result(ctx, _clean_success_result(), completion_verification=_evaluated(("c0", False)))@@ -472,6 +506,187 @@ def test_verified_workflow_presented_on_recognized_canceled_run() -> None:assert _verified_workflow_or_none(ctx) == (None, None)+async def test_page_observation_verification_recognizes_budgeted_outcome(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ seen_prompt: dict[str, str] = {}++ async def handler(**kwargs: object) -> dict:+ seen_prompt["prompt"] = str(kwargs.get("prompt") or "")+ return {"verdicts": [{"criterion_id": "c0", "satisfied": True, "reason_code": "evidence_confirms"}]}++ monkeypatch.setattr(+ "skyvern.forge.sdk.copilot.tools._completion_verification_handler",+ _completion_handler_lookup(handler),+ )+ ctx = _run_ctx()+ ctx.last_test_ok = False+ ctx.last_run_blocks_workflow_run_id = "wr_cancel"+ ctx.copilot_run_start_monotonic = time.monotonic()+ _record_composition_page_observation(+ ctx,+ source_tool="evaluate",+ url="https://example.com/cart",+ title="Shopping Cart",+ observed_data={+ "hasProduct": True,+ "excerpts": ["SKU-12345 is present in the cart"],+ "url": "https://example.com/cart",+ "title": "Shopping Cart",+ },+ )++ result = await _maybe_run_completion_verification_from_page_observation(+ ctx,+ url="https://example.com/cart",+ title="Shopping Cart",+ observed_data={+ "hasProduct": True,+ "excerpts": ["SKU-12345 is present in the cart"],+ },+ )++ assert result is not None+ assert result.is_fully_satisfied() is True+ assert ctx.completion_verification_result is result+ assert outcome_fully_verified(ctx) is True+ assert "current_page_observation" in seen_prompt["prompt"]+ assert "SKU-12345 is present in the cart" in seen_prompt["prompt"]+++async def test_page_observation_verification_does_not_overwrite_satisfied_verdict(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ async def handler(**_: object) -> dict:+ raise AssertionError("handler should not be called once the outcome is verified")++ monkeypatch.setattr(+ "skyvern.forge.sdk.copilot.tools._completion_verification_handler",+ _completion_handler_lookup(handler),+ )+ ctx = _run_ctx()+ ctx.last_test_ok = False+ ctx.last_run_blocks_workflow_run_id = "wr_cancel"+ existing = _evaluated(("c0", True))+ ctx.completion_verification_result = existing+ _record_composition_page_observation(+ ctx,+ source_tool="evaluate",+ url="https://example.com/cart",+ observed_data={"hasProduct": True},+ )++ result = await _maybe_run_completion_verification_from_page_observation(+ ctx,+ url="https://example.com/cart",+ observed_data={"hasProduct": True},+ )++ assert result is existing+ assert ctx.completion_verification_result is existing+++async def test_page_observation_verification_preserves_existing_unsatisfied_verdict(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ handler_calls = 0++ async def handler(**_: object) -> dict:+ nonlocal handler_calls+ handler_calls += 1+ return {"verdicts": [{"criterion_id": "c0", "satisfied": False, "reason_code": "no_evidence"}]}++ monkeypatch.setattr(+ "skyvern.forge.sdk.copilot.tools._completion_verification_handler",+ _completion_handler_lookup(handler),+ )+ ctx = _run_ctx()+ ctx.last_test_ok = False+ ctx.last_run_blocks_workflow_run_id = "wr_cancel"+ existing = _evaluated(("c0", False))+ ctx.completion_verification_result = existing+ _record_composition_page_observation(+ ctx,+ source_tool="evaluate",+ url="https://example.com/cart",+ observed_data={"hasProduct": False},+ )++ result = await _maybe_run_completion_verification_from_page_observation(+ ctx,+ url="https://example.com/cart",+ observed_data={"hasProduct": False},+ )++ assert handler_calls == 1+ assert result is existing+ assert ctx.completion_verification_result is existing+++async def test_page_observation_verification_can_upgrade_unsatisfied_verdict(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ async def handler(**_: object) -> dict:+ return {"verdicts": [{"criterion_id": "c0", "satisfied": True, "reason_code": "evidence_confirms"}]}++ monkeypatch.setattr(+ "skyvern.forge.sdk.copilot.tools._completion_verification_handler",+ _completion_handler_lookup(handler),+ )+ ctx = _run_ctx()+ ctx.last_test_ok = False+ ctx.last_run_blocks_workflow_run_id = "wr_cancel"+ existing = _evaluated(("c0", False))+ ctx.completion_verification_result = existing+ _record_composition_page_observation(+ ctx,+ source_tool="evaluate",+ url="https://example.com/cart",+ observed_data={"hasProduct": True},+ )++ result = await _maybe_run_completion_verification_from_page_observation(+ ctx,+ url="https://example.com/cart",+ observed_data={"hasProduct": True},+ )++ assert result is not None+ assert result is not existing+ assert result.is_fully_satisfied() is True+ assert ctx.completion_verification_result is result+++def test_failed_test_rewrite_recognizes_post_budget_verified_outcome() -> None:+ ctx = _canceled_gate_ctx()+ ctx.last_workflow = SimpleNamespace()+ ctx.last_workflow_yaml = "workflow: {}"+ ctx.last_update_block_count = 5+ ctx.completion_verification_result = _evaluated(("c0", True))++ response = _rewrite_failed_test_response("The test failed.", ctx)++ assert "verified the requested outcome" in response+ assert "test failed" not in response.lower()+++def test_failed_test_rewrite_does_not_render_zero_block_verified_outcome() -> None:+ ctx = _canceled_gate_ctx()+ ctx.last_workflow = SimpleNamespace()+ ctx.last_workflow_yaml = "workflow: {}"+ ctx.last_update_block_count = 0+ ctx.completion_verification_result = _evaluated(("c0", True))++ response = _rewrite_failed_test_response("The test failed.", ctx)++ assert "0 blocks" not in response+ assert "workflow with 0" not in response++# --- SKY-10576: recognition governed by whole-workflow outcome, not per-block prefix ---## A clean ok=True run can reach the goal (its outcome block produced data and the@@ -608,6 +823,150 @@ def test_snapshot_uses_current_run_blocks_not_stale_outputs() -> None:assert snap2.block_outputs.get("b3") == {"extracted_information": {"price": "9.99"}}+def test_snapshot_includes_verified_context_labels_without_prior_outputs() -> None:+ ctx = _run_ctx()+ labels = [+ "open_bacb_homepage",+ "click_find_a_certificant",+ "search_noor_assi_rbt",+ "expand_noor_assi_result",+ "extract_credential_details",+ ]+ ctx.last_workflow = SimpleNamespace(+ workflow_definition=SimpleNamespace(+ blocks=[SimpleNamespace(label=label, block_type="task") for label in labels]+ )+ )+ ctx.verified_prefix_labels = list(labels)+ ctx.verified_block_outputs["expand_noor_assi_result"] = {"stale": "prior run output"}++ run = {+ "ok": True,+ "data": {+ "workflow_run_id": "wr_extract",+ "current_url": "https://www.bacb.com/services/o.php?page=101135",+ "executed_block_labels": ["extract_credential_details"],+ "blocks": [+ {+ "label": "extract_credential_details",+ "block_type": "EXTRACTION",+ "status": "completed",+ "extracted_data": {+ "extracted_information": {+ "credentials": [+ {+ "credential_type": "Registered Behavior Technician",+ "credential_number": "RBT-19-98341",+ "expiration_date": "09/06/2022",+ }+ ]+ }+ },+ }+ ],+ },+ }++ snapshot = _build_run_evidence_snapshot(ctx, run)++ assert snapshot.verified_context_block_labels == labels[:-1]+ assert snapshot.block_outputs == {+ "extract_credential_details": {+ "extracted_information": {+ "credentials": [+ {+ "credential_type": "Registered Behavior Technician",+ "credential_number": "RBT-19-98341",+ "expiration_date": "09/06/2022",+ }+ ]+ }+ }+ }+ assert "expand_noor_assi_result" not in snapshot.block_outputs+ rendered = snapshot.render_prompt_block()+ assert "verified_context_block_labels: open_bacb_homepage" in rendered+ assert "expand_noor_assi_result" in rendered+++async def test_completion_verification_receives_verified_context_labels(monkeypatch: pytest.MonkeyPatch) -> None:+ seen_prompt: dict[str, str] = {}++ async def handler(**kwargs: object) -> dict:+ seen_prompt["prompt"] = str(kwargs.get("prompt") or "")+ return {+ "verdicts": [+ {"criterion_id": "c0", "satisfied": True, "reason_code": "evidence_confirms"},+ {"criterion_id": "c1", "satisfied": True, "reason_code": "evidence_confirms"},+ ]+ }++ monkeypatch.setattr(+ "skyvern.forge.sdk.copilot.tools._completion_verification_handler",+ _completion_handler_lookup(handler),+ )+ monkeypatch.setattr(settings, "COPILOT_OUTCOME_VERIFICATION_ENABLED", True)+ ctx = _run_ctx()+ ctx.request_policy = RequestPolicy(+ completion_criteria=[+ _criterion("c0", "credential type, number, and expiration date are reported"),+ _criterion("c1", "the data came from the expanded certificant result"),+ ]+ )+ labels = [+ "open_bacb_homepage",+ "click_find_a_certificant",+ "search_noor_assi_rbt",+ "expand_noor_assi_result",+ "extract_credential_details",+ ]+ ctx.last_workflow = SimpleNamespace(+ workflow_definition=SimpleNamespace(+ blocks=[+ SimpleNamespace(label=label, block_type="extraction" if label.startswith("extract") else "task")+ for label in labels+ ]+ )+ )+ ctx.verified_prefix_labels = list(labels)+ result = {+ "ok": True,+ "data": {+ "workflow_run_id": "wr_extract",+ "current_url": "https://www.bacb.com/services/o.php?page=101135",+ "executed_block_labels": ["extract_credential_details"],+ "blocks": [+ {+ "label": "extract_credential_details",+ "block_type": "EXTRACTION",+ "status": "completed",+ "extracted_data": {+ "extracted_information": {+ "person_name": "NOOR ASSI",+ "credentials": [+ {+ "credential_type": "Registered Behavior Technician",+ "credential_number": "RBT-19-98341",+ "expiration_date": "09/06/2022",+ }+ ],+ }+ },+ }+ ],+ },+ }++ verification = await _maybe_run_completion_verification(ctx, result, time.monotonic())++ assert verification is not None+ assert verification.is_fully_satisfied() is True+ assert "verified_context_block_labels" in seen_prompt["prompt"]+ assert "expand_noor_assi_result" in seen_prompt["prompt"]+ assert "RBT-19-98341" in seen_prompt["prompt"]++@pytest.mark.asyncioasync def test_maybe_run_completion_verification_unavailable_on_low_budget(monkeypatch: pytest.MonkeyPatch) -> None:async def handler(**_: object) -> dict:diff --git a/tests/unit/test_copilot_composition_evidence.py b/tests/unit/test_copilot_composition_evidence.pyindex e95eef05f..aa4a899f4 100644--- a/tests/unit/test_copilot_composition_evidence.py+++ b/tests/unit/test_copilot_composition_evidence.py@@ -10,6 +10,7 @@ import yamlfrom skyvern.forge.sdk.copilot.build_phase import BuildPhasefrom skyvern.forge.sdk.copilot.composition_evidence import (composition_page_evidence_error,+ has_bounded_page_schema,merge_visual_composition_evidence,normalize_block_observation_refs,page_evidence_needs_visual_fallback,@@ -124,6 +125,109 @@ def test_composition_parse_html_extracts_labeled_fields_and_submit_controls() ->assert parsed["source_tool"] == "inspect_page_for_composition"+def test_composition_parse_html_extracts_modal_overlay_controls() -> None:+ parsed = parse_composition_html(+ """+ <html><body>+ <div id="newsletter" role="dialog" aria-modal="true" class="promo-modal">+ <h2>Get updates</h2>+ <p>Join our list before browsing.</p>+ <button aria-label="Close modal">x</button>+ <button>No thanks</button>+ </div>+ </body></html>+ """,+ inspected_url="https://example.com/results",+ current_url="https://example.com/results",+ )++ assert parsed["modal_overlays"][0]["selector"] == "#newsletter"+ assert parsed["modal_overlays"][0]["role"] == "dialog"+ assert parsed["modal_overlays"][0]["dismiss_controls"][0]["text"] == "x"+ assert parsed["modal_overlays"][0]["dismiss_controls"][0]["aria_label"] == "Close modal"+ assert parsed["page_obstructions"][0]["kind"] == "modal_overlay"+ assert parsed["page_obstructions"][0]["visible_controls"][0]["text"] == "x"+ assert has_bounded_page_schema(parsed) is True+++def test_composition_parse_html_extracts_class_only_modal_overlay() -> None:+ parsed = parse_composition_html(+ """+ <html><body>+ <div class="promo modal">+ <h2>Before you continue</h2>+ <button>Close</button>+ </div>+ </body></html>+ """,+ inspected_url="https://example.com/results",+ current_url="https://example.com/results",+ )++ assert parsed["modal_overlays"][0]["class"] == "promo modal"+ assert parsed["modal_overlays"][0]["dismiss_controls"][0]["text"] == "Close"+ assert parsed["page_obstructions"][0]["visible_controls"][0]["text"] == "Close"+ assert has_bounded_page_schema(parsed) is True+++def test_composition_parse_html_ignores_hidden_modal_overlay_markup() -> None:+ parsed = parse_composition_html(+ """+ <html><body>+ <div id="closedDialog" role="dialog" aria-modal="true" aria-hidden="true">+ <button>Close</button>+ </div>+ <div id="closedModal" class="modal" style="display: none;">+ <button>Dismiss</button>+ </div>+ <div aria-hidden="true">+ <div id="wrappedDialog" role="dialog">+ <button>Close</button>+ </div>+ </div>+ </body></html>+ """,+ inspected_url="https://example.com/results",+ current_url="https://example.com/results",+ )++ assert parsed["modal_overlays"] == []+ assert parsed["page_obstructions"] == []+++def test_composition_parse_html_ignores_empty_modal_root_as_bounded_schema() -> None:+ parsed = parse_composition_html(+ """+ <html><body>+ <div id="modal-root"></div>+ </body></html>+ """,+ inspected_url="https://example.com/results",+ current_url="https://example.com/results",+ )++ assert parsed["modal_overlays"] == []+ assert has_bounded_page_schema(parsed) is False+++def test_composition_parse_html_does_not_treat_next_as_modal_dismiss_control() -> None:+ parsed = parse_composition_html(+ """+ <html><body>+ <div id="modal-root">+ <button>Next</button>+ <button>Export</button>+ </div>+ </body></html>+ """,+ inspected_url="https://example.com/results",+ current_url="https://example.com/results",+ )++ assert parsed["modal_overlays"] == []+ assert has_bounded_page_schema(parsed) is False++def test_composition_parse_html_preserves_stable_control_selectors_and_values() -> None:parsed = parse_composition_html("""@@ -288,6 +392,11 @@ def test_merge_visual_composition_evidence_keeps_screenshot_bounded_and_typed()"challenge_location": "Below the acknowledgement checkbox and above the Search button.","submit_blocked": True,"blocked_submit_controls": ["Search"],+ "page_obstruction_detected": True,+ "obstruction_kind": "verification_panel",+ "obstruction_location": "Centered above the search form.",+ "underlying_page_blocked": True,+ "visible_dismiss_controls": ["Continue"],"omissions": ["Result rows are not visible before verification."],},)@@ -302,9 +411,39 @@ def test_merge_visual_composition_evidence_keeps_screenshot_bounded_and_typed())assert merged["challenge_state"]["gates_submit_controls"] is Trueassert merged["challenge_state"]["gated_submit_controls"] == [{"text": "Search", "disabled": True}]+ assert merged["page_obstructions"] == [+ {+ "kind": "verification_panel",+ "source": "vision_summary",+ "visual_location": "Centered above the search form.",+ "visible_controls": [{"text": "Continue"}],+ "underlying_page_blocked": True,+ }+ ]assert merged["visual_evidence_omissions"] == ["Result rows are not visible before verification."]+def test_merge_visual_composition_evidence_keeps_false_underlying_page_blocked() -> None:+ parsed = parse_composition_html(+ "<html><head><title>Search</title></head><body><form><input name='q' /></form></body></html>",+ inspected_url="https://example.com/search",+ current_url="https://example.com/search",+ )++ merged = merge_visual_composition_evidence(+ parsed,+ visual_summary={+ "summary": "A banner is visible but the search form remains usable.",+ "page_obstruction_detected": True,+ "obstruction_kind": "banner",+ "obstruction_location": "Bottom of viewport.",+ "underlying_page_blocked": False,+ },+ )++ assert merged["page_obstructions"][0]["underlying_page_blocked"] is False++def test_composition_parse_html_surfaces_human_verification_controls_after_long_page_preamble() -> None:parsed = parse_composition_html(f"""@@ -1269,6 +1408,27 @@ def test_composition_gate_rejects_click_reached_blocks_reusing_entrypoint_observassert "add_first_result (action)" in error+def test_composition_gate_allows_current_page_read_after_matching_interaction_reached_page() -> None:+ workflow_yaml = _yaml(+ {"block_type": "goto_url", "label": "open_home", "url": "https://example.com/"},+ {"block_type": "action", "label": "search_product", "navigation_goal": "Search for the product."},+ {"block_type": "action", "label": "add_first_result", "navigation_goal": "Add the first result to the cart."},+ )+ ctx = _Ctx(+ flow_evidence=[+ _flow_entry("https://example.com/", reached_via="navigate", step=0),+ _flow_entry("https://example.com/results?s=1", reached_via="interaction", step=1),+ _flow_entry("https://example.com/results?s=1", reached_via="current_page", step=2),+ ],+ block_observation_refs={+ "search_product": 0,+ "add_first_result": 2,+ },+ )++ assert composition_page_evidence_error(ctx, workflow_yaml) is None++def test_composition_gate_reports_missing_referenced_observation_step() -> None:workflow_yaml = _yaml({"block_type": "goto_url", "label": "open_home", "url": "https://example.com/"},diff --git a/tests/unit/test_copilot_discovery.py b/tests/unit/test_copilot_discovery.pyindex a4283a504..1f16c0a96 100644--- a/tests/unit/test_copilot_discovery.py+++ b/tests/unit/test_copilot_discovery.py@@ -7,6 +7,7 @@ from typing import Anyimport pytestfrom skyvern.forge.sdk.copilot import tools as tools_module+from skyvern.forge.sdk.copilot.runtime import PendingBrowserInteractionObservationfrom skyvern.forge.sdk.copilot.tools import _discovery_walk, _inspect_page_for_composition_implfrom skyvern.forge.sdk.copilot.verification_evidence import WorkflowVerificationEvidence@@ -18,6 +19,9 @@ class _Ctx:self.discovery_step_count = 0self.prior_page_inspection_calls_made = 0self.page_inspection_calls_this_turn = 0+ self.flow_evidence: list[dict[str, Any]] = []+ self.composition_page_evidence = None+ self.pending_browser_interaction_observation = Noneself.workflow_verification_evidence = WorkflowVerificationEvidence()@@ -179,6 +183,27 @@ class _CurrentPageServer:raise AssertionError(f"unexpected tool: {tool_name}")+class _TargetThenCurrentPageServer:+ def __init__(self) -> None:+ self.calls: list[tuple[str, dict[str, Any]]] = []+ self.current_url = ""++ async def call_internal_tool(self, tool_name: str, arguments: dict[str, Any]) -> dict[str, Any]:+ self.calls.append((tool_name, arguments))+ if tool_name == "skyvern_navigate":+ self.current_url = arguments["url"]+ return {"ok": True, "data": {"url": self.current_url}}+ if tool_name == "skyvern_get_html":+ assert arguments == {"selector": "body"}+ return {+ "ok": True,+ "data": {+ "html": "<html><body><form><input name='firstName'><button>Search</button></form></body></html>"+ },+ }+ raise AssertionError(f"unexpected tool: {tool_name}")++@pytest.mark.asyncioasync def test_discovery_navigation_failure_falls_back_to_entry_url() -> None:result = await _discovery_walk(@@ -321,6 +346,94 @@ async def test_post_run_current_page_inspection_budget_bypass_does_not_consume_cassert ctx.post_run_current_page_inspection_workflow_run_id == "wr_123" # type: ignore[attr-defined]+async def test_current_page_inspection_without_earned_interaction_is_not_click_reached(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ server = _CurrentPageServer()+ ctx = _Ctx(server)++ async def fake_fallback_page_info(_ctx: object) -> tuple[str, str]:+ return "https://www.example.com/results", "Results"++ monkeypatch.setattr(tools_module, "_fallback_page_info", fake_fallback_page_info)++ result = await _inspect_page_for_composition_impl(ctx, "current_page")++ assert result["ok"] is True+ assert result["reached_via"] == "current_page"+ assert ctx.flow_evidence[0]["reached_via"] == "current_page"+++async def test_current_page_inspection_after_browser_action_is_click_reached_once(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ server = _CurrentPageServer()+ ctx = _Ctx(server)+ ctx.pending_browser_interaction_observation = PendingBrowserInteractionObservation(+ tool_name="click",+ url="https://www.example.com/results",+ )++ async def fake_fallback_page_info(_ctx: object) -> tuple[str, str]:+ return "https://www.example.com/results", "Results"++ monkeypatch.setattr(tools_module, "_fallback_page_info", fake_fallback_page_info)++ result = await _inspect_page_for_composition_impl(ctx, "current_page")++ assert result["ok"] is True+ assert result["reached_via"] == "interaction"+ assert ctx.flow_evidence[0]["reached_via"] == "interaction"+ assert ctx.pending_browser_interaction_observation is None+++async def test_inspection_budget_steers_progress_check_instead_of_authoring() -> None:+ ctx = _Ctx(server=object())+ ctx.page_inspection_calls_this_turn = 999++ result = await _inspect_page_for_composition_impl(ctx, "current_page")++ assert result["ok"] is False+ assert "not evidence that scouting is complete" in result["error"]+ assert "evaluate" in result["error"]+ assert "get_browser_screenshot" in result["error"]+ assert "browser action on the current page" in result["error"]+ assert "Do not author downstream result" in result["error"]+ assert "Compose from existing evidence" not in result["error"]+++async def test_target_url_inspection_clears_pending_interaction_credit(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ server = _TargetThenCurrentPageServer()+ ctx = _Ctx(server)+ ctx.pending_browser_interaction_observation = PendingBrowserInteractionObservation(+ tool_name="click",+ url="https://www.example.com/results",+ )++ target_result = await _inspect_page_for_composition_impl(ctx, "https://www.example.com/results")++ assert target_result["ok"] is True+ assert target_result["reached_via"] == "navigate"+ assert ctx.pending_browser_interaction_observation is None++ async def fake_fallback_page_info(_ctx: object) -> tuple[str, str]:+ return "https://www.example.com/results", "Results"++ monkeypatch.setattr(tools_module, "_fallback_page_info", fake_fallback_page_info)++ current_result = await _inspect_page_for_composition_impl(ctx, "current_page")++ assert current_result["ok"] is True+ assert current_result["reached_via"] == "current_page"+ assert [entry["reached_via"] for entry in ctx.flow_evidence] == ["navigate", "current_page"]++class _SizeCappedHtmlStrippedFallbackServer:"""Every page's get_html is dropped by the MCP size cap (a heavy DOM exceeds it).The stripped-body evaluate fallback recovers each page, so the resolver can stilldiff --git a/tests/unit/test_copilot_evaluate_page_evidence.py b/tests/unit/test_copilot_evaluate_page_evidence.pynew file mode 100644index 000000000..003028851--- /dev/null+++ b/tests/unit/test_copilot_evaluate_page_evidence.py@@ -0,0 +1,149 @@+from __future__ import annotations++from unittest.mock import MagicMock++import pytest++from skyvern.forge.sdk.copilot.build_phase import BuildPhase+from skyvern.forge.sdk.copilot.context import CopilotContext+from skyvern.forge.sdk.copilot.request_policy import RequestPolicy+from skyvern.forge.sdk.copilot.tools import (+ _evaluate_post_hook,+ _inspect_page_for_composition_impl,+ _mark_pending_browser_interaction_observation,+)+from skyvern.forge.sdk.copilot.turn_intent import TurnIntent, TurnIntentAuthority, TurnIntentMode+++def _ctx() -> CopilotContext:+ return CopilotContext(+ organization_id="org-1",+ workflow_id="wf-1",+ workflow_permanent_id="wfp-1",+ workflow_yaml="",+ browser_session_id=None,+ stream=MagicMock(),+ request_policy=RequestPolicy(),+ build_phase=BuildPhase.COMPOSING,+ turn_intent=TurnIntent(+ mode=TurnIntentMode.BUILD,+ authority=TurnIntentAuthority(may_update_workflow=True, may_run_blocks=True),+ ),+ )+++async def test_evaluate_nested_rows_records_interaction_observation_step() -> None:+ ctx = _ctx()+ _mark_pending_browser_interaction_observation(+ ctx,+ tool_name="click",+ url="https://example.test/search/results?s=1",+ )++ result = {+ "ok": True,+ "data": {+ "url": "https://example.test/search/results?s=1",+ "title": "Search Results",+ "result": {+ "rows": [+ {+ "cells": [+ {"text": "Example Person"},+ {"text": "Credential A"},+ {"text": "Number 123"},+ {"text": "Expiration 2030-01-01"},+ ]+ }+ ]+ },+ },+ }++ updated = await _evaluate_post_hook(result, raw={}, ctx=ctx)++ assert updated["observation_step"] == 0+ assert updated["data"]["observation_step"] == 0+ assert len(ctx.flow_evidence) == 1+ assert ctx.flow_evidence[0]["reached_via"] == "interaction"+ assert ctx.flow_evidence[0]["had_bounded_schema"] is True+ assert ctx.flow_evidence[0]["step"] == 0+ evidence = ctx.flow_evidence[0]["evidence"]+ assert evidence["source_tool"] == "evaluate"+ assert evidence["current_url"] == "https://example.test/search/results?s=1"+ assert evidence["result_containers"][0]["row_count"] == 1+ assert "Credential A" in evidence["result_containers"][0]["sample_rows"][0]+ assert ctx.composition_page_evidence is evidence+++async def test_evaluate_turnstile_key_records_challenge_observation_step() -> None:+ ctx = _ctx()+ _mark_pending_browser_interaction_observation(+ ctx,+ tool_name="type_text",+ url="https://example.test/certificant-search",+ )++ result = {+ "ok": True,+ "data": {+ "url": "https://example.test/certificant-search",+ "title": "Certificant Search",+ "text": "Verify you are human before searching.",+ "turnstile": True,+ "btnDisabled": True,+ },+ }++ updated = await _evaluate_post_hook(result, raw={}, ctx=ctx)++ assert updated["observation_step"] == 0+ assert len(ctx.flow_evidence) == 1+ evidence = ctx.flow_evidence[0]["evidence"]+ assert evidence["source_tool"] == "evaluate"+ assert evidence["challenge_state"]["detected"] is True+ assert evidence["challenge_state"]["kind"] == "captcha"+ assert evidence["challenge_state"]["gates_submit_controls"] is True+ assert evidence["challenge_state"]["gated_submit_controls"][0]["disabled"] is True+ assert "turnstile" in evidence["anti_bot_indicators"]+ assert ctx.composition_page_evidence is evidence+++async def test_target_url_inspection_does_not_navigate_away_from_interaction_evidence(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ ctx = _ctx()+ ctx.flow_evidence.append(+ {+ "evidence": {+ "source_tool": "evaluate",+ "current_url": "https://example.test/search/results?s=1",+ "inspected_url": "https://example.test/search/results?s=1",+ "forms": [],+ "navigation_targets": [],+ "result_containers": [{"tag": "table", "selector": "#results"}],+ "challenge_controls": [],+ },+ "reached_via": "interaction",+ "had_bounded_schema": True,+ "step": 4,+ }+ )++ async def unexpected_navigate(*_: object, **__: object) -> dict[str, object]:+ raise AssertionError("target_url inspection should not navigate away from reached evidence")++ monkeypatch.setattr("skyvern.forge.sdk.copilot.tools._discovery_navigate", unexpected_navigate)++ result = await _inspect_page_for_composition_impl(ctx, "https://example.test/")++ assert result["ok"] is False+ assert result["data"] == {+ "current_url": "https://example.test/search/results?s=1",+ "observation_step": 4,+ }+ assert 'target_url="current_page"' in result["error"]+ assert "observation_step 4" in result["error"]diff --git a/tests/unit/test_copilot_frontier.py b/tests/unit/test_copilot_frontier.pyindex 4dbf8ec31..8ae578fca 100644--- a/tests/unit/test_copilot_frontier.py+++ b/tests/unit/test_copilot_frontier.py@@ -543,6 +543,27 @@ def test_frontier_run_size_error_limits_long_page_changing_frontier() -> None:assert "Do not remove later blocks" in error+def test_frontier_run_size_result_steers_to_smaller_saved_frontier() -> None:+ result = tools._frontier_run_size_result(+ "frontier too long",+ ["open", "set_search", "submit_search", "expand_results", "extract"],+ ["open", "set_search", "submit_search", "expand_results", "extract"],+ )++ data = result["data"]+ assert result["ok"] is False+ assert data["workflow_run_skipped"] is True+ assert data["suggested_block_labels"] == ["open", "set_search"]+ assert data["deferred_block_labels"] == ["submit_search", "expand_results", "extract"]+ assert data["control_signal"] == {+ "kind": "intermediate_success",+ "user_facing_summary": data["user_facing_summary"],+ "next_tool": "run_blocks_and_collect_debug",+ "next_block_labels": ["open", "set_search"],+ "preserve_workflow_yaml": True,+ }++def test_frontier_run_size_error_allows_tool_expanded_runtime_anchor() -> None:definition = _FakeDefinition([diff --git a/tests/unit/test_copilot_hooks.py b/tests/unit/test_copilot_hooks.pyindex 4d52e8400..930907a5d 100644--- a/tests/unit/test_copilot_hooks.py+++ b/tests/unit/test_copilot_hooks.py@@ -714,3 +714,69 @@ class TestVerifyScoutTypeLanded:assert result is Nonectx.discovery_mcp_server.call_internal_tool.assert_not_awaited()+++class TestBrowserInteractionObservationHooks:+ @pytest.mark.asyncio+ async def test_click_hook_marks_pending_interaction_observation(self) -> None:+ from skyvern.forge.sdk.copilot.tools import _click_post_hook++ ctx = SimpleNamespace(pending_browser_interaction_observation=None, discovery_mcp_server=None)+ result = await _click_post_hook(+ {"ok": True, "data": {"selector": "#add-to-cart"}},+ {"browser_context": {"url": "https://example.com/results", "title": "Results"}},+ ctx,+ )++ assert result["data"] == {+ "selector": "#add-to-cart",+ "url": "https://example.com/results",+ "title": "Results",+ }+ assert ctx.pending_browser_interaction_observation is not None+ assert ctx.pending_browser_interaction_observation.tool_name == "click"+ assert ctx.pending_browser_interaction_observation.url == "https://example.com/results"++ @pytest.mark.asyncio+ async def test_failed_click_hook_clears_stale_pending_interaction_observation(self) -> None:+ from skyvern.forge.sdk.copilot.runtime import PendingBrowserInteractionObservation+ from skyvern.forge.sdk.copilot.tools import _click_post_hook++ ctx = SimpleNamespace(+ pending_browser_interaction_observation=PendingBrowserInteractionObservation(+ tool_name="click",+ url="https://example.com/results",+ ),+ discovery_mcp_server=None,+ )++ result = await _click_post_hook(+ {"ok": False, "error": "element not found"},+ {"browser_context": {"url": "https://example.com/results", "title": "Results"}},+ ctx,+ )++ assert result == {"ok": False, "error": "element not found"}+ assert ctx.pending_browser_interaction_observation is None++ @pytest.mark.asyncio+ async def test_type_hook_does_not_mark_pending_interaction_when_readback_fails(+ self,+ monkeypatch: pytest.MonkeyPatch,+ ) -> None:+ from skyvern.forge.sdk.copilot import tools as tools_module++ async def fake_verify(*_args: object, **_kwargs: object) -> dict[str, object]:+ return {"ok": False, "error": "field is still empty"}++ monkeypatch.setattr(tools_module, "_verify_scout_type_landed", fake_verify)+ ctx = SimpleNamespace(pending_browser_interaction_observation=None, discovery_mcp_server=None)++ result = await tools_module._type_text_post_hook(+ {"ok": True, "data": {"selector": "#q", "text_length": 12}},+ {"browser_context": {"url": "https://example.com/search", "title": "Search"}},+ ctx,+ )++ assert result == {"ok": False, "error": "field is still empty"}+ assert ctx.pending_browser_interaction_observation is Nonediff --git a/tests/unit/test_copilot_output_policy.py b/tests/unit/test_copilot_output_policy.pyindex dd39c23d6..1d62a346a 100644--- a/tests/unit/test_copilot_output_policy.py+++ b/tests/unit/test_copilot_output_policy.py@@ -1204,6 +1204,48 @@ workflow_definition:assert sanitized_tool_names == ["update_and_run_blocks"]+async def test_update_workflow_runs_composition_evidence_precheck_before_saving(monkeypatch) -> None:+ from skyvern.forge.sdk.copilot import tools as tools_module++ workflow_yaml = """+workflow_definition:+ parameters: []+ blocks:+ - block_type: goto_url+ label: open_lookup+ url: https://example.com/lookup+ - block_type: navigation+ label: search_lookup+ navigation_goal: Enter the observed field and submit.+ - block_type: action+ label: expand_first_result+ navigation_goal: Expand the first result row.+"""+ ctx = _ctx(+ build_phase=BuildPhase.COMPOSING,+ turn_intent=TurnIntent(mode=TurnIntentMode.BUILD),+ request_policy=RequestPolicy(),+ )++ async def unexpected_update_workflow(*args: object, **kwargs: object) -> object:+ raise AssertionError("composition evidence precheck must run before update_workflow saves")++ monkeypatch.setattr(tools_module, "_tool_loop_error", lambda *args, **kwargs: None)+ monkeypatch.setattr(tools_module, "_update_workflow", unexpected_update_workflow)+ monkeypatch.setattr(tools_module, "_record_diagnosis_repair_contract", lambda *args, **kwargs: None)++ result = await tools_module.update_workflow_tool.on_invoke_tool(+ SimpleNamespace(context=ctx, tool_name="update_workflow"),+ json.dumps({"workflow_yaml": workflow_yaml}),+ )++ payload = json.loads(result)+ assert payload["ok"] is False+ assert "page-dependent build blocks need observed page evidence" in payload["error"]+ assert "https://example.com/lookup" in payload["error"]++@pytest.mark.asyncioasync def test_update_and_run_blocks_precheck_uses_proposed_block_observation_refs(monkeypatch) -> None:from skyvern.forge.sdk.copilot import tools as tools_modulediff --git a/tests/unit/test_copilot_session_injection.py b/tests/unit/test_copilot_session_injection.pyindex 80fdb4156..f2e52e8b1 100644--- a/tests/unit/test_copilot_session_injection.py+++ b/tests/unit/test_copilot_session_injection.py@@ -3,6 +3,7 @@from __future__ import annotationsimport asyncio+import timefrom typing import Anyfrom unittest.mock import AsyncMock, MagicMock@@ -12,6 +13,7 @@ from skyvern.cli.core import session_managerfrom skyvern.cli.core.result import BrowserContext as MCPBrowserContextfrom skyvern.cli.core.session_manager import SessionState, scoped_sessionfrom skyvern.forge.sdk.copilot.runtime import AgentContext, mcp_to_copilot+from skyvern.forge.sdk.copilot.tools import _same_page_ignoring_fragment@pytest.fixture(autouse=True)@@ -41,6 +43,12 @@ def _make_ctx(**overrides: Any) -> AgentContext:return AgentContext(**defaults)+def test_copilot_same_page_ignoring_fragment_matches_trailing_slash_variants() -> None:+ assert _same_page_ignoring_fragment("https://example.test/results#section", "https://example.test/results") is True+ assert _same_page_ignoring_fragment("https://example.test/results/", "https://example.test/results") is True+ assert _same_page_ignoring_fragment("https://example.test/results?page=2", "https://example.test/results") is False++@pytest.mark.asyncioasync def test_scoped_session_pushes_and_restores() -> None:"""scoped_session sets the ContextVar within scope and restores on exit."""@@ -278,6 +286,41 @@ class TestScreenshotAdapter:assert ctx.composition_page_evidence["source_tool"] == "get_browser_screenshot"assert ctx.composition_page_evidence["current_url"] == "https://example.com"+ @pytest.mark.asyncio+ async def test_screenshot_post_hook_does_not_verify_from_url_title_only(+ self,+ monkeypatch: pytest.MonkeyPatch,+ ) -> None:+ from skyvern.forge.sdk.copilot.request_policy import CompletionCriterion, RequestPolicy+ from skyvern.forge.sdk.copilot.tools import _screenshot_post_hook++ async def handler(**_: object) -> dict[str, object]:+ raise AssertionError("screenshot-only observation must not invoke completion verification")++ async def handler_lookup(_: object) -> object:+ return handler++ monkeypatch.setattr("skyvern.forge.sdk.copilot.tools._completion_verification_handler", handler_lookup)+ ctx = _make_ctx(+ request_policy=RequestPolicy(+ completion_criteria=[CompletionCriterion(id="c0", outcome="the requested item is visible")]+ ),+ last_test_ok=False,+ last_run_blocks_workflow_run_id="wr_failed",+ copilot_run_start_monotonic=time.monotonic(),+ )+ raw = {"browser_context": {"url": "https://example.com/results", "title": "Results"}}+ result = {+ "ok": True,+ "data": {"data": "iVBOR...", "mime": "image/png", "bytes": 1234},+ }++ adapted = await _screenshot_post_hook(result, raw, ctx)++ assert adapted["data"]["url"] == "https://example.com/results"+ assert ctx.post_run_page_observation_after_failed_test is True+ assert ctx.completion_verification_result is None+class TestNavigateAdapter:@pytest.mark.asyncio@@ -369,7 +412,15 @@ class TestEvaluateAdapter:assert adapted["data"]["url"] == "https://example.com/results"assert ctx.composition_page_evidence["source_tool"] == "evaluate"assert ctx.composition_page_evidence["current_url"] == "https://example.com/results"- assert ctx.composition_page_evidence["result_containers"] == []+ assert ctx.composition_page_evidence["result_containers"] == [+ {+ "tag": "table",+ "id": "",+ "selector": "",+ "row_count": 1,+ "sample_rows": ["Test User"],+ }+ ]@pytest.mark.asyncioasync def test_evaluate_post_hook_lifts_bounded_page_schema_from_mcp_observation(self) -> None:diff --git a/tests/unit/test_copilot_turn_intent.py b/tests/unit/test_copilot_turn_intent.pyindex 1ae017280..a018cd6f1 100644--- a/tests/unit/test_copilot_turn_intent.py+++ b/tests/unit/test_copilot_turn_intent.py@@ -321,6 +321,20 @@ def test_build_turn_intent_marks_run_context_for_diagnose() -> None:assert RequiredContextKey.LATEST_RUN_RESULT in intent.required_context+def test_build_turn_intent_does_not_require_unassembled_browser_state() -> None:+ intent = build_turn_intent(+ user_message="Continue in the open browser",+ workflow_yaml="workflow_definition:\n blocks: []",+ chat_history=[],+ global_llm_context="",+ request_policy=RequestPolicy(),+ browser_session_id="pbs_123",+ )++ assert RequiredContextKey.BROWSER_STATE not in intent.required_context+ assert TurnIntentReasonCode.BROWSER_CONTEXT_PRESENT in intent.reason_codes++def test_build_turn_intent_keeps_explicit_fix_as_edit() -> None:intent = build_turn_intent(user_message="Fix the error after login.",diff --git a/tests/unit/test_real_browser_state_navigate.py b/tests/unit/test_real_browser_state_navigate.pyindex 965222962..ff4cf1d4a 100644--- a/tests/unit/test_real_browser_state_navigate.py+++ b/tests/unit/test_real_browser_state_navigate.py@@ -10,7 +10,7 @@ import pytestfrom playwright.async_api import TimeoutError as PlaywrightTimeoutErrorfrom skyvern.exceptions import FailedToNavigateToUrl-from skyvern.webeye.real_browser_state import RealBrowserState+from skyvern.webeye.real_browser_state import RealBrowserState, _same_page_ignoring_fragment@pytest.fixture@@ -20,6 +20,12 @@ def browser_state() -> RealBrowserState:return state+def test_same_page_ignoring_fragment_matches_fragment_only_differences() -> None:+ assert _same_page_ignoring_fragment("https://example.test/results#section", "https://example.test/results") is True+ assert _same_page_ignoring_fragment("https://example.test/results/", "https://example.test/results") is True+ assert _same_page_ignoring_fragment("https://example.test/results?page=2", "https://example.test/results") is False++@pytest.mark.asyncioasync def test_navigate_to_url_progresses_from_load_to_domcontentloaded(browser_state: RealBrowserState,