Skip to content

Automate browser based workflows with AI

SKY-10562 Ground Copilot block authoring in observations (#6363)

Andrew Neilsonabf5e7660e24
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.py
index e2acf7d44..64e9a1970 100644
--- a/skyvern/forge/sdk/copilot/agent.py
+++ b/skyvern/forge/sdk/copilot/agent.py
@@ -51,6 +51,7 @@ from skyvern.forge.sdk.copilot.context import (
TurnNarrativePayload,
finalize_discovery_counter_in_global_llm_context,
)
+from skyvern.forge.sdk.copilot.enforcement import outcome_fully_verified
from skyvern.forge.sdk.copilot.outcome_verification_trace import (
finalize_outcome_verification_trace,
record_gate_decision,
@@ -685,31 +686,40 @@ def _partial_verification_response(ctx: CopilotContext) -> str | None:
def _rewrite_failed_test_response(user_response: str, ctx: CopilotContext) -> str:
has_keepable_draft = ctx.last_workflow is not None and bool(ctx.last_workflow_yaml)
keep_draft_affordance = " Keep the draft to iterate on, or discard." if has_keepable_draft else ""
+ block_count = ctx.last_update_block_count if isinstance(ctx.last_update_block_count, int) else None
+ positive_block_count = block_count if block_count is not None and block_count > 0 else None
+
+ if outcome_fully_verified(ctx) and has_keepable_draft and positive_block_count is not None:
+ block_word = "block" if positive_block_count == 1 else "blocks"
+ return (
+ f"I created a workflow with {positive_block_count} {block_word} and verified the requested "
+ "outcome from the current browser page after the run. The workflow is ready to review."
+ )
policy = ctx.request_policy if isinstance(ctx.request_policy, RequestPolicy) else None
if (
policy is not None
and policy.clarification_reason == "workflow_credential_inputs_unbound"
and ctx.last_workflow is not None
- and ctx.last_update_block_count is not None
+ and block_count is not None
):
- if ctx.last_update_block_count <= 0:
+ if positive_block_count is None:
draft_phrase = "a draft workflow"
else:
- block_word = "block" if ctx.last_update_block_count == 1 else "blocks"
- draft_phrase = f"a draft workflow with {ctx.last_update_block_count} {block_word}"
+ block_word = "block" if positive_block_count == 1 else "blocks"
+ draft_phrase = f"a draft workflow with {positive_block_count} {block_word}"
return (
f"I applied your requested change as {draft_phrase}. "
f"I couldn't test the modified workflow because I couldn't find the required credentials — "
f"please add them via the Credentials UI, then I can try again.{keep_draft_affordance}"
)
- if ctx.last_test_ok is False and ctx.last_update_block_count is not None:
- if ctx.last_update_block_count <= 0:
+ if ctx.last_test_ok is False and block_count is not None:
+ if positive_block_count is None:
draft_phrase = "a draft workflow"
else:
- block_word = "block" if ctx.last_update_block_count == 1 else "blocks"
- draft_phrase = f"a draft workflow with {ctx.last_update_block_count} {block_word}"
+ block_word = "block" if positive_block_count == 1 else "blocks"
+ draft_phrase = f"a draft workflow with {positive_block_count} {block_word}"
failure_summary = _normalize_failure_reason(ctx.last_test_failure_reason)
follow_up = _FAILURE_FOLLOW_UP.get(ctx.last_failure_category_top or "", "")
@@ -723,7 +733,7 @@ def _rewrite_failed_test_response(user_response: str, ctx: CopilotContext) -> st
if partial_reply is not None:
return partial_reply
- if ctx.last_test_ok is None and ctx.last_update_block_count is not None and ctx.last_workflow is not None:
+ if ctx.last_test_ok is None and block_count is not None and ctx.last_workflow is not None:
if policy is not None and policy.raw_secret_handling == "redacted_draft":
return (
"I drafted the workflow with the pasted secret redacted. "
@@ -770,8 +780,6 @@ def _completion_contract_not_violated(ctx: CopilotContext) -> bool:
def _verified_workflow_or_none(ctx: CopilotContext) -> tuple[Any, str | None]:
"""Surface a proposal when it passed a test this turn, or when the outcome judge
confirmed the goal from evidence even though the run did not finish cleanly."""
- from skyvern.forge.sdk.copilot.enforcement import outcome_fully_verified
-
run_status_clean = ctx.last_test_ok is True and ctx.last_full_workflow_test_ok is True
if (
ctx.last_workflow is not None
diff --git a/skyvern/forge/sdk/copilot/completion_verification.py b/skyvern/forge/sdk/copilot/completion_verification.py
index 2eb05121a..bc31dd915 100644
--- a/skyvern/forge/sdk/copilot/completion_verification.py
+++ b/skyvern/forge/sdk/copilot/completion_verification.py
@@ -70,6 +70,7 @@ class RunEvidenceSnapshot:
current_url: str | None = None
page_title: str | None = None
executed_block_labels: list[str] = field(default_factory=list)
+ verified_context_block_labels: list[str] = field(default_factory=list)
def has_evidence(self) -> bool:
return bool(self.block_outputs or self.current_url or self.page_title)
@@ -82,6 +83,10 @@ class RunEvidenceSnapshot:
lines.append(f"observed_end_state_url: {self.current_url}")
if self.page_title:
lines.append(f"observed_end_state_page_title: {self.page_title}")
+ if self.verified_context_block_labels:
+ lines.append(
+ "verified_context_block_labels: " + ", ".join(self.verified_context_block_labels[:_MAX_BLOCK_OUTPUTS])
+ )
if self.executed_block_labels:
lines.append("executed_block_labels: " + ", ".join(self.executed_block_labels[:_MAX_BLOCK_OUTPUTS]))
if self.block_outputs:
diff --git a/skyvern/forge/sdk/copilot/composition_evidence.py b/skyvern/forge/sdk/copilot/composition_evidence.py
index 997a5b27d..58a0f991a 100644
--- a/skyvern/forge/sdk/copilot/composition_evidence.py
+++ b/skyvern/forge/sdk/copilot/composition_evidence.py
@@ -43,6 +43,24 @@ _MAX_RESULT_CONTAINERS = 8
_MAX_NAVIGATION_TARGETS = 20
_MAX_SELECT_OPTIONS = 30
_MAX_CHALLENGE_CONTROLS = 8
+_MAX_MODAL_OVERLAYS = 5
+_MAX_MODAL_DISMISS_CONTROLS = 6
+_MAX_PAGE_OBSTRUCTIONS = 5
+_MAX_VISIBLE_CONTROLS = 6
+_MODAL_IDENTITY_PATTERNS: frozenset[str] = frozenset({"modal", "popup", "overlay", "dialog", "drawer", "lightbox"})
+_MODAL_ROLE_VALUES: frozenset[str] = frozenset({"dialog", "alertdialog"})
+_MODAL_DISMISS_HINTS: frozenset[str] = frozenset(
+ {
+ "cancel",
+ "close",
+ "dismiss",
+ "got it",
+ "no thanks",
+ "not now",
+ "skip",
+ }
+)
+_MODAL_DISMISS_SYMBOLS: frozenset[str] = frozenset({"x", "\u00d7"})
DOM_EVIDENCE_SOURCE = "dom_html"
SCREENSHOT_EVIDENCE_SOURCE = "screenshot"
VISION_EVIDENCE_SOURCE = "vision_summary"
@@ -50,6 +68,8 @@ _ANTI_BOT_PATTERNS = (
"just a moment",
"captcha",
"challenge",
+ "turnstile",
+ "cf-turnstile",
"human-verification",
"human verification",
"verify you are human",
@@ -157,6 +177,37 @@ def _evidence_metadata(
}
+def _bounded_visual_controls(values: Iterable[Any]) -> list[dict[str, Any]]:
+ controls: list[dict[str, Any]] = []
+ for value in values:
+ if len(controls) >= _MAX_VISIBLE_CONTROLS:
+ break
+ text = _bounded_string(value, 120)
+ if text:
+ controls.append({"text": text})
+ return controls
+
+
+def _page_obstructions_from_visual_summary(visual_summary: dict[str, Any]) -> list[dict[str, Any]]:
+ if visual_summary.get("page_obstruction_detected") is not True:
+ return []
+ obstruction: dict[str, Any] = {
+ "kind": _bounded_string(visual_summary.get("obstruction_kind"), 80) or "visual_obstruction",
+ "source": VISION_EVIDENCE_SOURCE,
+ "visual_location": _bounded_string(visual_summary.get("obstruction_location"), 180),
+ "visible_controls": _bounded_visual_controls(visual_summary.get("visible_dismiss_controls") or []),
+ }
+ if isinstance(visual_summary.get("underlying_page_blocked"), bool):
+ obstruction["underlying_page_blocked"] = visual_summary["underlying_page_blocked"]
+ return [
+ {
+ key: value
+ for key, value in obstruction.items()
+ if value or key in {"visible_controls", "underlying_page_blocked"}
+ }
+ ]
+
+
def page_evidence_needs_visual_fallback(evidence: dict[str, Any]) -> bool:
"""Return True when DOM evidence should be augmented with visual evidence."""
@@ -232,6 +283,10 @@ def merge_visual_composition_evidence(
]
challenge_state["gated_submit_controls"] = (existing_controls + visual_blocked_controls)[:5]
merged["challenge_state"] = challenge_state
+ visual_obstructions = _page_obstructions_from_visual_summary(visual_summary)
+ if visual_obstructions:
+ existing_obstructions = [item for item in merged.get("page_obstructions") or [] if isinstance(item, dict)]
+ merged["page_obstructions"] = (existing_obstructions + visual_obstructions)[:_MAX_PAGE_OBSTRUCTIONS]
# Empty-page classification is visual-only: DOM parsing can tell that a page
# is schema-empty, but only the screenshot summary distinguishes settled empty
# pages from loading shells.
@@ -495,6 +550,22 @@ def has_bounded_page_schema(evidence: dict[str, Any]) -> bool:
value = evidence.get(key)
if isinstance(value, list) and value:
return True
+ modal_overlays = evidence.get("modal_overlays")
+ if isinstance(modal_overlays, list):
+ for overlay in modal_overlays:
+ if not isinstance(overlay, dict):
+ continue
+ dismiss_controls = overlay.get("dismiss_controls")
+ if isinstance(dismiss_controls, list) and dismiss_controls:
+ return True
+ page_obstructions = evidence.get("page_obstructions")
+ if isinstance(page_obstructions, list):
+ for obstruction in page_obstructions:
+ if not isinstance(obstruction, dict):
+ continue
+ visible_controls = obstruction.get("visible_controls")
+ if isinstance(visible_controls, list) and visible_controls:
+ return True
challenge_state = evidence.get("challenge_state")
if isinstance(challenge_state, dict) and challenge_state.get("detected") is True:
return True
@@ -691,6 +762,27 @@ def _associated_observation_satisfies_block(
return _evidence_matches_target(evidence, observed_url, allow_post_run_browser_observation=allow_post_run)
+def _current_page_evidence_has_reached_page_credit(
+ evidence: dict[str, Any],
+ reached_via: str,
+ *,
+ flow_evidence_by_step: dict[int, tuple[dict[str, Any], str]],
+) -> bool:
+ if reached_via != "current_page" or not has_bounded_page_schema(evidence):
+ return False
+ observed_url = _evidence_observed_url(evidence)
+ if not observed_url:
+ return False
+ for prior_evidence, prior_reached_via in flow_evidence_by_step.values():
+ if prior_reached_via not in {"interaction", "post_run"}:
+ continue
+ if not has_bounded_page_schema(prior_evidence):
+ continue
+ if _same_page(observed_url, _evidence_observed_url(prior_evidence)):
+ return True
+ return False
+
+
def _block_has_observed_page(
ctx: Any,
block: dict[str, Any],
@@ -705,11 +797,20 @@ def _block_has_observed_page(
if evidence_entry is None:
return False
evidence, reached_via = evidence_entry
+ effective_reached_via = (
+ "interaction"
+ if _current_page_evidence_has_reached_page_credit(
+ evidence,
+ reached_via,
+ flow_evidence_by_step=flow_evidence_by_step,
+ )
+ else reached_via
+ )
return _associated_observation_satisfies_block(
evidence,
block.get("target_url"),
acts_via=str(block.get("acts_via") or ""),
- reached_via=reached_via,
+ reached_via=effective_reached_via,
requires_observation_ref=block.get("requires_observation_ref") is True,
allow_post_run=allow_post_run,
)
@@ -772,9 +873,15 @@ def _wrong_reached_via_observation_ref(
evidence_entry = flow_evidence_by_step.get(step)
if evidence_entry is None:
return None
- _, reached_via = evidence_entry
+ evidence, reached_via = evidence_entry
if reached_via in {"interaction", "post_run"}:
return None
+ if _current_page_evidence_has_reached_page_credit(
+ evidence,
+ reached_via,
+ flow_evidence_by_step=flow_evidence_by_step,
+ ):
+ return None
return step, reached_via or "<missing>"
@@ -840,7 +947,7 @@ def composition_page_evidence_error(
return (
"Workflow validation failed: a block_observation_refs entry uses observation_step "
f"{string_step!r} as a string. Pass the integer observation_step returned by "
- "inspect_page_for_composition for click-reached blocks. "
+ "inspect_page_for_composition or evaluate for click-reached blocks. "
f"Offending blocks: {_format_page_block_findings([block])}"
)
if _required_observation_ref_missing(block, block_observation_refs):
@@ -868,8 +975,8 @@ def composition_page_evidence_error(
return (
"Workflow validation failed: a block references observation_step "
f"{missing_step}, but {missing_reason}. "
- "Call inspect_page_for_composition again for the reached page and pass the new "
- "observation_step in block_observation_refs before composing page-dependent blocks. "
+ "Inspect or evaluate the reached page again and pass the new observation_step in "
+ "block_observation_refs before composing page-dependent blocks. "
f"Offending blocks: {_format_page_block_findings([block])}"
)
return (
@@ -896,6 +1003,8 @@ def _empty_evidence(inspected_url: str, current_url: str) -> dict[str, Any]:
"result_containers": [],
"anti_bot_indicators": [],
"challenge_controls": [],
+ "modal_overlays": [],
+ "page_obstructions": [],
"schema_empty_page": False,
"observed_empty_page": False,
"empty_page_visual_state": None,
@@ -1122,6 +1231,145 @@ def _challenge_controls(soup: Any) -> list[dict[str, Any]]:
return controls
+def _modal_identity(node: Any) -> str:
+ values = (
+ getattr(node, "name", ""),
+ _attr_value(node, "id"),
+ " ".join(_classes_for(node)),
+ _attr_value(node, "role"),
+ _attr_value(node, "aria-label"),
+ _attr_value(node, "title"),
+ _attr_value(node, "data-testid"),
+ _attr_value(node, "data-test"),
+ _attr_value(node, "data-dismiss"),
+ )
+ return " ".join(str(value or "") for value in values).lower()
+
+
+def _is_modal_overlay_candidate(node: Any) -> bool:
+ role = _attr_value(node, "role").strip().lower()
+ if role in _MODAL_ROLE_VALUES:
+ return True
+ if _attr_value(node, "aria-modal").strip().lower() == "true":
+ return True
+ return any(pattern in _modal_identity(node) for pattern in _MODAL_IDENTITY_PATTERNS)
+
+
+def _is_hidden_modal_candidate(node: Any) -> bool:
+ current = node
+ while current is not None:
+ if _attr_value(current, "aria-hidden").strip().lower() == "true":
+ return True
+ if hasattr(current, "has_attr") and current.has_attr("hidden"):
+ return True
+ style = _attr_value(current, "style").replace(" ", "").lower()
+ if "display:none" in style or "visibility:hidden" in style:
+ return True
+ current = getattr(current, "parent", None)
+ return False
+
+
+def _modal_dismiss_controls(node: Any) -> list[dict[str, Any]]:
+ controls: list[dict[str, Any]] = []
+ seen_selectors: set[str] = set()
+ for control in node.find_all(["button", "a", "input"]):
+ if len(controls) >= _MAX_MODAL_DISMISS_CONTROLS:
+ break
+ selector = _selector_for(control)[:160]
+ if selector in seen_selectors:
+ continue
+ text = _schema_text(_node_text(control) or _attr_value(control, "value"), 120)
+ aria_label = _schema_text(_attr_value(control, "aria-label"), 120)
+ title = _schema_text(_attr_value(control, "title"), 120)
+ explicit_values = {text.strip().lower(), aria_label.strip().lower(), title.strip().lower()}
+ identity = f"{text} {aria_label} {title} {_modal_identity(control)}".lower()
+ # Controls are typed as Any because BeautifulSoup is optional at import time.
+ has_data_dismiss = hasattr(control, "has_attr") and control.has_attr("data-dismiss")
+ has_symbol_hint = bool(explicit_values & _MODAL_DISMISS_SYMBOLS)
+ has_text_hint = any(hint in identity for hint in _MODAL_DISMISS_HINTS)
+ if not (has_data_dismiss or has_symbol_hint or has_text_hint):
+ continue
+ seen_selectors.add(selector)
+ controls.append(
+ {
+ "tag": str(getattr(control, "name", "") or "").lower()[:40],
+ "text": text,
+ "aria_label": aria_label,
+ "title": title,
+ "selector": selector,
+ "type": _attr_value(control, "type")[:40],
+ }
+ )
+ return [{key: value for key, value in entry.items() if value} for entry in controls]
+
+
+def _modal_overlay_entry(node: Any) -> dict[str, Any]:
+ return {
+ "role": _attr_value(node, "role")[:80],
+ "aria_modal": _attr_value(node, "aria-modal").strip().lower() == "true",
+ "id": _attr_value(node, "id")[:120],
+ "class": " ".join(_classes_for(node)[:5])[:160],
+ "selector": _selector_for(node)[:160],
+ "text": _schema_text(_node_text(node), 240),
+ "dismiss_controls": _modal_dismiss_controls(node),
+ }
+
+
+def _modal_overlays(nodes: Iterable[Any]) -> list[dict[str, Any]]:
+ overlays: list[dict[str, Any]] = []
+ seen_selectors: set[str] = set()
+ for node in nodes:
+ if len(overlays) >= _MAX_MODAL_OVERLAYS:
+ break
+ if not _is_modal_overlay_candidate(node):
+ continue
+ if _is_hidden_modal_candidate(node):
+ continue
+ selector = _selector_for(node)[:160]
+ if selector in seen_selectors:
+ continue
+ entry = _modal_overlay_entry(node)
+ dismiss_controls = entry.get("dismiss_controls")
+ has_dismiss_controls = isinstance(dismiss_controls, list) and bool(dismiss_controls)
+ has_explicit_modal_semantics = bool(entry.get("role") or entry.get("aria_modal") is True)
+ if not (has_explicit_modal_semantics or has_dismiss_controls):
+ continue
+ seen_selectors.add(selector)
+ # Omit aria_modal=False from compact evidence: only affirmative modal
+ # semantics matter for obstruction handoff.
+ compact_entry = {key: value for key, value in entry.items() if value or key == "dismiss_controls"}
+ overlays.append(compact_entry)
+ return overlays
+
+
+def _page_obstructions_from_modal_overlays(modal_overlays: list[dict[str, Any]]) -> list[dict[str, Any]]:
+ obstructions: list[dict[str, Any]] = []
+ for overlay in modal_overlays[:_MAX_PAGE_OBSTRUCTIONS]:
+ if not isinstance(overlay, dict):
+ continue
+ visible_controls: list[dict[str, Any]] = []
+ for control in overlay.get("dismiss_controls") or []:
+ if len(visible_controls) >= _MAX_VISIBLE_CONTROLS:
+ break
+ if not isinstance(control, dict):
+ continue
+ text = _bounded_string(control.get("text") or control.get("aria_label") or control.get("title"), 120)
+ selector = _bounded_string(control.get("selector"), 160)
+ if text or selector:
+ visible_controls.append(
+ {key: value for key, value in {"text": text, "selector": selector}.items() if value}
+ )
+ entry = {
+ "kind": "modal_overlay",
+ "source": DOM_EVIDENCE_SOURCE,
+ "selector": _bounded_string(overlay.get("selector"), 160),
+ "text": _bounded_string(overlay.get("text"), 240),
+ "visible_controls": visible_controls,
+ }
+ obstructions.append({key: value for key, value in entry.items() if value or key == "visible_controls"})
+ return obstructions
+
+
def _anti_bot_indicators(html: str, page_title: str) -> list[str]:
haystack = f"{page_title}\n{html[:_ANTI_BOT_SCAN_BYTES]}".lower()
return [pattern for pattern in _ANTI_BOT_PATTERNS if pattern in haystack]
@@ -1144,6 +1392,8 @@ def parse_composition_html(html: str, *, inspected_url: str, current_url: str) -
node.decompose()
all_nodes = soup.find_all(True)
+ modal_overlays = _modal_overlays(all_nodes)
+ page_obstructions = _page_obstructions_from_modal_overlays(modal_overlays)
forms: list[dict[str, Any]] = []
for form in soup.find_all("form")[:_MAX_FORMS]:
@@ -1235,7 +1485,12 @@ def parse_composition_html(html: str, *, inspected_url: str, current_url: str) -
body_text = _node_text(soup.body if soup.body is not None else soup).strip()
# Schema-empty means the page had content, but no bounded form/link/result/challenge structure.
schema_empty_page = bool((html or "").strip() or page_title or body_text) and not (
- forms or navigation_targets or result_containers or challenge_controls or anti_bot_indicators
+ forms
+ or navigation_targets
+ or result_containers
+ or challenge_controls
+ or page_obstructions
+ or anti_bot_indicators
)
# Higher confidence means the parser saw a more complete form surface.
confidence = 0.85 if field_count and control_count else 0.6 if field_count else 0.3 if forms else 0.1
@@ -1248,6 +1503,8 @@ def parse_composition_html(html: str, *, inspected_url: str, current_url: str) -
"result_containers": result_containers,
"anti_bot_indicators": anti_bot_indicators,
"challenge_controls": challenge_controls,
+ "modal_overlays": modal_overlays,
+ "page_obstructions": page_obstructions,
"schema_empty_page": schema_empty_page,
"observed_empty_page": False,
"empty_page_visual_state": None,
diff --git a/skyvern/forge/sdk/copilot/runtime.py b/skyvern/forge/sdk/copilot/runtime.py
index ff0feac62..1b68ab171 100644
--- a/skyvern/forge/sdk/copilot/runtime.py
+++ b/skyvern/forge/sdk/copilot/runtime.py
@@ -95,6 +95,12 @@ async def _get_persistent_browser_session(session_id: str, organization_id: str)
return cast("PersistentBrowserSession | None", persistent_result)
+@dataclass
+class PendingBrowserInteractionObservation:
+ tool_name: str
+ url: str = ""
+
+
@dataclass
class AgentContext:
organization_id: str
@@ -178,6 +184,7 @@ class AgentContext:
# while scouting the goal path, each tagged with how that state was reached.
# Feeds the per-acted-page composition gate; never persisted into workflow YAML.
flow_evidence: list[dict[str, Any]] = field(default_factory=list)
+ pending_browser_interaction_observation: PendingBrowserInteractionObservation | None = None
# In-turn side channel from workflow mutation calls: block label -> flow_evidence
# observation step used to ground the newly authored page-acting block.
block_observation_refs: dict[str, int] = field(default_factory=dict)
diff --git a/skyvern/forge/sdk/copilot/tools.py b/skyvern/forge/sdk/copilot/tools.py
index 0ef835816..02660250b 100644
--- a/skyvern/forge/sdk/copilot/tools.py
+++ b/skyvern/forge/sdk/copilot/tools.py
@@ -106,7 +106,11 @@ from skyvern.forge.sdk.copilot.output_utils import (
truncate_output,
)
from skyvern.forge.sdk.copilot.request_policy import CREDENTIAL_DEFERRED_DRAFT_REASONS, RequestPolicy
-from skyvern.forge.sdk.copilot.runtime import AgentContext, ensure_browser_session
+from skyvern.forge.sdk.copilot.runtime import (
+ AgentContext,
+ PendingBrowserInteractionObservation,
+ ensure_browser_session,
+)
from skyvern.forge.sdk.copilot.screenshot_utils import enqueue_screenshot_from_result
from skyvern.forge.sdk.copilot.streaming_adapter import emit_workflow_draft, maybe_emit_design_end
from skyvern.forge.sdk.copilot.tracing_setup import copilot_span
@@ -1360,6 +1364,57 @@ def _mark_page_inspected(ctx: AgentContext) -> None:
ctx.post_budget_page_inspection_run_id = None
+def _same_page_ignoring_fragment(left: str | None, right: str | None) -> bool:
+ if not left or not right:
+ return False
+ try:
+ left_parsed = urlparse(left)
+ right_parsed = urlparse(right)
+ except Exception as exc:
+ LOG.debug("copilot_same_page_url_parse_failed", left=left, right=right, error=str(exc))
+ return False
+ left_url = left_parsed._replace(fragment="").geturl().rstrip("/")
+ right_url = right_parsed._replace(fragment="").geturl().rstrip("/")
+ return left_url == right_url
+
+
+def _clear_pending_browser_interaction_observation(ctx: AgentContext) -> None:
+ ctx.pending_browser_interaction_observation = None
+
+
+def _mark_pending_browser_interaction_observation(ctx: AgentContext, *, tool_name: str, url: str) -> None:
+ if not url.strip():
+ _clear_pending_browser_interaction_observation(ctx)
+ return
+ ctx.pending_browser_interaction_observation = PendingBrowserInteractionObservation(
+ tool_name=tool_name,
+ url=url.strip(),
+ )
+
+
+def _consume_pending_browser_interaction_observation(
+ ctx: AgentContext,
+ *,
+ current_url: str,
+ evidence: dict[str, Any],
+) -> bool:
+ pending = ctx.pending_browser_interaction_observation
+ if pending is None:
+ return False
+ _clear_pending_browser_interaction_observation(ctx)
+ if not has_bounded_page_schema(evidence):
+ return False
+ if not _same_page_ignoring_fragment(pending.url, current_url):
+ LOG.warning(
+ "copilot_pending_browser_interaction_observation_page_mismatch",
+ tool_name=pending.tool_name,
+ pending_url=pending.url,
+ current_url=current_url,
+ )
+ return False
+ return True
+
+
def _mark_post_run_page_observed(ctx: AgentContext, *, source_tool: str, url: str) -> None:
run_id = getattr(ctx, "last_run_blocks_workflow_run_id", None)
if not isinstance(run_id, str) or not run_id:
@@ -4262,6 +4317,47 @@ def _normalize_observed_forms(observed_data: dict[str, Any]) -> list[dict[str, A
return [{"id": "", "name": "", "action": "", "method": "", "fields": fields, "submit_controls": []}]
+def _evaluate_observed_payload(observed_data: object) -> dict[str, Any]:
+ if not isinstance(observed_data, dict):
+ return {}
+ raw_result = observed_data.get("result")
+ if isinstance(raw_result, dict):
+ payload: dict[str, Any] = dict(raw_result)
+ elif isinstance(raw_result, list):
+ payload = {"rows": raw_result}
+ else:
+ payload = {}
+ for key, value in observed_data.items():
+ if key == "result":
+ continue
+ if key not in payload or payload.get(key) in (None, "", [], {}):
+ payload[key] = value
+ return payload or observed_data
+
+
+def _observed_row_text(row: object) -> str:
+ if isinstance(row, str):
+ return _bounded_observation_text(row, 500)
+ if not isinstance(row, dict):
+ return ""
+ parts: list[str] = []
+ for key in ("text", "label", "name", "title"):
+ value = row.get(key)
+ if isinstance(value, str):
+ parts.append(value)
+ cells = row.get("cells")
+ if isinstance(cells, list):
+ for cell in cells[:12]:
+ if isinstance(cell, dict):
+ for key in ("text", "value"):
+ value = cell.get(key)
+ if isinstance(value, str):
+ parts.append(value)
+ elif isinstance(cell, str):
+ parts.append(cell)
+ return _bounded_observation_text(" ".join(parts), 500)
+
+
def _normalize_observed_result_containers(observed_data: dict[str, Any]) -> list[dict[str, Any]]:
raw_results = observed_data.get("result_containers") or observed_data.get("tables")
containers: list[dict[str, Any]] = []
@@ -4277,14 +4373,74 @@ def _normalize_observed_result_containers(observed_data: dict[str, Any]) -> list
)
if containers:
return containers
+ raw_rows = observed_data.get("rows")
+ if isinstance(raw_rows, list) and raw_rows:
+ sample_rows = [text for row in raw_rows[:5] if (text := _observed_row_text(row))]
+ return [
+ {
+ "tag": "table",
+ "id": _bounded_observation_text(observed_data.get("id"), 120),
+ "selector": _bounded_observation_text(observed_data.get("selector"), 160),
+ "row_count": len(raw_rows),
+ "sample_rows": sample_rows,
+ }
+ ]
table = observed_data.get("table")
if table:
return [{"tag": "table", "id": "", "selector": ""}]
return []
+def _normalize_evaluate_challenge_state(
+ observed_data: dict[str, Any],
+ anti_bot_indicators: list[str],
+) -> dict[str, Any]:
+ detected = bool(anti_bot_indicators)
+ gated_controls: list[dict[str, Any]] = []
+ for key, value in observed_data.items():
+ normalized_key = str(key).strip().lower().replace("_", " ").replace("-", " ")
+ if "disabled" not in normalized_key or value is not True:
+ continue
+ if any(token in normalized_key for token in ("submit", "search", "button", "btn")):
+ gated_controls.append(
+ {
+ "text": _bounded_observation_text(str(key), 120),
+ "id": "",
+ "name": _bounded_observation_text(str(key), 120),
+ "selector": "",
+ "disabled": True,
+ }
+ )
+ indicator_text = " ".join(anti_bot_indicators).lower()
+ if "access denied" in indicator_text:
+ kind = "access_denied"
+ elif "turnstile" in indicator_text or "captcha" in indicator_text or "are you a robot" in indicator_text:
+ kind = "captcha"
+ elif detected:
+ kind = "human_verification"
+ else:
+ kind = "none"
+ return {
+ "detected": detected,
+ "kind": kind,
+ "source": "mcp_evaluate" if detected else "",
+ "indicators": anti_bot_indicators[:8],
+ "requires_human_verification": detected,
+ "visual_location": "",
+ "gates_submit_controls": bool(detected and gated_controls),
+ "gated_submit_controls": gated_controls[:5] if detected else [],
+ }
+
+
+def _evaluate_anti_bot_indicators(observed_data: dict[str, Any], text: str) -> list[str]:
+ key_text = " ".join(str(key) for key in observed_data.keys()).lower()
+ combined = f"{text} {key_text}"
+ return [pattern for pattern in _DISCOVERY_ANTI_BOT_PATTERNS if pattern in combined]
+
+
def _normalize_evaluate_page_schema(observed_data: object) -> dict[str, Any]:
- if not isinstance(observed_data, dict):
+ observed_data = _evaluate_observed_payload(observed_data)
+ if not observed_data:
return {}
text_parts = [
observed_data.get("body"),
@@ -4293,7 +4449,7 @@ def _normalize_evaluate_page_schema(observed_data: object) -> dict[str, Any]:
observed_data.get("html"),
]
text = " ".join(part for part in text_parts if isinstance(part, str)).lower()[:4096]
- anti_bot = [pattern for pattern in _DISCOVERY_ANTI_BOT_PATTERNS if pattern in text]
+ anti_bot = _evaluate_anti_bot_indicators(observed_data, text)
forms = _normalize_observed_forms(observed_data)
result_containers = _normalize_observed_result_containers(observed_data)
if not forms and not result_containers and not anti_bot:
@@ -4302,6 +4458,7 @@ def _normalize_evaluate_page_schema(observed_data: object) -> dict[str, Any]:
"forms": forms,
"result_containers": result_containers,
"anti_bot_indicators": anti_bot,
+ "challenge_state": _normalize_evaluate_challenge_state(observed_data, anti_bot),
"evidence_sources": ["mcp_evaluate"],
"evidence_confidence": (
_EVALUATE_EVIDENCE_CONFIDENCE_WITH_SCHEMA
@@ -4318,15 +4475,15 @@ def _record_composition_page_observation(
url: str,
title: str = "",
observed_data: object | None = None,
-) -> None:
+ append_to_flow: bool = False,
+ reached_via: str = "current_page",
+) -> int | None:
if not url:
- return
+ return None
_mark_post_run_page_observed(ctx, source_tool=source_tool, url=url)
if title:
_workflow_verification_evidence(ctx).page_title = title[:160]
existing = ctx.composition_page_evidence
- if isinstance(existing, dict) and existing.get("source_tool") == "inspect_page_for_composition":
- return
evidence: dict[str, Any] = {
"inspected_url": url,
@@ -4357,7 +4514,157 @@ def _record_composition_page_observation(
if isinstance(run_id, str) and run_id:
evidence["workflow_run_id"] = run_id
evidence["observed_after_workflow_run"] = True
- ctx.composition_page_evidence = evidence
+
+ observation_step: int | None = None
+ if append_to_flow and has_bounded_page_schema(evidence):
+ actual_reached_via = reached_via
+ if reached_via == "auto":
+ if isinstance(run_id, str) and run_id:
+ actual_reached_via = "post_run"
+ elif _consume_pending_browser_interaction_observation(ctx, current_url=url, evidence=evidence):
+ actual_reached_via = "interaction"
+ else:
+ actual_reached_via = "current_page"
+ observation_step = _append_flow_evidence(ctx, evidence, reached_via=actual_reached_via)
+
+ if not _should_keep_existing_composition_page_evidence(existing, evidence):
+ ctx.composition_page_evidence = evidence
+ return observation_step
+
+
+def _composition_evidence_page_url(evidence: dict[str, Any] | None) -> str | None:
+ if not isinstance(evidence, dict):
+ return None
+ for key in ("current_url", "inspected_url"):
+ value = evidence.get(key)
+ if isinstance(value, str) and value.strip() and value != "current_page":
+ return value.strip()
+ return None
+
+
+def _should_keep_existing_composition_page_evidence(
+ existing: dict[str, Any] | None,
+ incoming: dict[str, Any],
+) -> bool:
+ if not isinstance(existing, dict) or existing.get("source_tool") != "inspect_page_for_composition":
+ return False
+ if incoming.get("source_tool") != "evaluate":
+ return True
+ if not has_bounded_page_schema(incoming):
+ return True
+ if not has_bounded_page_schema(existing):
+ return False
+ existing_url = _composition_evidence_page_url(existing)
+ incoming_url = _composition_evidence_page_url(incoming)
+ if existing_url and incoming_url and not _same_page_ignoring_fragment(existing_url, incoming_url):
+ return False
+ return True
+
+
+def _completion_verification_criteria(copilot_ctx: Any) -> list[Any]:
+ policy = getattr(copilot_ctx, "request_policy", None)
+ # A method-mandated criterion asserts HOW the goal was reached; the outcome
+ # judge sees only end-state evidence.
+ return [c for c in (policy.completion_criteria if policy is not None else []) if not c.method_mandated]
+
+
+def _verified_context_block_labels_for_snapshot(
+ copilot_ctx: Any, current_labels: list[str], executed_labels: list[str]
+) -> list[str]:
+ current_label_set = set(current_labels)
+ executed_label_set = set(executed_labels)
+ candidates: set[str] = set()
+ for raw_values in (
+ getattr(copilot_ctx, "verified_prefix_labels", None),
+ getattr(getattr(copilot_ctx, "workflow_verification_evidence", None), "block_verified", None),
+ ):
+ if not isinstance(raw_values, list):
+ continue
+ candidates.update(str(label) for label in raw_values if isinstance(label, str) and label in current_label_set)
+ return [label for label in current_labels if label in candidates and label not in executed_label_set]
+
+
+def _build_page_observation_evidence_snapshot(
+ copilot_ctx: Any,
+ *,
+ url: str,
+ title: str = "",
+ observed_data: object | None = None,
+) -> RunEvidenceSnapshot:
+ run_id = getattr(copilot_ctx, "last_run_blocks_workflow_run_id", None)
+ block_outputs: dict[str, Any] = {}
+ if isinstance(observed_data, dict) and observed_data:
+ block_outputs["current_page_observation"] = observed_data
+ elif observed_data is not None:
+ block_outputs["current_page_observation"] = str(observed_data)
+ return RunEvidenceSnapshot(
+ workflow_run_id=run_id if isinstance(run_id, str) else None,
+ block_outputs=block_outputs,
+ current_url=_valid_runtime_anchor_url(url),
+ page_title=title if isinstance(title, str) and title.strip() else None,
+ )
+
+
+async def _maybe_run_completion_verification_from_page_observation(
+ copilot_ctx: Any,
+ *,
+ url: str,
+ title: str = "",
+ observed_data: object | None = None,
+) -> CompletionVerificationResult | None:
+ """Verify completion only for post-run page observations after failed tests."""
+
+ if not settings.COPILOT_OUTCOME_VERIFICATION_ENABLED:
+ return None
+ existing = getattr(copilot_ctx, "completion_verification_result", None)
+ if isinstance(existing, CompletionVerificationResult) and existing.is_fully_satisfied():
+ return existing
+ if getattr(copilot_ctx, "post_run_page_observation_after_failed_test", False) is not True:
+ return None
+ criteria = _completion_verification_criteria(copilot_ctx)
+ if not criteria:
+ return None
+ handler = await _completion_verification_handler(copilot_ctx)
+ if handler is None:
+ return None
+ remaining = _copilot_seconds_remaining(copilot_ctx)
+ if (
+ remaining is not None
+ and remaining
+ <= settings.COPILOT_FEASIBILITY_GATE_TIMEOUT_SECONDS + _COMPLETION_VERIFICATION_BUDGET_MARGIN_SECONDS
+ ):
+ verification = CompletionVerificationResult(status="unavailable")
+ else:
+ snapshot = _build_page_observation_evidence_snapshot(
+ copilot_ctx,
+ url=url,
+ title=title,
+ observed_data=observed_data,
+ )
+ if not snapshot.has_evidence():
+ verification = CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=[criterion.id for criterion in criteria],
+ verdicts=[
+ CriterionVerdict(criterion_id=criterion.id, satisfied=False, reason_code="no_evidence")
+ for criterion in criteria
+ ],
+ )
+ else:
+ verification = await evaluate_completion_criteria(criteria, snapshot, handler)
+
+ if (
+ isinstance(existing, CompletionVerificationResult)
+ and not verification.is_fully_satisfied()
+ and not (existing.status == "unavailable" and verification.status == "evaluated")
+ ):
+ return existing
+
+ copilot_ctx.completion_verification_result = verification
+ record_completion_verification(copilot_ctx, verification)
+ if verification.status == "evaluated":
+ _emit_completion_verification_trace(copilot_ctx, verification)
+ return verification
# Block types the copilot must never emit. They delegate the entire goal to
@@ -4709,6 +5016,7 @@ async def _navigate_post_hook(
raw: dict[str, Any],
ctx: AgentContext,
) -> dict[str, Any]:
+ _clear_pending_browser_interaction_observation(ctx)
if result.get("ok"):
data = result.pop("data", {})
result["url"] = data.get("url", "")
@@ -4743,9 +5051,11 @@ async def _click_post_hook(
raw: dict[str, Any],
ctx: AgentContext,
) -> dict[str, Any]:
+ _clear_pending_browser_interaction_observation(ctx)
if result.get("ok") and result.get("data"):
data = result["data"]
url, title = await _resolve_url_title(raw, ctx)
+ _mark_pending_browser_interaction_observation(ctx, tool_name="click", url=url)
result["data"] = {
"selector": data.get("selector", ""),
"url": url,
@@ -4818,6 +5128,7 @@ async def _type_text_post_hook(
raw: dict[str, Any],
ctx: AgentContext,
) -> dict[str, Any]:
+ _clear_pending_browser_interaction_observation(ctx)
if result.get("ok") and result.get("data"):
data = result["data"]
selector = data.get("selector", "")
@@ -4831,6 +5142,7 @@ async def _type_text_post_hook(
landing_failure = await _verify_scout_type_landed(ctx, selector=selector, typed_length=typed_length)
if landing_failure is not None:
return landing_failure
+ _mark_pending_browser_interaction_observation(ctx, tool_name="type_text", url=url)
return result
@@ -4850,12 +5162,23 @@ async def _evaluate_post_hook(
title = str(result["data"].get("title") or "")
if not title:
_, title = await _resolve_url_title(raw, ctx)
- _record_composition_page_observation(
+ observation_step = _record_composition_page_observation(
ctx,
source_tool="evaluate",
url=url,
title=title,
observed_data=result["data"],
+ append_to_flow=True,
+ reached_via="auto",
+ )
+ if observation_step is not None:
+ result["observation_step"] = observation_step
+ result["data"]["observation_step"] = observation_step
+ await _maybe_run_completion_verification_from_page_observation(
+ ctx,
+ url=url,
+ title=title,
+ observed_data=result["data"],
)
return result
@@ -4881,9 +5204,11 @@ async def _select_option_post_hook(
raw: dict[str, Any],
ctx: AgentContext,
) -> dict[str, Any]:
+ _clear_pending_browser_interaction_observation(ctx)
if result.get("ok") and result.get("data"):
data = result["data"]
url, _ = await _resolve_url_title(raw, ctx)
+ _mark_pending_browser_interaction_observation(ctx, tool_name="select_option", url=url)
result["data"] = {
"selector": data.get("selector", ""),
"value": data.get("value", ""),
@@ -4897,9 +5222,11 @@ async def _press_key_post_hook(
raw: dict[str, Any],
ctx: AgentContext,
) -> dict[str, Any]:
+ _clear_pending_browser_interaction_observation(ctx)
if result.get("ok") and result.get("data"):
data = result["data"]
url, _ = await _resolve_url_title(raw, ctx)
+ _mark_pending_browser_interaction_observation(ctx, tool_name="press_key", url=url)
result["data"] = {
"key": data.get("key", ""),
"selector": data.get("selector", ""),
@@ -5384,7 +5711,8 @@ def _is_unfinished_run_verification_candidate(copilot_ctx: Any, result: dict[str
def _build_run_evidence_snapshot(copilot_ctx: Any, result: dict[str, Any]) -> RunEvidenceSnapshot:
data = result.get("data")
data = data if isinstance(data, dict) else {}
- current_labels = set(_current_workflow_block_labels(copilot_ctx))
+ current_label_order = _current_workflow_block_labels(copilot_ctx)
+ current_labels = set(current_label_order)
# Evidence must be what THIS run produced. ``verified_block_outputs`` accumulates
# across incremental runs, so sourcing from it would let an output from a prior
# run satisfy a criterion the current run never re-produced.
@@ -5399,6 +5727,7 @@ def _build_run_evidence_snapshot(copilot_ctx: Any, result: dict[str, Any]) -> Ru
if isinstance(label, str) and label in current_labels and _is_meaningful_extracted_data(output):
block_outputs[label] = output
executed = data.get("executed_block_labels")
+ executed_block_labels = [str(label) for label in executed] if isinstance(executed, list) else []
page_title = data.get("page_title")
run_id = data.get("workflow_run_id")
return RunEvidenceSnapshot(
@@ -5406,7 +5735,12 @@ def _build_run_evidence_snapshot(copilot_ctx: Any, result: dict[str, Any]) -> Ru
block_outputs=block_outputs,
current_url=_valid_runtime_anchor_url(data.get("current_url")),
page_title=page_title if isinstance(page_title, str) and page_title.strip() else None,
- executed_block_labels=[str(label) for label in executed] if isinstance(executed, list) else [],
+ executed_block_labels=executed_block_labels,
+ verified_context_block_labels=_verified_context_block_labels_for_snapshot(
+ copilot_ctx,
+ current_label_order,
+ executed_block_labels,
+ ),
)
@@ -5417,11 +5751,7 @@ async def _maybe_run_completion_verification(
return None
if getattr(copilot_ctx, "copilot_total_timeout_exceeded", False):
return None
- policy = getattr(copilot_ctx, "request_policy", None)
- # A method-mandated criterion asserts HOW the goal was reached; this judge sees
- # only end-state evidence and would always return no_evidence, false-blocking a
- # legitimate success. Verifying method adherence needs an action trace (separate).
- criteria = [c for c in (policy.completion_criteria if policy is not None else []) if not c.method_mandated]
+ criteria = _completion_verification_criteria(copilot_ctx)
if not criteria:
return None
if not (
@@ -5495,6 +5825,42 @@ def _outcome_failure_warrants_repair(
return _current_workflow_has_evidence_block(copilot_ctx)
+def _tool_visible_result_after_completion_verification(
+ copilot_ctx: Any,
+ result: dict[str, Any],
+ completion_verification: CompletionVerificationResult | None,
+) -> dict[str, Any]:
+ outcome_unverified_reason = _outcome_unverified_reason(copilot_ctx, completion_verification)
+ if outcome_unverified_reason is None:
+ return result
+ if not _outcome_failure_warrants_repair(copilot_ctx, completion_verification):
+ return result
+
+ data = result.get("data")
+ copied_data = dict(data) if isinstance(data, dict) else {}
+ copied_data["failure_reason"] = outcome_unverified_reason
+ copied_data["completion_verification"] = (
+ completion_verification.to_trace_data() if completion_verification is not None else None
+ )
+ categories = copied_data.get("failure_categories")
+ copied_categories = list(categories) if isinstance(categories, list) else []
+ copied_categories.insert(
+ 0,
+ {
+ "category": "OUTCOME_UNVERIFIED",
+ "confidence_float": 1.0,
+ "reasoning": outcome_unverified_reason,
+ },
+ )
+ copied_data["failure_categories"] = copied_categories
+ return {
+ **result,
+ "ok": False,
+ "error": outcome_unverified_reason,
+ "data": copied_data,
+ }
+
+
def _emit_completion_verification_trace(
copilot_ctx: Any, completion_verification: CompletionVerificationResult
) -> None:
@@ -5769,6 +6135,27 @@ async def update_workflow_tool(
record_tool_step_result_for_ctx(copilot_ctx, "update_workflow", arguments, result)
return json.dumps(result)
+ with copilot_span(
+ "composition_evidence_precheck",
+ data={**_COMPOSITION_EVIDENCE_PRECHECK_TRACE_DATA, "tool_name": "update_workflow"},
+ ):
+ composition_evidence_error = _update_and_run_blocks_composition_evidence_precheck(
+ copilot_ctx,
+ workflow_yaml,
+ normalized_block_observation_refs,
+ block_observation_refs,
+ )
+ if composition_evidence_error:
+ result = {"ok": False, "error": composition_evidence_error}
+ record_tool_step_result_for_ctx(copilot_ctx, "update_workflow", arguments, result)
+ _record_diagnosis_repair_contract(
+ copilot_ctx,
+ source_tool="update_workflow",
+ result=result,
+ )
+ sanitized = sanitize_tool_result_for_llm("update_workflow", result)
+ return json.dumps(sanitized)
+
with copilot_span("update_workflow", data={"yaml_length": len(workflow_yaml)}):
result = await _update_workflow(
{**arguments, "raw_block_observation_refs": block_observation_refs},
@@ -5835,6 +6222,10 @@ def _frontier_run_size_result(
block_labels: list[str],
labels_to_execute: list[str],
) -> dict[str, Any]:
+ suggested_labels = list(labels_to_execute[:_MAX_INCREMENTAL_PAGE_FRONTIER_LABELS])
+ user_facing_summary = (
+ "Workflow draft saved; I still need to test the next smaller browser frontier before continuing."
+ )
return {
"ok": False,
"error": error,
@@ -5845,8 +6236,16 @@ def _frontier_run_size_result(
"requested_block_labels": list(block_labels),
"executed_block_labels": [],
"planned_block_labels": list(labels_to_execute),
- "suggested_block_labels": list(labels_to_execute[:_MAX_INCREMENTAL_PAGE_FRONTIER_LABELS]),
+ "suggested_block_labels": suggested_labels,
"deferred_block_labels": list(labels_to_execute[_MAX_INCREMENTAL_PAGE_FRONTIER_LABELS:]),
+ "control_signal": {
+ "kind": "intermediate_success",
+ "user_facing_summary": user_facing_summary,
+ "next_tool": "run_blocks_and_collect_debug",
+ "next_block_labels": suggested_labels,
+ "preserve_workflow_yaml": True,
+ },
+ "user_facing_summary": user_facing_summary,
},
}
@@ -5934,15 +6333,20 @@ async def run_blocks_tool(
)
completion_verification = await _maybe_run_completion_verification(copilot_ctx, result, handler_start)
_record_run_blocks_result(copilot_ctx, result, completion_verification=completion_verification)
- record_tool_step_result_for_ctx(copilot_ctx, "run_blocks_and_collect_debug", arguments, result)
+ tool_visible_result = _tool_visible_result_after_completion_verification(
+ copilot_ctx,
+ result,
+ completion_verification,
+ )
+ record_tool_step_result_for_ctx(copilot_ctx, "run_blocks_and_collect_debug", arguments, tool_visible_result)
_record_diagnosis_repair_contract(
copilot_ctx,
source_tool="run_blocks_and_collect_debug",
- result=result,
+ result=tool_visible_result,
)
enqueue_screenshot_from_result(copilot_ctx, result)
- sanitized = sanitize_tool_result_for_llm("run_blocks_and_collect_debug", result)
+ sanitized = sanitize_tool_result_for_llm("run_blocks_and_collect_debug", tool_visible_result)
return json.dumps(sanitized)
@@ -6029,8 +6433,8 @@ async def update_and_run_blocks_tool(
for the reusable actions/checks the workflow actually needs.
When you compose no-url blocks from a page reached by prior clicks, include
`block_observation_refs` entries with each block label and the
- `observation_step` returned by inspect_page_for_composition for the page
- that block acts on.
+ `observation_step` returned by inspect_page_for_composition or evaluate for
+ the page that block acts on.
When inspected evidence shows an anti-bot challenge gating a disabled
submit/search control, account for challenge resolution before submit;
do not compose a click against a control observed as disabled.
@@ -6076,6 +6480,8 @@ async def update_and_run_blocks_tool(
sanitized = sanitize_tool_result_for_llm("update_and_run_blocks", result)
return json.dumps(sanitized)
+ _clear_pending_browser_interaction_observation(copilot_ctx)
+
# Snapshot the prior workflow definition BEFORE _update_workflow saves
# the new one — we need the pre-update state to diff against.
prior_definition = await _get_prior_workflow_definition(copilot_ctx)
@@ -6217,18 +6623,23 @@ async def update_and_run_blocks_tool(
)
completion_verification = await _maybe_run_completion_verification(copilot_ctx, run_result, handler_start)
_record_run_blocks_result(copilot_ctx, run_result, completion_verification=completion_verification)
- record_tool_step_result_for_ctx(copilot_ctx, "update_and_run_blocks", arguments, run_result)
+ tool_visible_result = _tool_visible_result_after_completion_verification(
+ copilot_ctx,
+ run_result,
+ completion_verification,
+ )
+ record_tool_step_result_for_ctx(copilot_ctx, "update_and_run_blocks", arguments, tool_visible_result)
_record_diagnosis_repair_contract(
copilot_ctx,
source_tool="update_and_run_blocks",
- result=run_result,
+ result=tool_visible_result,
workflow_updated=True,
)
enqueue_screenshot_from_result(copilot_ctx, run_result)
if run_result.get("ok"):
advance_to_testing(copilot_ctx)
- sanitized = sanitize_tool_result_for_llm("run_blocks_and_collect_debug", run_result)
+ sanitized = sanitize_tool_result_for_llm("run_blocks_and_collect_debug", tool_visible_result)
return json.dumps(sanitized)
@@ -6268,6 +6679,8 @@ _DISCOVERY_ANTI_BOT_PATTERNS = (
"just a moment",
"captcha",
"challenge",
+ "turnstile",
+ "cf-turnstile",
"human-verification",
"human verification",
"verify you are human",
@@ -6652,17 +7065,21 @@ def _composition_visual_prompt(evidence: dict[str, Any]) -> str:
"challenge_state": evidence.get("challenge_state") or {},
"form_count": len(evidence.get("forms") or []),
"result_container_count": len(evidence.get("result_containers") or []),
+ "page_obstruction_count": len(evidence.get("page_obstructions") or []),
"schema_empty_page": evidence.get("schema_empty_page") is True,
}
return (
"Summarize this screenshot for Workflow Copilot build-time page evidence. "
"Return JSON only with keys: summary, challenge_detected, challenge_kind, "
"challenge_location, submit_blocked, blocked_submit_controls, empty_page_visible, "
- "loading_state_visible, omissions. Focus on "
+ "loading_state_visible, page_obstruction_detected, obstruction_kind, "
+ "obstruction_location, underlying_page_blocked, visible_dismiss_controls, omissions. Focus on "
"visible anti-bot or human-verification state, whether it appears to gate a submit/search "
- "control, and where it appears relative to the page controls. Do not include raw DOM, "
+ "control, whether any visible artificial barrier mechanically blocks the underlying page, "
+ "and where it appears relative to the page controls. Do not include raw DOM, "
"code, selectors, personal data, or workflow instructions. If no challenge is visible, "
- "set challenge_detected to false and submit_blocked to false. If DOM context shows a "
+ "set challenge_detected to false and submit_blocked to false. If no page obstruction is visible, "
+ "set page_obstruction_detected to false. If DOM context shows a "
"schema-empty page, set empty_page_visible to true only when the screenshot shows a "
"settled page with no visible forms, controls, result data, challenge, or loading/progress "
"state; set loading_state_visible to true when the page appears to be waiting, loading, "
@@ -6689,6 +7106,11 @@ def _normalize_visual_summary(value: Any) -> dict[str, Any] | None:
blocked_submit_controls = value.get("blocked_submit_controls")
empty_page_visible = value.get("empty_page_visible")
loading_state_visible = value.get("loading_state_visible")
+ page_obstruction_detected = value.get("page_obstruction_detected")
+ obstruction_kind = value.get("obstruction_kind")
+ obstruction_location = value.get("obstruction_location")
+ underlying_page_blocked = value.get("underlying_page_blocked")
+ visible_dismiss_controls = value.get("visible_dismiss_controls")
omissions = value.get("omissions")
return {
"summary": summary if isinstance(summary, str) else "",
@@ -6698,9 +7120,16 @@ def _normalize_visual_summary(value: Any) -> dict[str, Any] | None:
"submit_blocked": submit_blocked if isinstance(submit_blocked, bool) else None,
"empty_page_visible": empty_page_visible if isinstance(empty_page_visible, bool) else None,
"loading_state_visible": loading_state_visible if isinstance(loading_state_visible, bool) else None,
+ "page_obstruction_detected": page_obstruction_detected if isinstance(page_obstruction_detected, bool) else None,
+ "obstruction_kind": obstruction_kind if isinstance(obstruction_kind, str) else "",
+ "obstruction_location": obstruction_location if isinstance(obstruction_location, str) else "",
+ "underlying_page_blocked": underlying_page_blocked if isinstance(underlying_page_blocked, bool) else None,
"blocked_submit_controls": [item for item in blocked_submit_controls if isinstance(item, str)]
if isinstance(blocked_submit_controls, list)
else [],
+ "visible_dismiss_controls": [item for item in visible_dismiss_controls if isinstance(item, str)]
+ if isinstance(visible_dismiss_controls, list)
+ else [],
"omissions": [item for item in omissions if isinstance(item, str)] if isinstance(omissions, list) else [],
}
@@ -6820,18 +7249,19 @@ async def _composition_evidence_after_navigation_failure(
_MAX_FLOW_EVIDENCE = 64
-def _inspection_reached_via(*, use_current_page: bool, post_run: bool) -> str:
+def _inspection_reached_via(*, use_current_page: bool, post_run: bool, earned_interaction: bool) -> str:
"""How the just-inspected state was reached, for the flow-evidence trajectory.
A target_url inspection navigates there itself ("navigate"); a post-run
- current-page inspection observes the page the run left behind ("post_run"); any
- other current-page inspection follows an in-turn interaction on that page.
+ current-page inspection observes the page the run left behind ("post_run"); a
+ normal current-page inspection counts as an interaction only when a successful
+ browser action immediately earned that credit.
"""
if not use_current_page:
return "navigate"
if post_run:
return "post_run"
- return "interaction"
+ return "interaction" if earned_interaction else "current_page"
def _append_flow_evidence(copilot_ctx: Any, evidence: dict[str, Any], *, reached_via: str) -> int | None:
@@ -6869,6 +7299,61 @@ def _append_flow_evidence(copilot_ctx: Any, evidence: dict[str, Any], *, reached
return step
+def _latest_interaction_reached_flow_evidence(copilot_ctx: Any) -> tuple[int, str, dict[str, Any]] | None:
+ trajectory = getattr(copilot_ctx, "flow_evidence", None)
+ if not isinstance(trajectory, list):
+ return None
+ for entry in reversed(trajectory):
+ if not isinstance(entry, dict):
+ continue
+ reached_via = str(entry.get("reached_via") or "")
+ if reached_via not in {"interaction", "post_run"}:
+ continue
+ evidence = entry.get("evidence")
+ step = entry.get("step")
+ if isinstance(step, bool) or not isinstance(step, int) or not isinstance(evidence, dict):
+ continue
+ if not has_bounded_page_schema(evidence):
+ continue
+ observed_url = _composition_evidence_page_url(evidence)
+ if observed_url:
+ return step, observed_url, evidence
+ return None
+
+
+def _non_current_inspection_regression_error(copilot_ctx: Any, *, entry_url: str) -> dict[str, Any] | None:
+ latest = _latest_interaction_reached_flow_evidence(copilot_ctx)
+ if latest is None:
+ return None
+ observation_step, observed_url, _ = latest
+ if _same_page_ignoring_fragment(observed_url, entry_url):
+ return None
+ return {
+ "ok": False,
+ "data": {
+ "current_url": observed_url,
+ "observation_step": observation_step,
+ },
+ "error": (
+ "inspect_page_for_composition would navigate away from the latest interaction-reached page "
+ f'({observed_url}). Use inspect_page_for_composition(target_url="current_page") to inspect '
+ "the live page, or compose from the existing page evidence and pass observation_step "
+ f"{observation_step} in block_observation_refs for blocks that act on that reached page."
+ ),
+ }
+
+
+def _page_inspection_budget_error(copilot_ctx: Any, *, scope: Literal["turn", "chat"]) -> str:
+ scope_label = "turn" if scope == "turn" else "chat"
+ return (
+ f"inspect_page_for_composition reached the page-inspection budget for this {scope_label}. "
+ "This is not evidence that scouting is complete. Use evaluate, get_browser_screenshot, or a browser "
+ "action on the current page to determine whether the goal is already satisfied, whether progress is still "
+ "possible, or whether a real blocker exists. Do not author downstream result, extraction, or confirmation "
+ "blocks unless the existing evidence already shows the page state those blocks will act on."
+ )
+
+
_COMPOSITION_HOLLOW_RECAPTURE_RETRIES = 2
_COMPOSITION_HOLLOW_RECAPTURE_DELAY_SECONDS = 2.5
# The composition inspect navigates with `domcontentloaded`, so a heavier cap than
@@ -6998,6 +7483,8 @@ async def _inspect_page_for_composition_impl(
return result
use_current_page = (target_url or "").strip().lower() in _CURRENT_PAGE_INSPECTION_TARGETS
+ if not use_current_page:
+ _clear_pending_browser_interaction_observation(copilot_ctx)
bypass_budget_for_post_run_current_page = _allows_post_run_current_page_inspection_budget_bypass(
copilot_ctx,
use_current_page=use_current_page,
@@ -7010,11 +7497,7 @@ async def _inspect_page_for_composition_impl(
result = {
"ok": False,
"data": None,
- "error": (
- "inspect_page_for_composition reached the page-inspection budget for this turn. "
- "Compose from existing evidence or ask for clarification if inspection still cannot determine "
- "required page inputs."
- ),
+ "error": _page_inspection_budget_error(copilot_ctx, scope="turn"),
}
record_tool_step_result_for_ctx(copilot_ctx, "inspect_page_for_composition", arguments, result)
return result
@@ -7024,7 +7507,7 @@ async def _inspect_page_for_composition_impl(
result = {
"ok": False,
"data": None,
- "error": "inspect_page_for_composition has reached the page-inspection budget for this chat. Compose from existing evidence or ask for clarification.",
+ "error": _page_inspection_budget_error(copilot_ctx, scope="chat"),
}
record_tool_step_result_for_ctx(copilot_ctx, "inspect_page_for_composition", arguments, result)
return result
@@ -7046,6 +7529,10 @@ async def _inspect_page_for_composition_impl(
record_tool_step_result_for_ctx(copilot_ctx, "inspect_page_for_composition", arguments, result)
return result
entry_url = resolved_entry_url
+ regression_error = _non_current_inspection_regression_error(copilot_ctx, entry_url=entry_url)
+ if regression_error is not None:
+ record_tool_step_result_for_ctx(copilot_ctx, "inspect_page_for_composition", arguments, regression_error)
+ return regression_error
with copilot_span(
"inspect_page_for_composition",
@@ -7128,7 +7615,30 @@ async def _inspect_page_for_composition_impl(
if bypass_budget_for_post_run_current_page:
copilot_ctx.post_run_current_page_inspection_workflow_run_id = run_id
copilot_ctx.composition_page_evidence = evidence
- reached_via = _inspection_reached_via(use_current_page=use_current_page, post_run=bool(run_id))
+ if (
+ isinstance(run_id, str)
+ and run_id
+ and getattr(copilot_ctx, "post_run_page_observation_after_failed_test", False)
+ ):
+ page_title = evidence.get("page_title")
+ await _maybe_run_completion_verification_from_page_observation(
+ copilot_ctx,
+ url=str(evidence.get("current_url") or current_url or ""),
+ title=page_title if isinstance(page_title, str) else "",
+ observed_data=evidence,
+ )
+ earned_interaction = False
+ if use_current_page and not run_id:
+ earned_interaction = _consume_pending_browser_interaction_observation(
+ copilot_ctx,
+ current_url=str(evidence.get("current_url") or current_url or ""),
+ evidence=evidence,
+ )
+ reached_via = _inspection_reached_via(
+ use_current_page=use_current_page,
+ post_run=bool(run_id),
+ earned_interaction=earned_interaction,
+ )
observation_step = _append_flow_evidence(copilot_ctx, evidence, reached_via=reached_via)
if observation_step is None:
LOG.warning("copilot_flow_evidence_append_failed_no_trajectory")
diff --git a/skyvern/forge/sdk/copilot/turn_intent.py b/skyvern/forge/sdk/copilot/turn_intent.py
index 6b5fbec00..ca69ee976 100644
--- a/skyvern/forge/sdk/copilot/turn_intent.py
+++ b/skyvern/forge/sdk/copilot/turn_intent.py
@@ -570,7 +570,6 @@ def build_turn_intent(
required_context.append(RequiredContextKey.LATEST_RUN_RESULT)
reason_codes.append(TurnIntentReasonCode.RUN_CONTEXT_PRESENT)
if browser_session_id:
- required_context.append(RequiredContextKey.BROWSER_STATE)
reason_codes.append(TurnIntentReasonCode.BROWSER_CONTEXT_PRESENT)
if request_policy.credential_input_kind != "none" or request_policy.resolved_credentials:
required_context.append(RequiredContextKey.CREDENTIAL_METADATA)
diff --git a/skyvern/forge/sdk/routes/__init__.py b/skyvern/forge/sdk/routes/__init__.py
index dd58df058..3cca8d97d 100644
--- a/skyvern/forge/sdk/routes/__init__.py
+++ b/skyvern/forge/sdk/routes/__init__.py
@@ -1,3 +1,5 @@
+import importlib.util
+
from skyvern.forge.sdk.routes import agent_protocol # noqa: F401
from skyvern.forge.sdk.routes import browser_profiles # noqa: F401
from skyvern.forge.sdk.routes import browser_sessions # noqa: F401
@@ -10,9 +12,15 @@ from skyvern.forge.sdk.routes import runtime_config # noqa: F401
from skyvern.forge.sdk.routes import scripts # noqa: F401
from skyvern.forge.sdk.routes import sdk # noqa: F401
from skyvern.forge.sdk.routes import webhooks # noqa: F401
-from skyvern.forge.sdk.routes import workflow_copilot # noqa: F401
from skyvern.forge.sdk.routes import workflow_schedules # noqa: F401
from skyvern.forge.sdk.routes.streaming import cdp_input # noqa: F401
from skyvern.forge.sdk.routes.streaming import messages # noqa: F401
from skyvern.forge.sdk.routes.streaming import screenshot # noqa: F401
from skyvern.forge.sdk.routes.streaming import vnc # noqa: F401
+
+# Workflow Copilot depends on openai-agents, imported as `agents`, which is
+# intentionally server-only.
+# skyvern[local] starts the embedded API app without that extra, so skip these
+# private routes unless the SDK package is installed.
+if importlib.util.find_spec("agents") is not None:
+ from skyvern.forge.sdk.routes import workflow_copilot # noqa: F401
diff --git a/skyvern/webeye/real_browser_state.py b/skyvern/webeye/real_browser_state.py
index 93a5aba29..a62ecd3ea 100644
--- a/skyvern/webeye/real_browser_state.py
+++ b/skyvern/webeye/real_browser_state.py
@@ -36,6 +36,19 @@ SETTLE_TIME_MS = 750
SETTLE_JITTER_MS = 500
+def _same_page_ignoring_fragment(left: str | None, right: str | None) -> bool:
+ if not left or not right:
+ return False
+ try:
+ left_parsed = urlparse(left)
+ right_parsed = urlparse(right)
+ except Exception:
+ return False
+ left_url = left_parsed._replace(fragment="").geturl().rstrip("/")
+ right_url = right_parsed._replace(fragment="").geturl().rstrip("/")
+ return left_url == right_url
+
+
class RealBrowserState(BrowserState):
def __init__(
self,
@@ -118,7 +131,7 @@ class RealBrowserState(BrowserState):
use_existing_page = False
if browser_address and len(self.browser_context.pages) > 0:
pages = await self.list_valid_pages()
- if len(pages) > 0:
+ if pages:
page = pages[-1]
use_existing_page = True
if page is None:
@@ -128,7 +141,7 @@ class RealBrowserState(BrowserState):
if not use_existing_page:
await self._close_all_other_pages()
- if url and page.url.rstrip("/") != url.rstrip("/"):
+ if url and not _same_page_ignoring_fragment(page.url, url):
await self.navigate_to_url(page=page, url=url)
async def _wait_for_settle(self) -> None:
diff --git a/tests/unit/test_copilot_completion_verification.py b/tests/unit/test_copilot_completion_verification.py
index 1dfde1f07..1239f6eaf 100644
--- a/tests/unit/test_copilot_completion_verification.py
+++ b/tests/unit/test_copilot_completion_verification.py
@@ -7,7 +7,11 @@ from types import SimpleNamespace
import pytest
from skyvern.config import settings
-from skyvern.forge.sdk.copilot.agent import _completion_contract_not_violated, _verified_workflow_or_none
+from skyvern.forge.sdk.copilot.agent import (
+ _completion_contract_not_violated,
+ _rewrite_failed_test_response,
+ _verified_workflow_or_none,
+)
from skyvern.forge.sdk.copilot.completion_verification import (
CompletionVerificationResult,
CriterionVerdict,
@@ -39,9 +43,12 @@ from skyvern.forge.sdk.copilot.tools import (
_is_outcome_evidence_candidate,
_is_unfinished_run_verification_candidate,
_maybe_run_completion_verification,
+ _maybe_run_completion_verification_from_page_observation,
_outcome_failure_warrants_repair,
_outcome_unverified_reason,
+ _record_composition_page_observation,
_record_run_blocks_result,
+ _tool_visible_result_after_completion_verification,
)
@@ -320,6 +327,33 @@ def test_record_run_blocks_downgrades_when_confirmation_block_present_but_unmet(
assert "item in cart" in (ctx.last_test_failure_reason or "")
+def test_tool_visible_result_fails_when_confirmation_block_outcome_unmet() -> None:
+ ctx = _ctx_with_blocks("extraction")
+ result = _clean_success_result()
+ verification = _evaluated(("c0", False))
+
+ visible = _tool_visible_result_after_completion_verification(ctx, result, verification)
+
+ assert visible["ok"] is False
+ assert "item in cart" in visible["error"]
+ assert result["ok"] is True
+ assert visible["data"]["overall_status"] == "completed"
+ assert visible["data"]["completion_verification"]["fully_satisfied"] is False
+ assert visible["data"]["failure_categories"][0]["category"] == "OUTCOME_UNVERIFIED"
+
+
+def test_tool_visible_result_keeps_mid_build_run_visible_success() -> None:
+ ctx = _ctx_with_blocks("goto_url", "navigation")
+
+ visible = _tool_visible_result_after_completion_verification(
+ ctx,
+ _clean_success_result(),
+ _evaluated(("c0", False)),
+ )
+
+ assert visible["ok"] is True
+
+
def test_record_run_blocks_keeps_building_on_mid_build_no_evidence() -> None:
ctx = _ctx_with_blocks("goto_url", "navigation")
_record_run_blocks_result(ctx, _clean_success_result(), completion_verification=_evaluated(("c0", False)))
@@ -472,6 +506,187 @@ def test_verified_workflow_presented_on_recognized_canceled_run() -> None:
assert _verified_workflow_or_none(ctx) == (None, None)
+async def test_page_observation_verification_recognizes_budgeted_outcome(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ seen_prompt: dict[str, str] = {}
+
+ async def handler(**kwargs: object) -> dict:
+ seen_prompt["prompt"] = str(kwargs.get("prompt") or "")
+ return {"verdicts": [{"criterion_id": "c0", "satisfied": True, "reason_code": "evidence_confirms"}]}
+
+ monkeypatch.setattr(
+ "skyvern.forge.sdk.copilot.tools._completion_verification_handler",
+ _completion_handler_lookup(handler),
+ )
+ ctx = _run_ctx()
+ ctx.last_test_ok = False
+ ctx.last_run_blocks_workflow_run_id = "wr_cancel"
+ ctx.copilot_run_start_monotonic = time.monotonic()
+ _record_composition_page_observation(
+ ctx,
+ source_tool="evaluate",
+ url="https://example.com/cart",
+ title="Shopping Cart",
+ observed_data={
+ "hasProduct": True,
+ "excerpts": ["SKU-12345 is present in the cart"],
+ "url": "https://example.com/cart",
+ "title": "Shopping Cart",
+ },
+ )
+
+ result = await _maybe_run_completion_verification_from_page_observation(
+ ctx,
+ url="https://example.com/cart",
+ title="Shopping Cart",
+ observed_data={
+ "hasProduct": True,
+ "excerpts": ["SKU-12345 is present in the cart"],
+ },
+ )
+
+ assert result is not None
+ assert result.is_fully_satisfied() is True
+ assert ctx.completion_verification_result is result
+ assert outcome_fully_verified(ctx) is True
+ assert "current_page_observation" in seen_prompt["prompt"]
+ assert "SKU-12345 is present in the cart" in seen_prompt["prompt"]
+
+
+async def test_page_observation_verification_does_not_overwrite_satisfied_verdict(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ async def handler(**_: object) -> dict:
+ raise AssertionError("handler should not be called once the outcome is verified")
+
+ monkeypatch.setattr(
+ "skyvern.forge.sdk.copilot.tools._completion_verification_handler",
+ _completion_handler_lookup(handler),
+ )
+ ctx = _run_ctx()
+ ctx.last_test_ok = False
+ ctx.last_run_blocks_workflow_run_id = "wr_cancel"
+ existing = _evaluated(("c0", True))
+ ctx.completion_verification_result = existing
+ _record_composition_page_observation(
+ ctx,
+ source_tool="evaluate",
+ url="https://example.com/cart",
+ observed_data={"hasProduct": True},
+ )
+
+ result = await _maybe_run_completion_verification_from_page_observation(
+ ctx,
+ url="https://example.com/cart",
+ observed_data={"hasProduct": True},
+ )
+
+ assert result is existing
+ assert ctx.completion_verification_result is existing
+
+
+async def test_page_observation_verification_preserves_existing_unsatisfied_verdict(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ handler_calls = 0
+
+ async def handler(**_: object) -> dict:
+ nonlocal handler_calls
+ handler_calls += 1
+ return {"verdicts": [{"criterion_id": "c0", "satisfied": False, "reason_code": "no_evidence"}]}
+
+ monkeypatch.setattr(
+ "skyvern.forge.sdk.copilot.tools._completion_verification_handler",
+ _completion_handler_lookup(handler),
+ )
+ ctx = _run_ctx()
+ ctx.last_test_ok = False
+ ctx.last_run_blocks_workflow_run_id = "wr_cancel"
+ existing = _evaluated(("c0", False))
+ ctx.completion_verification_result = existing
+ _record_composition_page_observation(
+ ctx,
+ source_tool="evaluate",
+ url="https://example.com/cart",
+ observed_data={"hasProduct": False},
+ )
+
+ result = await _maybe_run_completion_verification_from_page_observation(
+ ctx,
+ url="https://example.com/cart",
+ observed_data={"hasProduct": False},
+ )
+
+ assert handler_calls == 1
+ assert result is existing
+ assert ctx.completion_verification_result is existing
+
+
+async def test_page_observation_verification_can_upgrade_unsatisfied_verdict(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ async def handler(**_: object) -> dict:
+ return {"verdicts": [{"criterion_id": "c0", "satisfied": True, "reason_code": "evidence_confirms"}]}
+
+ monkeypatch.setattr(
+ "skyvern.forge.sdk.copilot.tools._completion_verification_handler",
+ _completion_handler_lookup(handler),
+ )
+ ctx = _run_ctx()
+ ctx.last_test_ok = False
+ ctx.last_run_blocks_workflow_run_id = "wr_cancel"
+ existing = _evaluated(("c0", False))
+ ctx.completion_verification_result = existing
+ _record_composition_page_observation(
+ ctx,
+ source_tool="evaluate",
+ url="https://example.com/cart",
+ observed_data={"hasProduct": True},
+ )
+
+ result = await _maybe_run_completion_verification_from_page_observation(
+ ctx,
+ url="https://example.com/cart",
+ observed_data={"hasProduct": True},
+ )
+
+ assert result is not None
+ assert result is not existing
+ assert result.is_fully_satisfied() is True
+ assert ctx.completion_verification_result is result
+
+
+def test_failed_test_rewrite_recognizes_post_budget_verified_outcome() -> None:
+ ctx = _canceled_gate_ctx()
+ ctx.last_workflow = SimpleNamespace()
+ ctx.last_workflow_yaml = "workflow: {}"
+ ctx.last_update_block_count = 5
+ ctx.completion_verification_result = _evaluated(("c0", True))
+
+ response = _rewrite_failed_test_response("The test failed.", ctx)
+
+ assert "verified the requested outcome" in response
+ assert "test failed" not in response.lower()
+
+
+def test_failed_test_rewrite_does_not_render_zero_block_verified_outcome() -> None:
+ ctx = _canceled_gate_ctx()
+ ctx.last_workflow = SimpleNamespace()
+ ctx.last_workflow_yaml = "workflow: {}"
+ ctx.last_update_block_count = 0
+ ctx.completion_verification_result = _evaluated(("c0", True))
+
+ response = _rewrite_failed_test_response("The test failed.", ctx)
+
+ assert "0 blocks" not in response
+ assert "workflow with 0" not in response
+
+
# --- SKY-10576: recognition governed by whole-workflow outcome, not per-block prefix ---
#
# A clean ok=True run can reach the goal (its outcome block produced data and the
@@ -608,6 +823,150 @@ def test_snapshot_uses_current_run_blocks_not_stale_outputs() -> None:
assert snap2.block_outputs.get("b3") == {"extracted_information": {"price": "9.99"}}
+def test_snapshot_includes_verified_context_labels_without_prior_outputs() -> None:
+ ctx = _run_ctx()
+ labels = [
+ "open_bacb_homepage",
+ "click_find_a_certificant",
+ "search_noor_assi_rbt",
+ "expand_noor_assi_result",
+ "extract_credential_details",
+ ]
+ ctx.last_workflow = SimpleNamespace(
+ workflow_definition=SimpleNamespace(
+ blocks=[SimpleNamespace(label=label, block_type="task") for label in labels]
+ )
+ )
+ ctx.verified_prefix_labels = list(labels)
+ ctx.verified_block_outputs["expand_noor_assi_result"] = {"stale": "prior run output"}
+
+ run = {
+ "ok": True,
+ "data": {
+ "workflow_run_id": "wr_extract",
+ "current_url": "https://www.bacb.com/services/o.php?page=101135",
+ "executed_block_labels": ["extract_credential_details"],
+ "blocks": [
+ {
+ "label": "extract_credential_details",
+ "block_type": "EXTRACTION",
+ "status": "completed",
+ "extracted_data": {
+ "extracted_information": {
+ "credentials": [
+ {
+ "credential_type": "Registered Behavior Technician",
+ "credential_number": "RBT-19-98341",
+ "expiration_date": "09/06/2022",
+ }
+ ]
+ }
+ },
+ }
+ ],
+ },
+ }
+
+ snapshot = _build_run_evidence_snapshot(ctx, run)
+
+ assert snapshot.verified_context_block_labels == labels[:-1]
+ assert snapshot.block_outputs == {
+ "extract_credential_details": {
+ "extracted_information": {
+ "credentials": [
+ {
+ "credential_type": "Registered Behavior Technician",
+ "credential_number": "RBT-19-98341",
+ "expiration_date": "09/06/2022",
+ }
+ ]
+ }
+ }
+ }
+ assert "expand_noor_assi_result" not in snapshot.block_outputs
+ rendered = snapshot.render_prompt_block()
+ assert "verified_context_block_labels: open_bacb_homepage" in rendered
+ assert "expand_noor_assi_result" in rendered
+
+
+async def test_completion_verification_receives_verified_context_labels(monkeypatch: pytest.MonkeyPatch) -> None:
+ seen_prompt: dict[str, str] = {}
+
+ async def handler(**kwargs: object) -> dict:
+ seen_prompt["prompt"] = str(kwargs.get("prompt") or "")
+ return {
+ "verdicts": [
+ {"criterion_id": "c0", "satisfied": True, "reason_code": "evidence_confirms"},
+ {"criterion_id": "c1", "satisfied": True, "reason_code": "evidence_confirms"},
+ ]
+ }
+
+ monkeypatch.setattr(
+ "skyvern.forge.sdk.copilot.tools._completion_verification_handler",
+ _completion_handler_lookup(handler),
+ )
+ monkeypatch.setattr(settings, "COPILOT_OUTCOME_VERIFICATION_ENABLED", True)
+ ctx = _run_ctx()
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ _criterion("c0", "credential type, number, and expiration date are reported"),
+ _criterion("c1", "the data came from the expanded certificant result"),
+ ]
+ )
+ labels = [
+ "open_bacb_homepage",
+ "click_find_a_certificant",
+ "search_noor_assi_rbt",
+ "expand_noor_assi_result",
+ "extract_credential_details",
+ ]
+ ctx.last_workflow = SimpleNamespace(
+ workflow_definition=SimpleNamespace(
+ blocks=[
+ SimpleNamespace(label=label, block_type="extraction" if label.startswith("extract") else "task")
+ for label in labels
+ ]
+ )
+ )
+ ctx.verified_prefix_labels = list(labels)
+ result = {
+ "ok": True,
+ "data": {
+ "workflow_run_id": "wr_extract",
+ "current_url": "https://www.bacb.com/services/o.php?page=101135",
+ "executed_block_labels": ["extract_credential_details"],
+ "blocks": [
+ {
+ "label": "extract_credential_details",
+ "block_type": "EXTRACTION",
+ "status": "completed",
+ "extracted_data": {
+ "extracted_information": {
+ "person_name": "NOOR ASSI",
+ "credentials": [
+ {
+ "credential_type": "Registered Behavior Technician",
+ "credential_number": "RBT-19-98341",
+ "expiration_date": "09/06/2022",
+ }
+ ],
+ }
+ },
+ }
+ ],
+ },
+ }
+
+ verification = await _maybe_run_completion_verification(ctx, result, time.monotonic())
+
+ assert verification is not None
+ assert verification.is_fully_satisfied() is True
+ assert "verified_context_block_labels" in seen_prompt["prompt"]
+ assert "expand_noor_assi_result" in seen_prompt["prompt"]
+ assert "RBT-19-98341" in seen_prompt["prompt"]
+
+
@pytest.mark.asyncio
async def test_maybe_run_completion_verification_unavailable_on_low_budget(monkeypatch: pytest.MonkeyPatch) -> None:
async def handler(**_: object) -> dict:
diff --git a/tests/unit/test_copilot_composition_evidence.py b/tests/unit/test_copilot_composition_evidence.py
index e95eef05f..aa4a899f4 100644
--- a/tests/unit/test_copilot_composition_evidence.py
+++ b/tests/unit/test_copilot_composition_evidence.py
@@ -10,6 +10,7 @@ import yaml
from skyvern.forge.sdk.copilot.build_phase import BuildPhase
from skyvern.forge.sdk.copilot.composition_evidence import (
composition_page_evidence_error,
+ has_bounded_page_schema,
merge_visual_composition_evidence,
normalize_block_observation_refs,
page_evidence_needs_visual_fallback,
@@ -124,6 +125,109 @@ def test_composition_parse_html_extracts_labeled_fields_and_submit_controls() ->
assert parsed["source_tool"] == "inspect_page_for_composition"
+def test_composition_parse_html_extracts_modal_overlay_controls() -> None:
+ parsed = parse_composition_html(
+ """
+ <html><body>
+ <div id="newsletter" role="dialog" aria-modal="true" class="promo-modal">
+ <h2>Get updates</h2>
+ <p>Join our list before browsing.</p>
+ <button aria-label="Close modal">x</button>
+ <button>No thanks</button>
+ </div>
+ </body></html>
+ """,
+ inspected_url="https://example.com/results",
+ current_url="https://example.com/results",
+ )
+
+ assert parsed["modal_overlays"][0]["selector"] == "#newsletter"
+ assert parsed["modal_overlays"][0]["role"] == "dialog"
+ assert parsed["modal_overlays"][0]["dismiss_controls"][0]["text"] == "x"
+ assert parsed["modal_overlays"][0]["dismiss_controls"][0]["aria_label"] == "Close modal"
+ assert parsed["page_obstructions"][0]["kind"] == "modal_overlay"
+ assert parsed["page_obstructions"][0]["visible_controls"][0]["text"] == "x"
+ assert has_bounded_page_schema(parsed) is True
+
+
+def test_composition_parse_html_extracts_class_only_modal_overlay() -> None:
+ parsed = parse_composition_html(
+ """
+ <html><body>
+ <div class="promo modal">
+ <h2>Before you continue</h2>
+ <button>Close</button>
+ </div>
+ </body></html>
+ """,
+ inspected_url="https://example.com/results",
+ current_url="https://example.com/results",
+ )
+
+ assert parsed["modal_overlays"][0]["class"] == "promo modal"
+ assert parsed["modal_overlays"][0]["dismiss_controls"][0]["text"] == "Close"
+ assert parsed["page_obstructions"][0]["visible_controls"][0]["text"] == "Close"
+ assert has_bounded_page_schema(parsed) is True
+
+
+def test_composition_parse_html_ignores_hidden_modal_overlay_markup() -> None:
+ parsed = parse_composition_html(
+ """
+ <html><body>
+ <div id="closedDialog" role="dialog" aria-modal="true" aria-hidden="true">
+ <button>Close</button>
+ </div>
+ <div id="closedModal" class="modal" style="display: none;">
+ <button>Dismiss</button>
+ </div>
+ <div aria-hidden="true">
+ <div id="wrappedDialog" role="dialog">
+ <button>Close</button>
+ </div>
+ </div>
+ </body></html>
+ """,
+ inspected_url="https://example.com/results",
+ current_url="https://example.com/results",
+ )
+
+ assert parsed["modal_overlays"] == []
+ assert parsed["page_obstructions"] == []
+
+
+def test_composition_parse_html_ignores_empty_modal_root_as_bounded_schema() -> None:
+ parsed = parse_composition_html(
+ """
+ <html><body>
+ <div id="modal-root"></div>
+ </body></html>
+ """,
+ inspected_url="https://example.com/results",
+ current_url="https://example.com/results",
+ )
+
+ assert parsed["modal_overlays"] == []
+ assert has_bounded_page_schema(parsed) is False
+
+
+def test_composition_parse_html_does_not_treat_next_as_modal_dismiss_control() -> None:
+ parsed = parse_composition_html(
+ """
+ <html><body>
+ <div id="modal-root">
+ <button>Next</button>
+ <button>Export</button>
+ </div>
+ </body></html>
+ """,
+ inspected_url="https://example.com/results",
+ current_url="https://example.com/results",
+ )
+
+ assert parsed["modal_overlays"] == []
+ assert has_bounded_page_schema(parsed) is False
+
+
def test_composition_parse_html_preserves_stable_control_selectors_and_values() -> None:
parsed = parse_composition_html(
"""
@@ -288,6 +392,11 @@ def test_merge_visual_composition_evidence_keeps_screenshot_bounded_and_typed()
"challenge_location": "Below the acknowledgement checkbox and above the Search button.",
"submit_blocked": True,
"blocked_submit_controls": ["Search"],
+ "page_obstruction_detected": True,
+ "obstruction_kind": "verification_panel",
+ "obstruction_location": "Centered above the search form.",
+ "underlying_page_blocked": True,
+ "visible_dismiss_controls": ["Continue"],
"omissions": ["Result rows are not visible before verification."],
},
)
@@ -302,9 +411,39 @@ def test_merge_visual_composition_evidence_keeps_screenshot_bounded_and_typed()
)
assert merged["challenge_state"]["gates_submit_controls"] is True
assert merged["challenge_state"]["gated_submit_controls"] == [{"text": "Search", "disabled": True}]
+ assert merged["page_obstructions"] == [
+ {
+ "kind": "verification_panel",
+ "source": "vision_summary",
+ "visual_location": "Centered above the search form.",
+ "visible_controls": [{"text": "Continue"}],
+ "underlying_page_blocked": True,
+ }
+ ]
assert merged["visual_evidence_omissions"] == ["Result rows are not visible before verification."]
+def test_merge_visual_composition_evidence_keeps_false_underlying_page_blocked() -> None:
+ parsed = parse_composition_html(
+ "<html><head><title>Search</title></head><body><form><input name='q' /></form></body></html>",
+ inspected_url="https://example.com/search",
+ current_url="https://example.com/search",
+ )
+
+ merged = merge_visual_composition_evidence(
+ parsed,
+ visual_summary={
+ "summary": "A banner is visible but the search form remains usable.",
+ "page_obstruction_detected": True,
+ "obstruction_kind": "banner",
+ "obstruction_location": "Bottom of viewport.",
+ "underlying_page_blocked": False,
+ },
+ )
+
+ assert merged["page_obstructions"][0]["underlying_page_blocked"] is False
+
+
def test_composition_parse_html_surfaces_human_verification_controls_after_long_page_preamble() -> None:
parsed = parse_composition_html(
f"""
@@ -1269,6 +1408,27 @@ def test_composition_gate_rejects_click_reached_blocks_reusing_entrypoint_observ
assert "add_first_result (action)" in error
+def test_composition_gate_allows_current_page_read_after_matching_interaction_reached_page() -> None:
+ workflow_yaml = _yaml(
+ {"block_type": "goto_url", "label": "open_home", "url": "https://example.com/"},
+ {"block_type": "action", "label": "search_product", "navigation_goal": "Search for the product."},
+ {"block_type": "action", "label": "add_first_result", "navigation_goal": "Add the first result to the cart."},
+ )
+ ctx = _Ctx(
+ flow_evidence=[
+ _flow_entry("https://example.com/", reached_via="navigate", step=0),
+ _flow_entry("https://example.com/results?s=1", reached_via="interaction", step=1),
+ _flow_entry("https://example.com/results?s=1", reached_via="current_page", step=2),
+ ],
+ block_observation_refs={
+ "search_product": 0,
+ "add_first_result": 2,
+ },
+ )
+
+ assert composition_page_evidence_error(ctx, workflow_yaml) is None
+
+
def test_composition_gate_reports_missing_referenced_observation_step() -> None:
workflow_yaml = _yaml(
{"block_type": "goto_url", "label": "open_home", "url": "https://example.com/"},
diff --git a/tests/unit/test_copilot_discovery.py b/tests/unit/test_copilot_discovery.py
index a4283a504..1f16c0a96 100644
--- a/tests/unit/test_copilot_discovery.py
+++ b/tests/unit/test_copilot_discovery.py
@@ -7,6 +7,7 @@ from typing import Any
import pytest
from skyvern.forge.sdk.copilot import tools as tools_module
+from skyvern.forge.sdk.copilot.runtime import PendingBrowserInteractionObservation
from skyvern.forge.sdk.copilot.tools import _discovery_walk, _inspect_page_for_composition_impl
from skyvern.forge.sdk.copilot.verification_evidence import WorkflowVerificationEvidence
@@ -18,6 +19,9 @@ class _Ctx:
self.discovery_step_count = 0
self.prior_page_inspection_calls_made = 0
self.page_inspection_calls_this_turn = 0
+ self.flow_evidence: list[dict[str, Any]] = []
+ self.composition_page_evidence = None
+ self.pending_browser_interaction_observation = None
self.workflow_verification_evidence = WorkflowVerificationEvidence()
@@ -179,6 +183,27 @@ class _CurrentPageServer:
raise AssertionError(f"unexpected tool: {tool_name}")
+class _TargetThenCurrentPageServer:
+ def __init__(self) -> None:
+ self.calls: list[tuple[str, dict[str, Any]]] = []
+ self.current_url = ""
+
+ async def call_internal_tool(self, tool_name: str, arguments: dict[str, Any]) -> dict[str, Any]:
+ self.calls.append((tool_name, arguments))
+ if tool_name == "skyvern_navigate":
+ self.current_url = arguments["url"]
+ return {"ok": True, "data": {"url": self.current_url}}
+ if tool_name == "skyvern_get_html":
+ assert arguments == {"selector": "body"}
+ return {
+ "ok": True,
+ "data": {
+ "html": "<html><body><form><input name='firstName'><button>Search</button></form></body></html>"
+ },
+ }
+ raise AssertionError(f"unexpected tool: {tool_name}")
+
+
@pytest.mark.asyncio
async def test_discovery_navigation_failure_falls_back_to_entry_url() -> None:
result = await _discovery_walk(
@@ -321,6 +346,94 @@ async def test_post_run_current_page_inspection_budget_bypass_does_not_consume_c
assert ctx.post_run_current_page_inspection_workflow_run_id == "wr_123" # type: ignore[attr-defined]
+async def test_current_page_inspection_without_earned_interaction_is_not_click_reached(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ server = _CurrentPageServer()
+ ctx = _Ctx(server)
+
+ async def fake_fallback_page_info(_ctx: object) -> tuple[str, str]:
+ return "https://www.example.com/results", "Results"
+
+ monkeypatch.setattr(tools_module, "_fallback_page_info", fake_fallback_page_info)
+
+ result = await _inspect_page_for_composition_impl(ctx, "current_page")
+
+ assert result["ok"] is True
+ assert result["reached_via"] == "current_page"
+ assert ctx.flow_evidence[0]["reached_via"] == "current_page"
+
+
+async def test_current_page_inspection_after_browser_action_is_click_reached_once(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ server = _CurrentPageServer()
+ ctx = _Ctx(server)
+ ctx.pending_browser_interaction_observation = PendingBrowserInteractionObservation(
+ tool_name="click",
+ url="https://www.example.com/results",
+ )
+
+ async def fake_fallback_page_info(_ctx: object) -> tuple[str, str]:
+ return "https://www.example.com/results", "Results"
+
+ monkeypatch.setattr(tools_module, "_fallback_page_info", fake_fallback_page_info)
+
+ result = await _inspect_page_for_composition_impl(ctx, "current_page")
+
+ assert result["ok"] is True
+ assert result["reached_via"] == "interaction"
+ assert ctx.flow_evidence[0]["reached_via"] == "interaction"
+ assert ctx.pending_browser_interaction_observation is None
+
+
+async def test_inspection_budget_steers_progress_check_instead_of_authoring() -> None:
+ ctx = _Ctx(server=object())
+ ctx.page_inspection_calls_this_turn = 999
+
+ result = await _inspect_page_for_composition_impl(ctx, "current_page")
+
+ assert result["ok"] is False
+ assert "not evidence that scouting is complete" in result["error"]
+ assert "evaluate" in result["error"]
+ assert "get_browser_screenshot" in result["error"]
+ assert "browser action on the current page" in result["error"]
+ assert "Do not author downstream result" in result["error"]
+ assert "Compose from existing evidence" not in result["error"]
+
+
+async def test_target_url_inspection_clears_pending_interaction_credit(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ server = _TargetThenCurrentPageServer()
+ ctx = _Ctx(server)
+ ctx.pending_browser_interaction_observation = PendingBrowserInteractionObservation(
+ tool_name="click",
+ url="https://www.example.com/results",
+ )
+
+ target_result = await _inspect_page_for_composition_impl(ctx, "https://www.example.com/results")
+
+ assert target_result["ok"] is True
+ assert target_result["reached_via"] == "navigate"
+ assert ctx.pending_browser_interaction_observation is None
+
+ async def fake_fallback_page_info(_ctx: object) -> tuple[str, str]:
+ return "https://www.example.com/results", "Results"
+
+ monkeypatch.setattr(tools_module, "_fallback_page_info", fake_fallback_page_info)
+
+ current_result = await _inspect_page_for_composition_impl(ctx, "current_page")
+
+ assert current_result["ok"] is True
+ assert current_result["reached_via"] == "current_page"
+ assert [entry["reached_via"] for entry in ctx.flow_evidence] == ["navigate", "current_page"]
+
+
class _SizeCappedHtmlStrippedFallbackServer:
"""Every page's get_html is dropped by the MCP size cap (a heavy DOM exceeds it).
The stripped-body evaluate fallback recovers each page, so the resolver can still
diff --git a/tests/unit/test_copilot_evaluate_page_evidence.py b/tests/unit/test_copilot_evaluate_page_evidence.py
new file mode 100644
index 000000000..003028851
--- /dev/null
+++ b/tests/unit/test_copilot_evaluate_page_evidence.py
@@ -0,0 +1,149 @@
+from __future__ import annotations
+
+from unittest.mock import MagicMock
+
+import pytest
+
+from skyvern.forge.sdk.copilot.build_phase import BuildPhase
+from skyvern.forge.sdk.copilot.context import CopilotContext
+from skyvern.forge.sdk.copilot.request_policy import RequestPolicy
+from skyvern.forge.sdk.copilot.tools import (
+ _evaluate_post_hook,
+ _inspect_page_for_composition_impl,
+ _mark_pending_browser_interaction_observation,
+)
+from skyvern.forge.sdk.copilot.turn_intent import TurnIntent, TurnIntentAuthority, TurnIntentMode
+
+
+def _ctx() -> CopilotContext:
+ return CopilotContext(
+ organization_id="org-1",
+ workflow_id="wf-1",
+ workflow_permanent_id="wfp-1",
+ workflow_yaml="",
+ browser_session_id=None,
+ stream=MagicMock(),
+ request_policy=RequestPolicy(),
+ build_phase=BuildPhase.COMPOSING,
+ turn_intent=TurnIntent(
+ mode=TurnIntentMode.BUILD,
+ authority=TurnIntentAuthority(may_update_workflow=True, may_run_blocks=True),
+ ),
+ )
+
+
+async def test_evaluate_nested_rows_records_interaction_observation_step() -> None:
+ ctx = _ctx()
+ _mark_pending_browser_interaction_observation(
+ ctx,
+ tool_name="click",
+ url="https://example.test/search/results?s=1",
+ )
+
+ result = {
+ "ok": True,
+ "data": {
+ "url": "https://example.test/search/results?s=1",
+ "title": "Search Results",
+ "result": {
+ "rows": [
+ {
+ "cells": [
+ {"text": "Example Person"},
+ {"text": "Credential A"},
+ {"text": "Number 123"},
+ {"text": "Expiration 2030-01-01"},
+ ]
+ }
+ ]
+ },
+ },
+ }
+
+ updated = await _evaluate_post_hook(result, raw={}, ctx=ctx)
+
+ assert updated["observation_step"] == 0
+ assert updated["data"]["observation_step"] == 0
+ assert len(ctx.flow_evidence) == 1
+ assert ctx.flow_evidence[0]["reached_via"] == "interaction"
+ assert ctx.flow_evidence[0]["had_bounded_schema"] is True
+ assert ctx.flow_evidence[0]["step"] == 0
+ evidence = ctx.flow_evidence[0]["evidence"]
+ assert evidence["source_tool"] == "evaluate"
+ assert evidence["current_url"] == "https://example.test/search/results?s=1"
+ assert evidence["result_containers"][0]["row_count"] == 1
+ assert "Credential A" in evidence["result_containers"][0]["sample_rows"][0]
+ assert ctx.composition_page_evidence is evidence
+
+
+async def test_evaluate_turnstile_key_records_challenge_observation_step() -> None:
+ ctx = _ctx()
+ _mark_pending_browser_interaction_observation(
+ ctx,
+ tool_name="type_text",
+ url="https://example.test/certificant-search",
+ )
+
+ result = {
+ "ok": True,
+ "data": {
+ "url": "https://example.test/certificant-search",
+ "title": "Certificant Search",
+ "text": "Verify you are human before searching.",
+ "turnstile": True,
+ "btnDisabled": True,
+ },
+ }
+
+ updated = await _evaluate_post_hook(result, raw={}, ctx=ctx)
+
+ assert updated["observation_step"] == 0
+ assert len(ctx.flow_evidence) == 1
+ evidence = ctx.flow_evidence[0]["evidence"]
+ assert evidence["source_tool"] == "evaluate"
+ assert evidence["challenge_state"]["detected"] is True
+ assert evidence["challenge_state"]["kind"] == "captcha"
+ assert evidence["challenge_state"]["gates_submit_controls"] is True
+ assert evidence["challenge_state"]["gated_submit_controls"][0]["disabled"] is True
+ assert "turnstile" in evidence["anti_bot_indicators"]
+ assert ctx.composition_page_evidence is evidence
+
+
+async def test_target_url_inspection_does_not_navigate_away_from_interaction_evidence(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ ctx = _ctx()
+ ctx.flow_evidence.append(
+ {
+ "evidence": {
+ "source_tool": "evaluate",
+ "current_url": "https://example.test/search/results?s=1",
+ "inspected_url": "https://example.test/search/results?s=1",
+ "forms": [],
+ "navigation_targets": [],
+ "result_containers": [{"tag": "table", "selector": "#results"}],
+ "challenge_controls": [],
+ },
+ "reached_via": "interaction",
+ "had_bounded_schema": True,
+ "step": 4,
+ }
+ )
+
+ async def unexpected_navigate(*_: object, **__: object) -> dict[str, object]:
+ raise AssertionError("target_url inspection should not navigate away from reached evidence")
+
+ monkeypatch.setattr("skyvern.forge.sdk.copilot.tools._discovery_navigate", unexpected_navigate)
+
+ result = await _inspect_page_for_composition_impl(ctx, "https://example.test/")
+
+ assert result["ok"] is False
+ assert result["data"] == {
+ "current_url": "https://example.test/search/results?s=1",
+ "observation_step": 4,
+ }
+ assert 'target_url="current_page"' in result["error"]
+ assert "observation_step 4" in result["error"]
diff --git a/tests/unit/test_copilot_frontier.py b/tests/unit/test_copilot_frontier.py
index 4dbf8ec31..8ae578fca 100644
--- a/tests/unit/test_copilot_frontier.py
+++ b/tests/unit/test_copilot_frontier.py
@@ -543,6 +543,27 @@ def test_frontier_run_size_error_limits_long_page_changing_frontier() -> None:
assert "Do not remove later blocks" in error
+def test_frontier_run_size_result_steers_to_smaller_saved_frontier() -> None:
+ result = tools._frontier_run_size_result(
+ "frontier too long",
+ ["open", "set_search", "submit_search", "expand_results", "extract"],
+ ["open", "set_search", "submit_search", "expand_results", "extract"],
+ )
+
+ data = result["data"]
+ assert result["ok"] is False
+ assert data["workflow_run_skipped"] is True
+ assert data["suggested_block_labels"] == ["open", "set_search"]
+ assert data["deferred_block_labels"] == ["submit_search", "expand_results", "extract"]
+ assert data["control_signal"] == {
+ "kind": "intermediate_success",
+ "user_facing_summary": data["user_facing_summary"],
+ "next_tool": "run_blocks_and_collect_debug",
+ "next_block_labels": ["open", "set_search"],
+ "preserve_workflow_yaml": True,
+ }
+
+
def test_frontier_run_size_error_allows_tool_expanded_runtime_anchor() -> None:
definition = _FakeDefinition(
[
diff --git a/tests/unit/test_copilot_hooks.py b/tests/unit/test_copilot_hooks.py
index 4d52e8400..930907a5d 100644
--- a/tests/unit/test_copilot_hooks.py
+++ b/tests/unit/test_copilot_hooks.py
@@ -714,3 +714,69 @@ class TestVerifyScoutTypeLanded:
assert result is None
ctx.discovery_mcp_server.call_internal_tool.assert_not_awaited()
+
+
+class TestBrowserInteractionObservationHooks:
+ @pytest.mark.asyncio
+ async def test_click_hook_marks_pending_interaction_observation(self) -> None:
+ from skyvern.forge.sdk.copilot.tools import _click_post_hook
+
+ ctx = SimpleNamespace(pending_browser_interaction_observation=None, discovery_mcp_server=None)
+ result = await _click_post_hook(
+ {"ok": True, "data": {"selector": "#add-to-cart"}},
+ {"browser_context": {"url": "https://example.com/results", "title": "Results"}},
+ ctx,
+ )
+
+ assert result["data"] == {
+ "selector": "#add-to-cart",
+ "url": "https://example.com/results",
+ "title": "Results",
+ }
+ assert ctx.pending_browser_interaction_observation is not None
+ assert ctx.pending_browser_interaction_observation.tool_name == "click"
+ assert ctx.pending_browser_interaction_observation.url == "https://example.com/results"
+
+ @pytest.mark.asyncio
+ async def test_failed_click_hook_clears_stale_pending_interaction_observation(self) -> None:
+ from skyvern.forge.sdk.copilot.runtime import PendingBrowserInteractionObservation
+ from skyvern.forge.sdk.copilot.tools import _click_post_hook
+
+ ctx = SimpleNamespace(
+ pending_browser_interaction_observation=PendingBrowserInteractionObservation(
+ tool_name="click",
+ url="https://example.com/results",
+ ),
+ discovery_mcp_server=None,
+ )
+
+ result = await _click_post_hook(
+ {"ok": False, "error": "element not found"},
+ {"browser_context": {"url": "https://example.com/results", "title": "Results"}},
+ ctx,
+ )
+
+ assert result == {"ok": False, "error": "element not found"}
+ assert ctx.pending_browser_interaction_observation is None
+
+ @pytest.mark.asyncio
+ async def test_type_hook_does_not_mark_pending_interaction_when_readback_fails(
+ self,
+ monkeypatch: pytest.MonkeyPatch,
+ ) -> None:
+ from skyvern.forge.sdk.copilot import tools as tools_module
+
+ async def fake_verify(*_args: object, **_kwargs: object) -> dict[str, object]:
+ return {"ok": False, "error": "field is still empty"}
+
+ monkeypatch.setattr(tools_module, "_verify_scout_type_landed", fake_verify)
+ ctx = SimpleNamespace(pending_browser_interaction_observation=None, discovery_mcp_server=None)
+
+ result = await tools_module._type_text_post_hook(
+ {"ok": True, "data": {"selector": "#q", "text_length": 12}},
+ {"browser_context": {"url": "https://example.com/search", "title": "Search"}},
+ ctx,
+ )
+
+ assert result == {"ok": False, "error": "field is still empty"}
+ assert ctx.pending_browser_interaction_observation is None
diff --git a/tests/unit/test_copilot_output_policy.py b/tests/unit/test_copilot_output_policy.py
index dd39c23d6..1d62a346a 100644
--- a/tests/unit/test_copilot_output_policy.py
+++ b/tests/unit/test_copilot_output_policy.py
@@ -1204,6 +1204,48 @@ workflow_definition:
assert sanitized_tool_names == ["update_and_run_blocks"]
+async def test_update_workflow_runs_composition_evidence_precheck_before_saving(monkeypatch) -> None:
+ from skyvern.forge.sdk.copilot import tools as tools_module
+
+ workflow_yaml = """
+workflow_definition:
+ parameters: []
+ blocks:
+ - block_type: goto_url
+ label: open_lookup
+ url: https://example.com/lookup
+ - block_type: navigation
+ label: search_lookup
+ navigation_goal: Enter the observed field and submit.
+ - block_type: action
+ label: expand_first_result
+ navigation_goal: Expand the first result row.
+"""
+ ctx = _ctx(
+ build_phase=BuildPhase.COMPOSING,
+ turn_intent=TurnIntent(mode=TurnIntentMode.BUILD),
+ request_policy=RequestPolicy(),
+ )
+
+ async def unexpected_update_workflow(*args: object, **kwargs: object) -> object:
+ raise AssertionError("composition evidence precheck must run before update_workflow saves")
+
+ monkeypatch.setattr(tools_module, "_tool_loop_error", lambda *args, **kwargs: None)
+ monkeypatch.setattr(tools_module, "_update_workflow", unexpected_update_workflow)
+ monkeypatch.setattr(tools_module, "_record_diagnosis_repair_contract", lambda *args, **kwargs: None)
+
+ result = await tools_module.update_workflow_tool.on_invoke_tool(
+ SimpleNamespace(context=ctx, tool_name="update_workflow"),
+ json.dumps({"workflow_yaml": workflow_yaml}),
+ )
+
+ payload = json.loads(result)
+ assert payload["ok"] is False
+ assert "page-dependent build blocks need observed page evidence" in payload["error"]
+ assert "https://example.com/lookup" in payload["error"]
+
+
@pytest.mark.asyncio
async def test_update_and_run_blocks_precheck_uses_proposed_block_observation_refs(monkeypatch) -> None:
from skyvern.forge.sdk.copilot import tools as tools_module
diff --git a/tests/unit/test_copilot_session_injection.py b/tests/unit/test_copilot_session_injection.py
index 80fdb4156..f2e52e8b1 100644
--- a/tests/unit/test_copilot_session_injection.py
+++ b/tests/unit/test_copilot_session_injection.py
@@ -3,6 +3,7 @@
from __future__ import annotations
import asyncio
+import time
from typing import Any
from unittest.mock import AsyncMock, MagicMock
@@ -12,6 +13,7 @@ from skyvern.cli.core import session_manager
from skyvern.cli.core.result import BrowserContext as MCPBrowserContext
from skyvern.cli.core.session_manager import SessionState, scoped_session
from skyvern.forge.sdk.copilot.runtime import AgentContext, mcp_to_copilot
+from skyvern.forge.sdk.copilot.tools import _same_page_ignoring_fragment
@pytest.fixture(autouse=True)
@@ -41,6 +43,12 @@ def _make_ctx(**overrides: Any) -> AgentContext:
return AgentContext(**defaults)
+def test_copilot_same_page_ignoring_fragment_matches_trailing_slash_variants() -> None:
+ assert _same_page_ignoring_fragment("https://example.test/results#section", "https://example.test/results") is True
+ assert _same_page_ignoring_fragment("https://example.test/results/", "https://example.test/results") is True
+ assert _same_page_ignoring_fragment("https://example.test/results?page=2", "https://example.test/results") is False
+
+
@pytest.mark.asyncio
async def test_scoped_session_pushes_and_restores() -> None:
"""scoped_session sets the ContextVar within scope and restores on exit."""
@@ -278,6 +286,41 @@ class TestScreenshotAdapter:
assert ctx.composition_page_evidence["source_tool"] == "get_browser_screenshot"
assert ctx.composition_page_evidence["current_url"] == "https://example.com"
+ @pytest.mark.asyncio
+ async def test_screenshot_post_hook_does_not_verify_from_url_title_only(
+ self,
+ monkeypatch: pytest.MonkeyPatch,
+ ) -> None:
+ from skyvern.forge.sdk.copilot.request_policy import CompletionCriterion, RequestPolicy
+ from skyvern.forge.sdk.copilot.tools import _screenshot_post_hook
+
+ async def handler(**_: object) -> dict[str, object]:
+ raise AssertionError("screenshot-only observation must not invoke completion verification")
+
+ async def handler_lookup(_: object) -> object:
+ return handler
+
+ monkeypatch.setattr("skyvern.forge.sdk.copilot.tools._completion_verification_handler", handler_lookup)
+ ctx = _make_ctx(
+ request_policy=RequestPolicy(
+ completion_criteria=[CompletionCriterion(id="c0", outcome="the requested item is visible")]
+ ),
+ last_test_ok=False,
+ last_run_blocks_workflow_run_id="wr_failed",
+ copilot_run_start_monotonic=time.monotonic(),
+ )
+ raw = {"browser_context": {"url": "https://example.com/results", "title": "Results"}}
+ result = {
+ "ok": True,
+ "data": {"data": "iVBOR...", "mime": "image/png", "bytes": 1234},
+ }
+
+ adapted = await _screenshot_post_hook(result, raw, ctx)
+
+ assert adapted["data"]["url"] == "https://example.com/results"
+ assert ctx.post_run_page_observation_after_failed_test is True
+ assert ctx.completion_verification_result is None
+
class TestNavigateAdapter:
@pytest.mark.asyncio
@@ -369,7 +412,15 @@ class TestEvaluateAdapter:
assert adapted["data"]["url"] == "https://example.com/results"
assert ctx.composition_page_evidence["source_tool"] == "evaluate"
assert ctx.composition_page_evidence["current_url"] == "https://example.com/results"
- assert ctx.composition_page_evidence["result_containers"] == []
+ assert ctx.composition_page_evidence["result_containers"] == [
+ {
+ "tag": "table",
+ "id": "",
+ "selector": "",
+ "row_count": 1,
+ "sample_rows": ["Test User"],
+ }
+ ]
@pytest.mark.asyncio
async def test_evaluate_post_hook_lifts_bounded_page_schema_from_mcp_observation(self) -> None:
diff --git a/tests/unit/test_copilot_turn_intent.py b/tests/unit/test_copilot_turn_intent.py
index 1ae017280..a018cd6f1 100644
--- a/tests/unit/test_copilot_turn_intent.py
+++ b/tests/unit/test_copilot_turn_intent.py
@@ -321,6 +321,20 @@ def test_build_turn_intent_marks_run_context_for_diagnose() -> None:
assert RequiredContextKey.LATEST_RUN_RESULT in intent.required_context
+def test_build_turn_intent_does_not_require_unassembled_browser_state() -> None:
+ intent = build_turn_intent(
+ user_message="Continue in the open browser",
+ workflow_yaml="workflow_definition:\n blocks: []",
+ chat_history=[],
+ global_llm_context="",
+ request_policy=RequestPolicy(),
+ browser_session_id="pbs_123",
+ )
+
+ assert RequiredContextKey.BROWSER_STATE not in intent.required_context
+ assert TurnIntentReasonCode.BROWSER_CONTEXT_PRESENT in intent.reason_codes
+
+
def test_build_turn_intent_keeps_explicit_fix_as_edit() -> None:
intent = build_turn_intent(
user_message="Fix the error after login.",
diff --git a/tests/unit/test_real_browser_state_navigate.py b/tests/unit/test_real_browser_state_navigate.py
index 965222962..ff4cf1d4a 100644
--- a/tests/unit/test_real_browser_state_navigate.py
+++ b/tests/unit/test_real_browser_state_navigate.py
@@ -10,7 +10,7 @@ import pytest
from playwright.async_api import TimeoutError as PlaywrightTimeoutError
from skyvern.exceptions import FailedToNavigateToUrl
-from skyvern.webeye.real_browser_state import RealBrowserState
+from skyvern.webeye.real_browser_state import RealBrowserState, _same_page_ignoring_fragment
@pytest.fixture
@@ -20,6 +20,12 @@ def browser_state() -> RealBrowserState:
return state
+def test_same_page_ignoring_fragment_matches_fragment_only_differences() -> None:
+ assert _same_page_ignoring_fragment("https://example.test/results#section", "https://example.test/results") is True
+ assert _same_page_ignoring_fragment("https://example.test/results/", "https://example.test/results") is True
+ assert _same_page_ignoring_fragment("https://example.test/results?page=2", "https://example.test/results") is False
+
+
@pytest.mark.asyncio
async def test_navigate_to_url_progresses_from_load_to_domcontentloaded(
browser_state: RealBrowserState,