Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
SKY-10346: Add page evidence parser contract (PR 1/12) (#6233)
Andrew Neilson330c27ec1ea5
diff --git a/skyvern/forge/sdk/copilot/composition_evidence.py b/skyvern/forge/sdk/copilot/composition_evidence.pynew file mode 100644index 000000000..4f0f2e44c--- /dev/null+++ b/skyvern/forge/sdk/copilot/composition_evidence.py@@ -0,0 +1,690 @@+"""Build-time page evidence contract for Workflow Copilot composition."""++from __future__ import annotations++from typing import Any+from urllib.parse import urljoin, urlparse++import yaml++try:+ from bs4 import BeautifulSoup # type: ignore[import-not-found]+except ImportError: # pragma: no cover - bs4 is a transitive dep but inspection degrades gracefully.+ BeautifulSoup = None # type: ignore[assignment, misc]++from skyvern.forge.sdk.copilot.build_phase import BuildPhase+from skyvern.utils.yaml_loader import safe_load_no_dates++_BUILD_MODE_VALUES: frozenset[str] = frozenset({"build", "draft_only", "edit", "unknown"})+_PAGE_SHAPING_BLOCK_TYPES: frozenset[str] = frozenset({"navigation", "login"})+_RESULT_CONTAINER_HINTS: frozenset[str] = frozenset({"result", "results", "record", "records", "row", "rows"})+_MAX_FORMS = 5+_MAX_FIELDS_PER_FORM = 20+_MAX_RESULT_CONTAINERS = 8+_MAX_NAVIGATION_TARGETS = 20+_MAX_SELECT_OPTIONS = 30+_MAX_CHALLENGE_CONTROLS = 8+DOM_EVIDENCE_SOURCE = "dom_html"+SCREENSHOT_EVIDENCE_SOURCE = "screenshot"+VISION_EVIDENCE_SOURCE = "vision_summary"+_ANTI_BOT_PATTERNS = (+ "just a moment",+ "captcha",+ "challenge",+ "human-verification",+ "human verification",+ "verify you are human",+ "access denied",+ "are you a robot",+)+_MAX_VISUAL_SUMMARY_CHARS = 500+_MAX_VISUAL_OMISSIONS = 5+++def _bounded_string(value: Any, max_chars: int) -> str:+ if not isinstance(value, str):+ return ""+ # Bounded evidence text is for Copilot-readable summaries; selectors are+ # built separately so whitespace-sensitive selector values are preserved.+ return " ".join(value.split())[:max_chars]+++def _challenge_kind(indicators: list[str]) -> str:+ indicator_text = " ".join(indicators).lower()+ if "captcha" in indicator_text or "are you a robot" in indicator_text:+ return "captcha"+ if "access denied" in indicator_text:+ return "access_denied"+ if any(term in indicator_text for term in ("challenge", "human verification", "verify you are human")):+ return "human_verification"+ return "unknown" if indicators else "none"+++def _challenge_state(+ indicators: list[str],+ *,+ source: str = DOM_EVIDENCE_SOURCE,+ gated_submit_controls: list[dict[str, Any]] | None = None,+) -> dict[str, Any]:+ detected = bool(indicators)+ gated_controls = gated_submit_controls or []+ return {+ "detected": detected,+ "kind": _challenge_kind(indicators),+ "source": source if detected else "",+ "indicators": indicators[:8],+ "requires_human_verification": detected,+ "visual_location": "",+ "gates_submit_controls": bool(detected and gated_controls),+ "gated_submit_controls": gated_controls[:5] if detected else [],+ }+++def _control_disabled(node: Any) -> bool:+ if not hasattr(node, "has_attr"):+ return False+ return bool(+ node.has_attr("disabled")+ or str(node.get("aria-disabled") or "").strip().lower() == "true"+ or str(node.get("data-disabled") or "").strip().lower() == "true"+ )+++def _gated_submit_controls(forms: list[dict[str, Any]]) -> list[dict[str, Any]]:+ controls: list[dict[str, Any]] = []+ for form in forms:+ if not isinstance(form, dict):+ continue+ for control in form.get("submit_controls") or []:+ if not isinstance(control, dict) or control.get("disabled") is not True:+ continue+ controls.append(+ {+ "text": _bounded_string(control.get("text") or control.get("value"), 120),+ "id": _bounded_string(control.get("id"), 120),+ "name": _bounded_string(control.get("name"), 120),+ "selector": _bounded_string(control.get("selector"), 160),+ "disabled": True,+ }+ )+ return controls[:5]+++def _evidence_metadata(+ indicators: list[str] | None = None,+ *,+ forms: list[dict[str, Any]] | None = None,+) -> dict[str, Any]:+ gated_controls = _gated_submit_controls(forms or [])+ return {+ "evidence_sources": [DOM_EVIDENCE_SOURCE],+ "screenshot_used": False,+ "visual_evidence_summary": "",+ "visual_evidence_omissions": [],+ "inspection_warnings": [],+ "challenge_state": _challenge_state(+ indicators or [],+ gated_submit_controls=gated_controls,+ ),+ }+++def page_evidence_needs_visual_fallback(evidence: dict[str, Any]) -> bool:+ """Return True when DOM evidence should be augmented with visual evidence."""++ challenge_state = evidence.get("challenge_state")+ if isinstance(challenge_state, dict) and challenge_state.get("detected") is True:+ return True+ return bool(evidence.get("anti_bot_indicators") or evidence.get("challenge_controls"))+++def merge_visual_composition_evidence(+ evidence: dict[str, Any],+ *,+ visual_summary: dict[str, Any] | None = None,+ visual_error: str | None = None,+) -> dict[str, Any]:+ """Add bounded screenshot/vision metadata to composition evidence.++ The raw screenshot stays tool-internal. This helper records only compact,+ typed facts that the main Copilot loop may use while composing.+ """++ merged = dict(evidence)+ sources = [str(source) for source in merged.get("evidence_sources") or [] if isinstance(source, str)]+ if SCREENSHOT_EVIDENCE_SOURCE not in sources:+ sources.append(SCREENSHOT_EVIDENCE_SOURCE)+ if visual_summary and VISION_EVIDENCE_SOURCE not in sources:+ sources.append(VISION_EVIDENCE_SOURCE)+ merged["evidence_sources"] = sources+ merged["screenshot_used"] = True++ omissions = [+ _bounded_string(item, 160)+ for item in (merged.get("visual_evidence_omissions") or [])+ if _bounded_string(item, 160)+ ][:_MAX_VISUAL_OMISSIONS]+ if visual_error:+ omissions.append(_bounded_string(f"visual_summary_error: {visual_error}", 160))++ if isinstance(visual_summary, dict):+ summary = _bounded_string(visual_summary.get("summary"), _MAX_VISUAL_SUMMARY_CHARS)+ if summary:+ merged["visual_evidence_summary"] = summary+ for item in visual_summary.get("omissions") or []:+ bounded = _bounded_string(item, 160)+ if bounded:+ omissions.append(bounded)+ challenge_state = dict(merged.get("challenge_state") or {})+ if visual_summary.get("challenge_detected") is True:+ challenge_state["detected"] = True+ challenge_state["requires_human_verification"] = True+ challenge_state["source"] = (+ "dom+screenshot" if challenge_state.get("source") else SCREENSHOT_EVIDENCE_SOURCE+ )+ challenge_kind = _bounded_string(visual_summary.get("challenge_kind"), 80)+ if challenge_kind:+ challenge_state["kind"] = challenge_kind+ challenge_location = _bounded_string(visual_summary.get("challenge_location"), 180)+ if challenge_location:+ challenge_state["visual_location"] = challenge_location+ if visual_summary.get("submit_blocked") is True:+ challenge_state["gates_submit_controls"] = True+ visual_blocked_controls = [+ {+ "text": _bounded_string(item, 120),+ "disabled": True,+ }+ for item in visual_summary.get("blocked_submit_controls") or []+ if _bounded_string(item, 120)+ ]+ if visual_blocked_controls:+ existing_controls = [+ item for item in challenge_state.get("gated_submit_controls") or [] if isinstance(item, dict)+ ]+ challenge_state["gated_submit_controls"] = (existing_controls + visual_blocked_controls)[:5]+ merged["challenge_state"] = challenge_state+ elif not merged.get("visual_evidence_summary"):+ merged["visual_evidence_summary"] = "Screenshot captured because DOM evidence indicated challenge state."++ merged["visual_evidence_omissions"] = list(dict.fromkeys(omissions))[:_MAX_VISUAL_OMISSIONS]+ return merged+++def _parse_workflow_blocks(workflow_yaml: str | None) -> list[dict[str, Any]]:+ if not workflow_yaml:+ return []+ try:+ parsed = safe_load_no_dates(workflow_yaml)+ except yaml.YAMLError:+ return []+ if not isinstance(parsed, dict):+ return []+ definition = parsed.get("workflow_definition")+ if not isinstance(definition, dict):+ return []+ blocks = definition.get("blocks")+ return [block for block in blocks if isinstance(block, dict)] if isinstance(blocks, list) else []+++def workflow_target_url(workflow_yaml: str | None) -> str | None:+ for block in _parse_workflow_blocks(workflow_yaml):+ url = block.get("url")+ if isinstance(url, str) and url.strip():+ return url.strip()+ return None+++def _mode_requires_evidence(ctx: Any) -> bool:+ mode_value = getattr(getattr(getattr(ctx, "turn_intent", None), "mode", None), "value", None)+ phase = getattr(ctx, "build_phase", None)+ return (+ isinstance(mode_value, str)+ and mode_value in _BUILD_MODE_VALUES+ and phase+ in (+ BuildPhase.COMPOSING,+ BuildPhase.TESTING,+ )+ )+++def _new_page_shaping_blocks(workflow_yaml: str | None, previous_workflow_yaml: str | None) -> list[dict[str, str]]:+ previous = {+ (str(block.get("label") or ""), str(block.get("block_type") or "").strip().lower())+ for block in _parse_workflow_blocks(previous_workflow_yaml)+ }+ blocks: list[dict[str, str]] = []+ for block in _parse_workflow_blocks(workflow_yaml):+ block_type = str(block.get("block_type") or "").strip().lower()+ if block_type not in _PAGE_SHAPING_BLOCK_TYPES:+ continue+ label = str(block.get("label") or "<missing label>")+ if (label, block_type) in previous:+ continue+ blocks.append({"label": label, "block_type": block_type})+ return blocks+++def _format_page_shaping_findings(blocks: list[dict[str, str]]) -> str:+ return ", ".join(f"{block['label']} ({block['block_type']})" for block in blocks[:5])+++def _same_page(left: str | None, right: str | None) -> bool:+ if not left or not right:+ return False+ try:+ left_parsed = urlparse(left)+ right_parsed = urlparse(right)+ except Exception:+ return False+ if not left_parsed.netloc or not right_parsed.netloc:+ return False+ if left_parsed.netloc.lower() != right_parsed.netloc.lower():+ return False+ left_path = (left_parsed.path or "/").rstrip("/") or "/"+ right_path = (right_parsed.path or "/").rstrip("/") or "/"+ return left_path == right_path+++def _same_origin(left: str | None, right: str | None) -> bool:+ if not left or not right:+ return False+ try:+ left_parsed = urlparse(left)+ right_parsed = urlparse(right)+ except Exception:+ return False+ if not left_parsed.netloc or not right_parsed.netloc:+ return False+ return left_parsed.netloc.lower() == right_parsed.netloc.lower()+++def _evidence_matches_target(evidence: dict[str, Any] | None, target_url: str | None) -> bool:+ if not evidence or not target_url:+ return False+ if evidence.get("source_tool") != "inspect_page_for_composition":+ return False+ current_url = evidence.get("current_url")+ inspected_url = evidence.get("inspected_url")+ return _same_page(current_url if isinstance(current_url, str) else None, target_url) or _same_page(+ inspected_url if isinstance(inspected_url, str) else None,+ target_url,+ )+++def composition_page_evidence_error(ctx: Any, workflow_yaml: str | None) -> str | None:+ """Return a mutation error when a build adds page-dependent blocks before inspection.++ This is deliberately structural rather than semantic: the contract says+ page-shaping workflow blocks need inspected page evidence first. Grounding+ quality is measured by evals and the page-evidence tool output, not by a+ growing natural-language classifier in the mutation path.+ """++ if not _mode_requires_evidence(ctx):+ return None+ page_shaping_blocks = _new_page_shaping_blocks(workflow_yaml, getattr(ctx, "workflow_yaml", None))+ if not page_shaping_blocks:+ return None++ target_url = workflow_target_url(workflow_yaml)+ evidence = getattr(ctx, "composition_page_evidence", None)+ if _evidence_matches_target(evidence, target_url):+ return None+ offending = _format_page_shaping_findings(page_shaping_blocks)+ return (+ "Workflow validation failed: page-dependent build blocks need observed page evidence before they are "+ f"authored. Call inspect_page_for_composition(target_url={target_url!r}) before composing navigation/login "+ "blocks, or save only the initial goto_url block and inspect the reached page before the next mutation. "+ f"Offending blocks: {offending}"+ )+++def _empty_evidence(inspected_url: str, current_url: str) -> dict[str, Any]:+ return {+ "inspected_url": inspected_url,+ "current_url": current_url,+ "page_title": "",+ "forms": [],+ "navigation_targets": [],+ "result_containers": [],+ "anti_bot_indicators": [],+ "challenge_controls": [],+ "evidence_confidence": 0.0,+ "source_tool": "inspect_page_for_composition",+ **_evidence_metadata([]),+ }+++def _node_text(node: Any) -> str:+ try:+ return node.get_text(" ", strip=True)+ except Exception:+ return ""+++def _schema_text(value: str, max_chars: int) -> str:+ return " ".join((value or "").split())[:max_chars]+++def _attr_value(node: Any, key: str) -> str:+ value = node.get(key) if hasattr(node, "get") else None+ return value.strip() if isinstance(value, str) else ""+++def _classes_for(node: Any) -> list[str]:+ class_value = node.get("class") if hasattr(node, "get") else None+ if isinstance(class_value, list):+ return [str(item).strip() for item in class_value if str(item).strip()]+ if isinstance(class_value, str):+ return [part for part in class_value.split() if part]+ return []+++def _css_attr(value: str) -> str:+ return value.replace("\\", "\\\\").replace('"', '\\"')+++def _simple_css_identifier(value: str) -> bool:+ if not value:+ return False+ first = value[0]+ if not (first.isalpha() or first in {"_", "-"}):+ return False+ return all(char.isalnum() or char in {"_", "-"} for char in value[1:])+++def _class_selector(classes: list[str]) -> str:+ parts: list[str] = []+ for class_name in classes[:3]:+ if _simple_css_identifier(class_name):+ parts.append(f".{class_name}")+ else:+ parts.append(f'[class~="{_css_attr(class_name)}"]')+ return "".join(parts)+++def _selector_for(node: Any) -> str:+ tag_name = getattr(node, "name", None) or "*"+ node_id = _attr_value(node, "id")+ if node_id:+ return f"#{node_id}"+ node_name = _attr_value(node, "name")+ node_value = _attr_value(node, "value")+ if node_name and node_value:+ return f'{tag_name}[name="{_css_attr(node_name)}"][value="{_css_attr(node_value)}"]'+ classes = _classes_for(node)+ class_selector = _class_selector(classes)+ if class_selector and node_value:+ return f'{tag_name}{class_selector}[value="{_css_attr(node_value)}"]'+ if node_name:+ return f'{tag_name}[name="{_css_attr(node_name)}"]'+ href = _attr_value(node, "href")+ if tag_name == "a" and href:+ return f'a[href="{_css_attr(href)}"]'+ if class_selector:+ return f"{tag_name}{class_selector}"+ return str(tag_name)+++def _adjacent_text(field: Any) -> str:+ for siblings in (getattr(field, "next_siblings", []), getattr(field, "previous_siblings", [])):+ for index, sibling in enumerate(siblings):+ if index >= 4:+ break+ sibling_name = str(getattr(sibling, "name", "") or "").lower()+ if sibling_name in {"input", "select", "textarea", "button"}:+ # Stop at the next control so labels are not borrowed from a neighboring field.+ break+ text = _node_text(sibling) if sibling_name else str(sibling).strip()+ if text:+ return text+ return ""+++def _parent_text_label(field: Any) -> str:+ for parent_name in ("td", "th", "li", "div", "span"):+ parent = field.find_parent(parent_name) if hasattr(field, "find_parent") else None+ if parent is None:+ continue+ text = _node_text(parent)+ if 0 < len(text) <= 240:+ return text+ return ""+++def _select_options(node: Any) -> list[dict[str, Any]]:+ options: list[dict[str, Any]] = []+ for option in node.find_all("option")[:_MAX_SELECT_OPTIONS]:+ options.append(+ {+ "text": _node_text(option)[:120],+ "value": _attr_value(option, "value")[:160],+ "selected": bool(option.has_attr("selected")),+ }+ )+ return options+++def _field_label(soup: Any, field: Any) -> str:+ field_id = _attr_value(field, "id")+ if field_id:+ label_tag = soup.find("label", attrs={"for": field_id})+ if label_tag is not None:+ label = _node_text(label_tag)+ if label:+ return label+ parent_label = field.find_parent("label") if hasattr(field, "find_parent") else None+ if parent_label is not None:+ label = _node_text(parent_label).replace(_node_text(field), "").strip()+ if label:+ return label+ for value in (+ _attr_value(field, "aria-label"),+ _adjacent_text(field),+ _parent_text_label(field),+ _attr_value(field, "title"),+ _attr_value(field, "value"),+ ):+ if value:+ return value+ return ""+++def _page_title(soup: Any) -> str:+ parts: list[str] = []+ for tag_name in ("title", "h1"):+ tag = soup.find(tag_name)+ text = _node_text(tag) if tag is not None else ""+ if text and text not in parts:+ parts.append(text)+ return " ".join(parts)[:240]+++def _result_container_entry(node: Any) -> dict[str, Any]:+ tag_name = str(getattr(node, "name", "") or "").lower()+ node_id = str(node.get("id") or "")+ selector = _selector_for(node)[:160]+ entry: dict[str, Any] = {"tag": tag_name, "id": node_id[:120], "selector": selector}+ if tag_name == "table":+ entry["row_selector"] = f"{selector} tbody tr"+ entry["expand_toggle_candidates"] = [+ f"{selector} tbody tr [aria-expanded]",+ f'{selector} tbody tr [role="button"]',+ f"{selector} tbody tr button",+ f"{selector} tbody tr a",+ f"{selector} tbody tr td:first-child",+ ]+ return entry+++def _challenge_identity(node: Any) -> str:+ parts = [+ str(getattr(node, "name", "") or ""),+ _attr_value(node, "id"),+ _attr_value(node, "name"),+ " ".join(_classes_for(node)),+ _attr_value(node, "src"),+ # Standard anti-bot widgets commonly expose this attribute.+ _attr_value(node, "data-sitekey"),+ _attr_value(node, "aria-label"),+ _attr_value(node, "title"),+ ]+ return " ".join(part for part in parts if part).lower()+++def _challenge_entry(node: Any) -> dict[str, Any]:+ return {+ "tag": str(getattr(node, "name", "") or "")[:40],+ "id": _attr_value(node, "id")[:120],+ "name": _attr_value(node, "name")[:120],+ "class": " ".join(_classes_for(node)[:5])[:160],+ "type": _attr_value(node, "type")[:40],+ "selector": _selector_for(node)[:160],+ }+++def _anti_bot_evidence(nodes: list[Any], html: str) -> tuple[list[str], list[dict[str, Any]]]:+ indicators: list[str] = []++ def add_indicator(value: str) -> None:+ if value not in indicators:+ indicators.append(value)++ html_prefix = (html or "").lower()[:4096]+ for pattern in _ANTI_BOT_PATTERNS:+ if pattern in html_prefix:+ add_indicator(pattern)++ challenge_controls: list[dict[str, Any]] = []+ for node in nodes:+ identity = _challenge_identity(node)+ matched = False+ for pattern in _ANTI_BOT_PATTERNS:+ if pattern in identity:+ add_indicator(pattern)+ matched = True+ tag_name = str(getattr(node, "name", "") or "").lower()+ if matched and tag_name in {"div", "iframe", "input", "button", "textarea"}:+ if len(challenge_controls) < _MAX_CHALLENGE_CONTROLS:+ challenge_controls.append(_challenge_entry(node))++ return indicators[:8], challenge_controls+++def parse_composition_html(html: str, *, inspected_url: str, current_url: str) -> dict[str, Any]:+ """Extract a compact page schema for build-time workflow composition."""++ if BeautifulSoup is None:+ return _empty_evidence(inspected_url, current_url)+ try:+ soup = BeautifulSoup(html or "", "html.parser")+ except Exception:+ return _empty_evidence(inspected_url, current_url)++ all_nodes = soup.find_all(True)++ forms: list[dict[str, Any]] = []+ for form in soup.find_all("form")[:_MAX_FORMS]:+ fields: list[dict[str, Any]] = []+ submit_controls: list[dict[str, Any]] = []+ for node in form.find_all(["input", "select", "textarea", "button"]):+ tag_name = str(getattr(node, "name", "") or "").lower()+ field_type = str(node.get("type") or tag_name or "text").lower()+ if tag_name == "input" and field_type in {"hidden", "reset"}:+ continue+ if tag_name == "button" or field_type in {"submit", "button"}:+ submit_controls.append(+ {+ "text": _schema_text(_node_text(node) or str(node.get("value") or ""), 120),+ "name": str(node.get("name") or "")[:120],+ "id": str(node.get("id") or "")[:120],+ "value": _attr_value(node, "value")[:160],+ "class": " ".join(_classes_for(node)[:5])[:160],+ "type": field_type[:40],+ "disabled": _control_disabled(node),+ "selector": _selector_for(node)[:160],+ }+ )+ continue+ if len(fields) >= _MAX_FIELDS_PER_FORM:+ continue+ fields.append(+ {+ "name": str(node.get("name") or "")[:120],+ "id": str(node.get("id") or "")[:120],+ "label": _schema_text(_field_label(soup, node), 240),+ "type": field_type[:40],+ "value": _attr_value(node, "value")[:160],+ "class": " ".join(_classes_for(node)[:5])[:160],+ "placeholder": _schema_text(str(node.get("placeholder") or ""), 240),+ "required": bool(+ node.has_attr("required") or str(node.get("aria-required") or "").lower() == "true"+ ),+ "disabled": _control_disabled(node),+ "checked": bool(node.has_attr("checked")),+ "options": _select_options(node) if tag_name == "select" else [],+ "selector": _selector_for(node)[:160],+ }+ )+ forms.append(+ {+ "id": str(form.get("id") or "")[:120],+ "name": str(form.get("name") or "")[:120],+ "action": str(form.get("action") or "")[:240],+ "method": str(form.get("method") or "")[:20],+ "fields": fields,+ "submit_controls": submit_controls[:10],+ }+ )++ navigation_targets: list[dict[str, Any]] = []+ for link in soup.find_all("a", href=True):+ if len(navigation_targets) >= _MAX_NAVIGATION_TARGETS:+ break+ href = str(link.get("href") or "").strip()+ if not href or href.startswith("#") or href.lower().startswith("javascript:"):+ continue+ resolved_href = urljoin(current_url or inspected_url, href)+ if not _same_origin(resolved_href, current_url or inspected_url):+ continue+ text = _node_text(link)+ navigation_targets.append(+ {+ "text": _schema_text(text, 160),+ "href": resolved_href[:300],+ "selector": _selector_for(link)[:160],+ }+ )++ result_containers: list[dict[str, Any]] = []+ for node in all_nodes:+ if len(result_containers) >= _MAX_RESULT_CONTAINERS:+ break+ tag_name = str(getattr(node, "name", "") or "").lower()+ node_id = str(node.get("id") or "")+ class_value = node.get("class") or []+ class_text = " ".join(class_value) if isinstance(class_value, list) else str(class_value)+ result_identity = f"{node_id} {class_text}".lower()+ if tag_name == "table" or any(hint in result_identity for hint in _RESULT_CONTAINER_HINTS):+ result_containers.append(_result_container_entry(node))++ field_count = sum(len(form.get("fields") or []) for form in forms)+ control_count = sum(len(form.get("submit_controls") or []) for form in forms)+ # Higher confidence means the parser saw a more complete form surface.+ confidence = 0.85 if field_count and control_count else 0.6 if field_count else 0.3 if forms else 0.1+ anti_bot_indicators, challenge_controls = _anti_bot_evidence(all_nodes, html)+ return {+ "inspected_url": inspected_url,+ "current_url": current_url,+ "page_title": _page_title(soup),+ "forms": forms,+ "navigation_targets": navigation_targets,+ "result_containers": result_containers,+ "anti_bot_indicators": anti_bot_indicators,+ "challenge_controls": challenge_controls,+ "evidence_confidence": confidence,+ "source_tool": "inspect_page_for_composition",+ **_evidence_metadata(anti_bot_indicators, forms=forms),+ }diff --git a/tests/unit/test_copilot_composition_evidence.py b/tests/unit/test_copilot_composition_evidence.pynew file mode 100644index 000000000..b38da94f7--- /dev/null+++ b/tests/unit/test_copilot_composition_evidence.py@@ -0,0 +1,359 @@+"""Tests for evidence-grounded Copilot composition."""++from __future__ import annotations++from dataclasses import dataclass, field++import yaml++from skyvern.forge.sdk.copilot.build_phase import BuildPhase+from skyvern.forge.sdk.copilot.composition_evidence import (+ composition_page_evidence_error,+ merge_visual_composition_evidence,+ page_evidence_needs_visual_fallback,+ parse_composition_html,+)+from skyvern.forge.sdk.copilot.turn_intent import TurnIntent, TurnIntentMode+++@dataclass+class _Ctx:+ build_phase: BuildPhase = BuildPhase.COMPOSING+ turn_intent: TurnIntent = field(default_factory=lambda: TurnIntent(mode=TurnIntentMode.BUILD))+ composition_page_evidence: dict | None = None+ workflow_yaml: str | None = None+++def _yaml(*blocks: dict) -> str:+ return yaml.safe_dump({"title": "wf", "workflow_definition": {"parameters": [], "blocks": list(blocks)}})+++def _field(label: str, name: str) -> dict:+ return {"name": name, "id": name, "label": label, "type": "text", "placeholder": "", "selector": f"#{name}"}+++def _first_last_evidence() -> dict:+ return {+ "inspected_url": "https://example.com/lookup",+ "current_url": "https://example.com/lookup",+ "source_tool": "inspect_page_for_composition",+ "forms": [+ {+ "fields": [_field("First Name", "firstName"), _field("Last Name", "lastName")],+ "submit_controls": [{"text": "Search", "id": "searchButton", "selector": "#searchButton"}],+ }+ ],+ }+++def test_composition_parse_html_extracts_labeled_fields_and_submit_controls() -> None:+ parsed = parse_composition_html(+ """+ <html><head><title>Lookup</title></head><body>+ <h1>Credential lookup</h1>+ <form id="searchForm" action="/results">+ <label for="firstName">First Name</label>+ <input id="firstName" name="firstName" type="text" required />+ <label for="lastName">Last Name</label>+ <input id="lastName" name="lastName" type="text" />+ <button id="searchButton" type="submit">Search</button>+ </form>+ <a id="openLookup" href="/registry/search">Find a Record</a>+ <a id="external" href="https://evil.example/steal">External</a>+ <table id="results"><tbody></tbody></table>+ </body></html>+ """,+ inspected_url="https://example.com/lookup",+ current_url="https://example.com/lookup",+ )++ assert parsed["page_title"] == "Lookup Credential lookup"+ assert parsed["forms"][0]["fields"][0]["name"] == "firstName"+ assert parsed["forms"][0]["fields"][0]["label"] == "First Name"+ assert parsed["forms"][0]["fields"][0]["required"] is True+ assert parsed["forms"][0]["fields"][1]["name"] == "lastName"+ assert parsed["forms"][0]["submit_controls"][0]["text"] == "Search"+ assert parsed["navigation_targets"][0]["text"] == "Find a Record"+ assert parsed["navigation_targets"][0]["href"] == "https://example.com/registry/search"+ assert [item["text"] for item in parsed["navigation_targets"]] == ["Find a Record"]+ assert parsed["result_containers"][0]["selector"] == "#results"+ assert parsed["result_containers"][0]["row_selector"] == "#results tbody tr"+ assert "#results tbody tr td:first-child" in parsed["result_containers"][0]["expand_toggle_candidates"]+ assert parsed["evidence_sources"] == ["dom_html"]+ assert parsed["screenshot_used"] is False+ assert parsed["visual_evidence_summary"] == ""+ assert parsed["challenge_state"]["detected"] is False+ assert parsed["source_tool"] == "inspect_page_for_composition"+++def test_composition_parse_html_preserves_stable_control_selectors_and_values() -> None:+ parsed = parse_composition_html(+ """+ <html><body>+ <form id="registrySearch">+ <input class="credentialTypeChoice" type="checkbox" value="STANDARD" /> Standard+ <input id="id-first_name" name="first_name" type="text" />+ <input id="id-last_name" name="last_name" type="text" />+ <select id="state" name="state">+ <option value="">Any State</option>+ <option value="MA">Massachusetts</option>+ </select>+ <input class="acknowledgementCheck" type="checkbox" value="yes" /> I agree+ <input id="btnSubmit" type="button" value="Search" />+ </form>+ </body></html>+ """,+ inspected_url="https://example.com/search",+ current_url="https://example.com/search",+ )++ fields = parsed["forms"][0]["fields"]+ assert fields[0]["selector"] == 'input.credentialTypeChoice[value="STANDARD"]'+ assert fields[0]["label"] == "Standard"+ assert fields[0]["value"] == "STANDARD"+ assert fields[1]["selector"] == "#id-first_name"+ assert fields[2]["selector"] == "#id-last_name"+ assert fields[3]["options"][1] == {"text": "Massachusetts", "value": "MA", "selected": False}+ assert fields[4]["selector"] == 'input.acknowledgementCheck[value="yes"]'+ assert fields[4]["label"] == "I agree"+ assert fields[4]["disabled"] is False+ assert parsed["forms"][0]["submit_controls"][0]["selector"] == "#btnSubmit"+ assert parsed["forms"][0]["submit_controls"][0]["value"] == "Search"+ assert parsed["forms"][0]["submit_controls"][0]["disabled"] is False+++def test_composition_parse_html_adds_challenge_state_for_anti_bot_dom() -> None:+ parsed = parse_composition_html(+ """+ <html><head><title>Just a moment...</title></head><body>+ <script src="https://verification.example/challenge.js"></script>+ <div class="human-verification">Verify you are human</div>+ </body></html>+ """,+ inspected_url="https://example.com/search",+ current_url="https://example.com/search",+ )++ assert page_evidence_needs_visual_fallback(parsed) is True+ assert "verify you are human" in parsed["anti_bot_indicators"]+ assert parsed["challenge_state"]["detected"] is True+ assert parsed["challenge_state"]["kind"] == "human_verification"+ assert parsed["challenge_state"]["source"] == "dom_html"+ assert parsed["challenge_state"]["gates_submit_controls"] is False+ assert parsed["challenge_state"]["gated_submit_controls"] == []+++def test_merge_visual_composition_evidence_keeps_screenshot_bounded_and_typed() -> None:+ parsed = parse_composition_html(+ "<html><head><title>Just a moment...</title></head><body>Human verification</body></html>",+ inspected_url="https://example.com/search",+ current_url="https://example.com/search",+ )++ merged = merge_visual_composition_evidence(+ parsed,+ visual_summary={+ "summary": "A human verification card is visible below the search form.",+ "challenge_detected": True,+ "challenge_kind": "human_verification",+ "challenge_location": "Below the acknowledgement checkbox and above the Search button.",+ "submit_blocked": True,+ "blocked_submit_controls": ["Search"],+ "omissions": ["Result rows are not visible before verification."],+ },+ )++ assert merged["evidence_sources"] == ["dom_html", "screenshot", "vision_summary"]+ assert merged["screenshot_used"] is True+ assert merged["visual_evidence_summary"] == "A human verification card is visible below the search form."+ assert merged["challenge_state"]["source"] == "dom+screenshot"+ assert (+ merged["challenge_state"]["visual_location"]+ == "Below the acknowledgement checkbox and above the Search button."+ )+ assert merged["challenge_state"]["gates_submit_controls"] is True+ assert merged["challenge_state"]["gated_submit_controls"] == [{"text": "Search", "disabled": True}]+ assert merged["visual_evidence_omissions"] == ["Result rows are not visible before verification."]+++def test_composition_parse_html_surfaces_turnstile_challenge_controls_after_long_page_preamble() -> None:+ parsed = parse_composition_html(+ f"""+ <html><head><title>Credential Registry</title></head><body>+ <form id="searchForm">+ <input class="credentialTypeChoice" type="radio" name="credentialType[]" value="STANDARD" /> Standard+ <input id="id-first_name" name="first_name" type="text" />+ <input id="id-last_name" name="last_name" type="text" />+ <input class="acknowledgementCheck" type="checkbox" name="acknowledgement" value="yes" /> I agree+ <input id="btnSubmit" name="btnSubmit" type="button" value="Search" disabled />+ </form>+ <div>{"x" * 5000}</div>+ <script src="https://verification.example/challenge.js"></script>+ <div id="human-verification-widget" class="human-verification challenge-widget"+ data-callback="verificationSuccess"></div>+ <input type="hidden" name="human-verification-response" id="human-verification-response" />+ </body></html>+ """,+ inspected_url="https://example.com/registry/search",+ current_url="https://example.com/registry/search",+ )++ assert {+ "challenge",+ "human-verification",+ }.issubset(set(parsed["anti_bot_indicators"]))+ assert {control["selector"] for control in parsed["challenge_controls"]} >= {+ "#human-verification-widget",+ "#human-verification-response",+ }+ assert parsed["forms"][0]["submit_controls"][0]["selector"] == "#btnSubmit"+ assert parsed["forms"][0]["submit_controls"][0]["disabled"] is True+ assert parsed["challenge_state"]["gates_submit_controls"] is True+ assert parsed["challenge_state"]["gated_submit_controls"] == [+ {+ "text": "Search",+ "id": "btnSubmit",+ "name": "btnSubmit",+ "selector": "#btnSubmit",+ "disabled": True,+ }+ ]+++def test_composition_gate_requires_page_evidence_before_page_dependent_blocks() -> None:+ goto_block = {"block_type": "goto_url", "label": "open_lookup", "url": "https://example.com/lookup"}+ search_block = {+ "block_type": "navigation",+ "label": "search_lookup",+ "navigation_goal": "Enter the person name into the name search field and click Search.",+ }++ assert composition_page_evidence_error(_Ctx(), _yaml(goto_block)) is None++ error = composition_page_evidence_error(_Ctx(), _yaml(goto_block, search_block))++ assert error is not None+ assert "inspect_page_for_composition" in error+ assert "save only the initial goto_url block" in error+ assert "search_lookup" in error+++def test_composition_gate_rejects_stale_page_evidence_from_another_origin() -> None:+ workflow_yaml = _yaml(+ {"block_type": "goto_url", "label": "open_lookup", "url": "https://example.com/lookup"},+ {+ "block_type": "navigation",+ "label": "search_lookup",+ "navigation_goal": "Enter {{ parameters.person_name }} into the name search field and submit.",+ },+ )+ evidence = {+ **_first_last_evidence(),+ "inspected_url": "https://other.example/lookup",+ "current_url": "https://other.example/lookup",+ }++ error = composition_page_evidence_error(_Ctx(composition_page_evidence=evidence), workflow_yaml)++ assert error is not None+ assert "page-dependent build blocks need observed page evidence" in error+++def test_composition_gate_rejects_stale_page_evidence_from_same_origin_different_path() -> None:+ workflow_yaml = _yaml(+ {"block_type": "goto_url", "label": "open_lookup", "url": "https://example.com/lookup"},+ {+ "block_type": "navigation",+ "label": "search_lookup",+ "navigation_goal": "Enter {{ parameters.person_name }} into the name search field and submit.",+ },+ )+ evidence = {+ **_first_last_evidence(),+ "inspected_url": "https://example.com/login",+ "current_url": "https://example.com/login",+ }++ error = composition_page_evidence_error(_Ctx(composition_page_evidence=evidence), workflow_yaml)++ assert error is not None+ assert "page-dependent build blocks need observed page evidence" in error+++def test_composition_gate_rejects_untyped_browser_observation_evidence() -> None:+ workflow_yaml = _yaml(+ {"block_type": "goto_url", "label": "open_lookup", "url": "https://example.com/lookup"},+ {+ "block_type": "navigation",+ "label": "search_lookup",+ "navigation_goal": "Enter {{ parameters.person_name }} into the name search field and submit.",+ },+ )+ evidence = {+ "inspected_url": "https://example.com/lookup",+ "current_url": "https://example.com/lookup",+ "forms": [],+ "source_tool": "get_browser_screenshot",+ }++ error = composition_page_evidence_error(_Ctx(composition_page_evidence=evidence), workflow_yaml)++ assert error is not None+ assert "inspect_page_for_composition" in error+++def test_composition_gate_applies_to_edit_turns_that_add_page_dependent_blocks() -> None:+ existing_yaml = _yaml(+ {"block_type": "goto_url", "label": "open_lookup", "url": "https://example.com/lookup"},+ )+ workflow_yaml = _yaml(+ {"block_type": "goto_url", "label": "open_lookup", "url": "https://example.com/lookup"},+ {+ "block_type": "navigation",+ "label": "search_lookup",+ "navigation_goal": "Enter {{ parameters.person_name }} into the name search field and submit.",+ },+ )+ ctx = _Ctx(+ turn_intent=TurnIntent(mode=TurnIntentMode.EDIT),+ composition_page_evidence=None,+ workflow_yaml=existing_yaml,+ )++ error = composition_page_evidence_error(ctx, workflow_yaml)++ assert error is not None+ assert "search_lookup" in error+++def test_composition_gate_rejects_page_dependent_blocks_without_target_url() -> None:+ workflow_yaml = _yaml(+ {+ "block_type": "navigation",+ "label": "search_lookup",+ "navigation_goal": "Enter {{ parameters.person_name }} into the name search field and submit.",+ },+ )++ error = composition_page_evidence_error(_Ctx(composition_page_evidence=_first_last_evidence()), workflow_yaml)++ assert error is not None+ assert "target_url=None" in error+++def test_composition_gate_allows_navigation_after_matching_evidence() -> None:+ workflow_yaml = _yaml(+ {"block_type": "goto_url", "label": "open_lookup", "url": "https://example.com/lookup"},+ {+ "block_type": "navigation",+ "label": "search_lookup",+ "navigation_goal": (+ "Enter the first name into the observed First Name field, enter the last name into the "+ "observed Last Name field, then click the observed Search button."+ ),+ },+ )++ error = composition_page_evidence_error(_Ctx(composition_page_evidence=_first_last_evidence()), workflow_yaml)++ assert error is None