Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
SKY-12368: capture click-to-reveal values into structured copilot page evidence (#7381)
Andrew Neilsonc8e2876ffb49
diff --git a/skyvern/forge/sdk/copilot/composition_browser_expressions.py b/skyvern/forge/sdk/copilot/composition_browser_expressions.pyindex fec7dfe7d..4dc714420 100644--- a/skyvern/forge/sdk/copilot/composition_browser_expressions.py+++ b/skyvern/forge/sdk/copilot/composition_browser_expressions.py@@ -18,6 +18,7 @@ from skyvern.forge.sdk.copilot.composition_evidence import (_MAX_PAGE_OBSTRUCTIONS,_MAX_RESULT_CONTAINERS,_MAX_RESULT_SAMPLE_ROWS,+ _MAX_REVEAL_KEY_VALUE_RELATIONS,_MAX_SELECT_OPTIONS,_MAX_TABLE_HEADERS,_MAX_VISIBLE_TEXT_EXCERPT_CHARS,@@ -202,6 +203,7 @@ _STRUCTURED_CONST_HEADER = (f"const MAX_NAVIGATION_TARGETS={int(_MAX_NAVIGATION_TARGETS)};"f"const MAX_RESULT_CONTAINERS={int(_MAX_RESULT_CONTAINERS)};"f"const MAX_KEY_VALUE_RELATIONS={int(_MAX_KEY_VALUE_RELATIONS)};"+ f"const MAX_REVEAL_KEY_VALUE_RELATIONS={int(_MAX_REVEAL_KEY_VALUE_RELATIONS)};"f"const MAX_TABLE_HEADERS={int(_MAX_TABLE_HEADERS)};"f"const MAX_RESULT_SAMPLE_ROWS={int(_MAX_RESULT_SAMPLE_ROWS)};"f"const MAX_SELECT_OPTIONS={int(_MAX_SELECT_OPTIONS)};"@@ -488,6 +490,46 @@ for (const node of all) {keyValueRelations.push({ key_text: keyText, value_text: valueText, container_selector: selector, container_match_count: matches, container_position: position, value_child_index: 1, direct_child_count: children.length, visible: true, value_visible: elementVisible(children[1]) });}+const revealHintTokens = (node) => (attr(node, 'id') + ' ' + classesFor(node).join(' ')).toLowerCase().split(/[^a-z0-9]+/).filter(Boolean);+const matchesResultHintToken = (node) => revealHintTokens(node).some((t) => RESULT_CONTAINER_HINTS.includes(t));+const revealHeadingTags = new Set(['h1', 'h2', 'h3', 'h4', 'h5', 'h6']);+let revealRelationCount = 0;+let revealRelationsTruncated = false;+for (const node of all) {+ const tag = (node.tagName || '').toLowerCase();+ if (keyValueSkipTags.has(tag) || !elementVisible(node)) continue;+ if (!matchesResultHintToken(node)) continue;+ const children = Array.from(node.children || []);+ if (children.length < 3 || children.length > 6) continue;+ if (children.some((c) => c.children && c.children.length > 0)) continue;+ const heading = children[0];+ if (!revealHeadingTags.has((heading.tagName || '').toLowerCase()) || !elementVisible(heading)) continue;+ const keyText = nodeText(heading);+ if (!keyText || keyText.length > 120) continue;+ const selector = selectorFor(node);+ const matches = selectorMatchCount(selector);+ if (!matches) continue;+ let position = -1;+ try { position = Array.from(document.querySelectorAll(selector)).indexOf(node); } catch (e) { position = -1; }+ if (position < 0) continue;+ const valueLeaves = [];+ for (let i = 1; i < children.length; i++) {+ const leaf = children[i];+ if (!elementVisible(leaf)) continue;+ const valueText = nodeText(leaf);+ if (!valueText || keyText === valueText) continue;+ valueLeaves.push({ index: i, valueText: valueText });+ }+ const revealKeyText = valueLeaves.length === 1 ? keyText : '';+ let capped = false;+ for (const leaf of valueLeaves) {+ if (keyValueRelations.length >= MAX_KEY_VALUE_RELATIONS || revealRelationCount >= MAX_REVEAL_KEY_VALUE_RELATIONS) { revealRelationsTruncated = true; capped = true; break; }+ keyValueRelations.push({ key_text: revealKeyText, value_text: leaf.valueText, container_selector: selector, container_match_count: matches, container_position: position, value_child_index: leaf.index, direct_child_count: children.length, visible: true, value_visible: true });+ revealRelationCount++;+ }+ if (capped) break;+}+const challengeControls = [];const seenChallenge = new Set();for (const node of all) {@@ -570,6 +612,7 @@ return JSON.stringify({result_containers_truncated: resultContainersTruncated,key_value_relations: keyValueRelations,key_value_relations_truncated: keyValueRelationsTruncated,+ reveal_relations_truncated: revealRelationsTruncated,clickable_controls: clickableControls,challenge_controls: challengeControls,modal_overlays: modalOverlays,diff --git a/skyvern/forge/sdk/copilot/composition_evidence.py b/skyvern/forge/sdk/copilot/composition_evidence.pyindex a4211b7cf..c0f8bc177 100644--- a/skyvern/forge/sdk/copilot/composition_evidence.py+++ b/skyvern/forge/sdk/copilot/composition_evidence.py@@ -2,6 +2,7 @@from __future__ import annotations+import refrom collections.abc import Iterablefrom typing import Any, Protocolfrom urllib.parse import urljoin, urlparse@@ -55,6 +56,7 @@ _MAX_FORMS = 5_MAX_FIELDS_PER_FORM = 20_MAX_RESULT_CONTAINERS = 8_MAX_KEY_VALUE_RELATIONS = 24+_MAX_REVEAL_KEY_VALUE_RELATIONS = 8_MAX_TABLE_HEADERS = 12_MAX_RESULT_SAMPLE_ROWS = 5_MAX_NAVIGATION_TARGETS = 20@@ -201,14 +203,16 @@ def _evidence_metadata(*,forms: list[dict[str, Any]] | None = None,challenge_controls: list[dict[str, Any]] | None = None,+ reveal_relations_truncated: bool = False,) -> dict[str, Any]:gated_controls = _gated_submit_controls(forms or [])+ inspection_warnings = ["reveal_relations_truncated"] if reveal_relations_truncated else []return {"evidence_sources": [DOM_EVIDENCE_SOURCE],"screenshot_used": False,"visual_evidence_summary": "","visual_evidence_omissions": [],- "inspection_warnings": [],+ "inspection_warnings": inspection_warnings,"challenge_state": _challenge_state(indicators or [],gated_submit_controls=gated_controls,@@ -1515,7 +1519,7 @@ def _selector_match_count(soup: Any, selector: str) -> int:return 0-def _key_value_relations(soup: Any) -> tuple[list[dict[str, Any]], bool]:+def _key_value_relations(soup: Any) -> tuple[list[dict[str, Any]], bool, bool]:relations: list[dict[str, Any]] = []truncated = Falsefor node in soup.find_all(True):@@ -1556,7 +1560,74 @@ def _key_value_relations(soup: Any) -> tuple[list[dict[str, Any]], bool]:"value_visible": True,})- return relations, truncated+ reveal_truncated = _append_reveal_shape_relations(soup, relations)+ return relations, truncated, reveal_truncated+++def _matches_result_hint_token(node: Any) -> bool:+ node_id = str(node.get("id") or "") if hasattr(node, "get") else ""+ raw = f"{node_id} {' '.join(_classes_for(node))}".lower()+ return any(token in _RESULT_CONTAINER_HINTS for token in re.split(r"[^a-z0-9]+", raw) if token)+++def _append_reveal_shape_relations(soup: Any, relations: list[dict[str, Any]]) -> bool:+ reveal_count = 0+ reveal_truncated = False+ for node in soup.find_all(True):+ tag_name = str(getattr(node, "name", "") or "").lower()+ if tag_name in {"body", "form", "html", "table", "tbody", "thead", "tr"}:+ continue+ if not _matches_result_hint_token(node):+ continue+ children = [child for child in node.find_all(recursive=False) if child.name]+ if len(children) < 3 or len(children) > 6:+ continue+ if any(child.find(True) is not None for child in children):+ continue+ heading = children[0]+ if str(getattr(heading, "name", "") or "").lower() not in {"h1", "h2", "h3", "h4", "h5", "h6"}:+ continue+ key_text = _schema_text(_node_text(heading), 240)+ if not key_text or len(key_text) > 120:+ continue+ selector = _selector_for(node)[:160]+ match_count = _selector_match_count(soup, selector)+ if match_count <= 0:+ continue+ try:+ position = soup.select(selector).index(node)+ except ValueError:+ continue+ value_leaves: list[tuple[int, str]] = []+ for index in range(1, len(children)):+ value_text = _schema_text(_node_text(children[index]), 240)+ if not value_text or key_text == value_text:+ continue+ value_leaves.append((index, value_text))+ reveal_key_text = key_text if len(value_leaves) == 1 else ""+ capped = False+ for index, value_text in value_leaves:+ if len(relations) >= _MAX_KEY_VALUE_RELATIONS or reveal_count >= _MAX_REVEAL_KEY_VALUE_RELATIONS:+ reveal_truncated = True+ capped = True+ break+ relations.append(+ {+ "key_text": reveal_key_text,+ "value_text": value_text,+ "container_selector": selector,+ "container_match_count": match_count,+ "container_position": position,+ "value_child_index": index,+ "direct_child_count": len(children),+ "visible": True,+ "value_visible": True,+ }+ )+ reveal_count += 1+ if capped:+ break+ return reveal_truncateddef _result_container_entry(node: Any, *, soup: Any) -> dict[str, Any]:@@ -1948,7 +2019,8 @@ def parse_composition_html(html: str, *, inspected_url: str, current_url: str) -breakresult_containers.append(_result_container_entry(node, soup=soup))- key_value_relations, key_value_relations_truncated = _key_value_relations(soup)+ key_value_relations, key_value_relations_truncated, reveal_relations_truncated = _key_value_relations(soup)+ reveal_relations_truncated = reveal_relations_truncated and not key_value_relations_truncatedused_selectors: set[str] = set()for form in forms:@@ -1998,7 +2070,12 @@ def parse_composition_html(html: str, *, inspected_url: str, current_url: str) -"empty_page_visual_state": None,"evidence_confidence": confidence,"source_tool": "inspect_page_for_composition",- **_evidence_metadata(anti_bot_indicators, forms=forms, challenge_controls=challenge_controls),+ **_evidence_metadata(+ anti_bot_indicators,+ forms=forms,+ challenge_controls=challenge_controls,+ reveal_relations_truncated=reveal_relations_truncated,+ ),}@@ -2226,7 +2303,8 @@ def _structured_key_value_relations(value: Any) -> list[dict[str, Any]]:position = item.get("container_position")child_index = item.get("value_child_index")child_count = item.get("direct_child_count")- if not key_text or not selector:+ value_text = _schema_text(_structured_str(item.get("value_text")), 240)+ if not selector or (not key_text and not value_text):continueif (not isinstance(match_count, int)@@ -2248,7 +2326,7 @@ def _structured_key_value_relations(value: Any) -> list[dict[str, Any]]:relations.append({"key_text": key_text,- "value_text": _schema_text(_structured_str(item.get("value_text")), 240),+ "value_text": value_text,"container_selector": selector,"container_match_count": match_count,"container_position": position,@@ -2383,6 +2461,9 @@ def parse_composition_structured(data: Any, *, inspected_url: str, current_url:navigation_targets = _structured_navigation_targets(data.get("navigation_targets"), base_url=base_url)result_containers = _structured_result_containers(data.get("result_containers"))key_value_relations = _structured_key_value_relations(data.get("key_value_relations"))+ reveal_relations_truncated = (+ data.get("reveal_relations_truncated") is True and data.get("key_value_relations_truncated") is not True+ )clickable_controls = _structured_clickable_controls(data.get("clickable_controls"))challenge_controls = _structured_challenge_controls(data.get("challenge_controls"))modal_overlays = _structured_modal_overlays(data.get("modal_overlays"))@@ -2430,5 +2511,10 @@ def parse_composition_structured(data: Any, *, inspected_url: str, current_url:"empty_page_visual_state": None,"evidence_confidence": confidence,"source_tool": "inspect_page_for_composition",- **_evidence_metadata(anti_bot_indicators, forms=forms, challenge_controls=challenge_controls),+ **_evidence_metadata(+ anti_bot_indicators,+ forms=forms,+ challenge_controls=challenge_controls,+ reveal_relations_truncated=reveal_relations_truncated,+ ),}diff --git a/tests/unit/test_copilot_build_test_outcome.py b/tests/unit/test_copilot_build_test_outcome.pyindex 90d2366f6..c2ab22481 100644--- a/tests/unit/test_copilot_build_test_outcome.py+++ b/tests/unit/test_copilot_build_test_outcome.py@@ -236,6 +236,37 @@ def test_hollow_outcome_carries_observed_value_excerpt_off_the_structural_key()assert "WTR-1842-DEMO" not in str(confirmation.structural_key_payload)+def test_hollow_outcome_reason_code_unchanged_with_value_carrying_relation() -> None:+ outcome = recorded_outcome_from_scout_act_observe_hollow(+ interaction_tool="click",+ selector="#view-statement",+ current_url="https://portal.example.com/statement",+ source_url="https://portal.example.com/statement",+ page_evidence={+ "page_title": "Statement",+ "forms": [],+ "key_value_relations": [+ {+ "key_text": "March 2026 statement",+ "value_text": "Amount due: $3,927.75",+ "container_selector": "#result",+ "value_child_index": 1,+ "direct_child_count": 3,+ "visible": True,+ "value_visible": True,+ }+ ],+ "key_value_relations_truncated": False,+ },+ recapture_attempted=True,+ recapture_result="hollow",+ )++ assert outcome.reason_code == "scout_act_observe_hollow_after_interaction"+ assert outcome.is_authoritative is True+ assert "$3,927.75" not in str(outcome.structural_key_payload)++def test_hollow_outcome_value_excerpt_falls_back_to_legacy_text_keys() -> None:from_visible_text = recorded_outcome_from_scout_act_observe_hollow(interaction_tool="click",diff --git a/tests/unit/test_copilot_composition_evidence.py b/tests/unit/test_copilot_composition_evidence.pyindex 372d2c28d..861e91e54 100644--- a/tests/unit/test_copilot_composition_evidence.py+++ b/tests/unit/test_copilot_composition_evidence.py@@ -2407,6 +2407,193 @@ def test_html_parse_marks_empty_cell_without_text() -> None:assert cells[1]["has_text"] is False+_REVEAL_URL = "https://portal.example.com/statement"+_REVEAL_HTML = """+<body>+ <header><h1>Business Billing</h1><p>Account 880314</p></header>+ <section class="results" id="result" STYLE>+ <h3 id="result-title">March 2026 statement</h3>+ <div class="amount" id="result-amount">Amount due: $3,927.75</div>+ <p class="muted" id="result-period">Billing period: Mar 1 - Mar 31, 2026</p>+ </section>+</body>+"""+++def test_html_reveal_shape_container_emits_value_relations() -> None:+ hidden = parse_composition_html(+ _REVEAL_HTML.replace("STYLE", 'style="display:none"'),+ inspected_url=_REVEAL_URL,+ current_url=_REVEAL_URL,+ )+ revealed = parse_composition_html(+ _REVEAL_HTML.replace("STYLE", ""), inspected_url=_REVEAL_URL, current_url=_REVEAL_URL+ )++ hidden_values = [relation["value_text"] for relation in hidden["key_value_relations"]]+ assert "Amount due: $3,927.75" not in hidden_values++ reveal_relations = [+ relation for relation in revealed["key_value_relations"] if relation["container_selector"] == "#result"+ ]+ assert [+ (relation["key_text"], relation["value_text"], relation["value_child_index"]) for relation in reveal_relations+ ] == [+ ("", "Amount due: $3,927.75", 1),+ ("", "Billing period: Mar 1 - Mar 31, 2026", 2),+ ]+ assert all(relation["direct_child_count"] == 3 for relation in reveal_relations)+ assert revealed["key_value_relations_truncated"] is False+ assert has_witnessed_value_content(revealed) is True+++def test_html_reveal_shape_single_value_leaf_keys_by_heading() -> None:+ parsed = parse_composition_html(+ '<body><section id="result"><h3>Statement total</h3><div>Amount due: $3,927.75</div><p></p></section></body>',+ inspected_url=_REVEAL_URL,+ current_url=_REVEAL_URL,+ )+ reveal = [r for r in parsed["key_value_relations"] if r["direct_child_count"] == 3]+ assert [(r["key_text"], r["value_text"], r["value_child_index"]) for r in reveal] == [+ ("Statement total", "Amount due: $3,927.75", 1)+ ]+++def test_html_reveal_shape_multi_value_leaves_carry_empty_key() -> None:+ parsed = parse_composition_html(+ '<body><section id="result"><h3>Statement</h3>'+ "<div>Amount due: $3,927.75</div><p>Billing period</p><span>Due date</span></section></body>",+ inspected_url=_REVEAL_URL,+ current_url=_REVEAL_URL,+ )+ reveal = [r for r in parsed["key_value_relations"] if r["direct_child_count"] == 4]+ assert len(reveal) == 3+ assert all(r["key_text"] == "" for r in reveal)+++def test_html_reveal_shape_rejects_structural_and_token_negatives() -> None:+ def reveal_relations(inner: str, container: str = 'id="result"') -> list[dict[str, Any]]:+ parsed = parse_composition_html(+ f"<body><section {container}>{inner}</section></body>",+ inspected_url=_REVEAL_URL,+ current_url=_REVEAL_URL,+ )+ return [r for r in parsed["key_value_relations"] if r["direct_child_count"] >= 3]++ three_leaf = "<h3>Heading</h3><div>A</div><p>B</p>"+ assert reveal_relations(three_leaf, 'id="arrow-box"') == []+ assert reveal_relations(three_leaf, 'class="browser-card"') == []+ assert reveal_relations(three_leaf, 'id="panel"') == []+ assert reveal_relations("<div>Heading</div><div>A</div><p>B</p>") == []+ assert reveal_relations("<h3>Heading</h3><div><span>A</span></div><p>B</p>") == []+ over_cap = "<h3>Heading</h3>" + "".join(f"<div>v{i}</div>" for i in range(6))+ assert reveal_relations(over_cap) == []+ long_heading = "x" * 121+ assert reveal_relations(f"<h3>{long_heading}</h3><div>A</div><p>B</p>") == []+++def test_html_reveal_shape_six_child_boundary_included() -> None:+ inner = "<h3>Heading</h3>" + "".join(f"<div>v{i}</div>" for i in range(5))+ parsed = parse_composition_html(+ f'<body><section id="result">{inner}</section></body>',+ inspected_url=_REVEAL_URL,+ current_url=_REVEAL_URL,+ )+ reveal = [r for r in parsed["key_value_relations"] if r["direct_child_count"] == 6]+ assert len(reveal) == 5+++def test_html_reveal_shape_two_child_container_emits_single_relation() -> None:+ parsed = parse_composition_html(+ '<body><section id="result"><h3>Key</h3><div>Value</div></section></body>',+ inspected_url=_REVEAL_URL,+ current_url=_REVEAL_URL,+ )+ assert [(r["key_text"], r["value_text"], r["direct_child_count"]) for r in parsed["key_value_relations"]] == [+ ("Key", "Value", 2)+ ]+++def test_html_reveal_shape_page_cap_emits_truncation_signal() -> None:+ inner = "<h3>Heading</h3>" + "".join(f"<div>v{i}</div>" for i in range(5))+ containers = "".join(f'<section id="result-{index}">{inner}</section>' for index in range(5))+ parsed = parse_composition_html(+ f"<body><header><h1>Title</h1><p>Sub</p></header>{containers}</body>",+ inspected_url=_REVEAL_URL,+ current_url=_REVEAL_URL,+ )+ relations = parsed["key_value_relations"]+ reveal = [r for r in relations if r["direct_child_count"] == 6]+ header = [r for r in relations if r["direct_child_count"] == 2]+ assert len(reveal) == 8+ assert [(r["key_text"], r["value_text"]) for r in header] == [("Title", "Sub")]+ assert parsed["key_value_relations_truncated"] is False+ assert parsed["inspection_warnings"] == ["reveal_relations_truncated"]+++def test_html_reveal_shape_non_truncating_multi_reveal_emits_no_warning() -> None:+ inner = "<h3>Heading</h3><div>A</div><p>B</p>"+ containers = "".join(f'<section id="result-{index}">{inner}</section>' for index in range(3))+ parsed = parse_composition_html(+ f"<body><header><h1>Title</h1><p>Sub</p></header>{containers}</body>",+ inspected_url=_REVEAL_URL,+ current_url=_REVEAL_URL,+ )+ reveal = [r for r in parsed["key_value_relations"] if r["direct_child_count"] == 3]+ header = [r for r in parsed["key_value_relations"] if r["direct_child_count"] == 2]+ assert len(reveal) == 6+ assert [(r["key_text"], r["value_text"]) for r in header] == [("Title", "Sub")]+ assert parsed["key_value_relations_truncated"] is False+ assert parsed["inspection_warnings"] == []+++def test_html_reveal_truncation_signal_gated_off_when_pass_one_truncated() -> None:+ kv_pairs = "".join(f'<div id="kv{i}"><span>k{i}</span><span>val{i}</span></div>' for i in range(26))+ inner = "<h3>Heading</h3>" + "".join(f"<div>v{i}</div>" for i in range(5))+ containers = "".join(f'<section id="result-{index}">{inner}</section>' for index in range(5))+ parsed = parse_composition_html(+ f"<body>{kv_pairs}{containers}</body>",+ inspected_url=_REVEAL_URL,+ current_url=_REVEAL_URL,+ )+ assert parsed["key_value_relations_truncated"] is True+ assert parsed["inspection_warnings"] == []+++def test_structured_rebinds_reveal_shape_relation_round_trip() -> None:+ payload = {+ "page_title": "Statement",+ "forms": [],+ "navigation_targets": [],+ "result_containers": [],+ "result_containers_truncated": False,+ "key_value_relations": [+ {+ "key_text": "March 2026 statement",+ "value_text": "Amount due: $3,927.75",+ "container_selector": "#result",+ "container_match_count": 1,+ "container_position": 0,+ "value_child_index": 2,+ "direct_child_count": 3,+ "visible": True,+ "value_visible": True,+ }+ ],+ "key_value_relations_truncated": False,+ "challenge_controls": [],+ "modal_overlays": [],+ "visual_obstruction_candidates": [],+ "visible_text_excerpt": "Amount due: $3,927.75",+ "anti_bot_indicators": [],+ }++ parsed = parse_composition_structured(payload, inspected_url=_REVEAL_URL, current_url=_REVEAL_URL)++ assert parsed is not None+ assert parsed["key_value_relations"] == payload["key_value_relations"]++def test_structured_detects_modal_overlay_with_dismiss_controls() -> None:payload = {"page_title": "",@@ -2860,6 +3047,126 @@ async def test_structured_extractor_matches_html_parser_on_standalone_controls_dassert 'div[data-action="selectAddress"]' in surfaced+_REVEAL_LIVE_URL = "https://portal.example.com/reveal"+_REVEAL_LIVE_HTML = """<!DOCTYPE html><html><head><title>Statement</title></head><body>+<header><h1>Business Billing</h1><p>Account 880314</p></header>+<section class="results" id="result" style="display:block">+ <h3 id="result-title">March 2026 statement</h3>+ <div class="amount" id="result-amount">Amount due: $3,927.75</div>+ <p class="muted" id="result-period">Billing period: Mar 1 - Mar 31, 2026</p>+</section>+<section class="results" id="result-hidden" style="display:none">+ <h3>Prior statement</h3><div>Amount due: $9,999.99</div><p>Old</p>+</section>+</body></html>"""+++@_skip_no_browser+async def test_structured_extractor_emits_reveal_shape_relation_on_live_dom() -> None:+ raw, content = await _capture_live_dom(_REVEAL_LIVE_URL, _REVEAL_LIVE_HTML, "#result-amount")++ structured = parse_composition_structured(+ json.loads(raw), inspected_url=_REVEAL_LIVE_URL, current_url=_REVEAL_LIVE_URL+ )+ html_parsed = parse_composition_html(content, inspected_url=_REVEAL_LIVE_URL, current_url=_REVEAL_LIVE_URL)++ assert structured is not None+ reveal = [+ (relation["key_text"], relation["value_text"], relation["value_child_index"])+ for relation in structured["key_value_relations"]+ if relation["container_selector"] == "#result"+ ]+ assert reveal == [+ ("", "Amount due: $3,927.75", 1),+ ("", "Billing period: Mar 1 - Mar 31, 2026", 2),+ ]+ assert all(relation["value_text"] != "Amount due: $9,999.99" for relation in structured["key_value_relations"])+ assert structured["key_value_relations"] == html_parsed["key_value_relations"]+ assert has_witnessed_value_content(structured) is True+++_REVEAL_HIDDEN_LEAF_URL = "https://portal.example.com/hidden-leaf"+_REVEAL_HIDDEN_LEAF_HTML = """<!DOCTYPE html><html><head><title>Statement</title></head><body>+<section class="results" id="result" style="display:block">+ <h3>Statement total</h3>+ <div id="visible-value">Amount due: $3,927.75</div>+ <p style="display:none">Hidden note leaf</p>+</section>+</body></html>"""+++@_skip_no_browser+async def test_structured_extractor_excludes_hidden_reveal_value_leaf_on_live_dom() -> None:+ raw, _ = await _capture_live_dom(_REVEAL_HIDDEN_LEAF_URL, _REVEAL_HIDDEN_LEAF_HTML, "#visible-value")++ structured = parse_composition_structured(+ json.loads(raw), inspected_url=_REVEAL_HIDDEN_LEAF_URL, current_url=_REVEAL_HIDDEN_LEAF_URL+ )++ assert structured is not None+ reveal = [+ (relation["key_text"], relation["value_text"])+ for relation in structured["key_value_relations"]+ if relation["container_selector"] == "#result"+ ]+ assert reveal == [("Statement total", "Amount due: $3,927.75")]+ assert all("Hidden note leaf" not in relation["value_text"] for relation in structured["key_value_relations"])+++_REVEAL_HIDDEN_HEADING_URL = "https://portal.example.com/hidden-heading"+_REVEAL_HIDDEN_HEADING_HTML = """<!DOCTYPE html><html><head><title>Statement</title></head><body>+<section class="results" id="result" style="display:block">+ <h3 style="display:none">Statement total</h3>+ <div id="visible-value">Amount due: $3,927.75</div>+ <p>Billing period: Mar 1 - Mar 31, 2026</p>+</section>+</body></html>"""+++@_skip_no_browser+async def test_structured_extractor_rejects_reveal_container_with_hidden_heading_on_live_dom() -> None:+ raw, _ = await _capture_live_dom(_REVEAL_HIDDEN_HEADING_URL, _REVEAL_HIDDEN_HEADING_HTML, "#visible-value")++ structured = parse_composition_structured(+ json.loads(raw), inspected_url=_REVEAL_HIDDEN_HEADING_URL, current_url=_REVEAL_HIDDEN_HEADING_URL+ )++ assert structured is not None+ assert [r for r in structured["key_value_relations"] if r["container_selector"] == "#result"] == []+++_REVEAL_CAP_LIVE_URL = "https://portal.example.com/reveal-cap"+_REVEAL_CAP_LIVE_HTML = (+ "<!DOCTYPE html><html><head><title>Statement</title></head><body>"+ + "".join(+ f'<section class="results" id="result-{index}"><h3>Heading {index}</h3>'+ + "".join(f"<div>value {index}-{leaf}</div>" for leaf in range(5))+ + "</section>"+ for index in range(5)+ )+ + "</body></html>"+)+++@_skip_no_browser+async def test_structured_extractor_emits_reveal_truncation_signal_on_live_dom() -> None:+ raw, _ = await _capture_live_dom(_REVEAL_CAP_LIVE_URL, _REVEAL_CAP_LIVE_HTML, "#result-0")+ data = json.loads(raw)++ assert data["reveal_relations_truncated"] is True++ structured = parse_composition_structured(+ data, inspected_url=_REVEAL_CAP_LIVE_URL, current_url=_REVEAL_CAP_LIVE_URL+ )+ assert structured is not None+ assert structured["key_value_relations_truncated"] is False+ assert structured["inspection_warnings"] == ["reveal_relations_truncated"]++# Tools-layer invariant: cheap path skips get_html; failure falls backdiff --git a/tests/unit/test_copilot_output_extraction_plan.py b/tests/unit/test_copilot_output_extraction_plan.pyindex f7c97d8ec..2f286bc6b 100644--- a/tests/unit/test_copilot_output_extraction_plan.py+++ b/tests/unit/test_copilot_output_extraction_plan.py@@ -9,6 +9,7 @@ from skyvern.forge.sdk.copilot.output_extraction_plan import (ShapeExpectation,ValueCardinality,ValueShape,+ _exact_path,_key_value_shape_bindings,_table_shape_bindings,array_parent_path,@@ -210,6 +211,76 @@ def test_only_exact_configured_aliases_bind_observed_labels() -> None:assert _derive(packet=fuzzy) is None+def _reveal_relation(key_text: str, *, value_child_index: int, value_text: str = "Amount due: $3,927.75") -> dict:+ return {+ "key_text": key_text,+ "value_text": value_text,+ "container_selector": "#result",+ "container_match_count": 1,+ "container_position": 0,+ "value_child_index": value_child_index,+ "direct_child_count": 4,+ "visible": True,+ "value_visible": True,+ }+++def _packet_with_reveal(*relations: dict) -> dict[str, object]:+ packet = _flow_packet()+ packet["evidence"]["key_value_relations"].extend(relations)+ return packet+++def test_exact_path_rejects_empty_label_and_binds_configured_label() -> None:+ assert _exact_path("", LABELS_BY_PATH) is None+ assert _exact_path("Overall State", LABELS_BY_PATH) == "output.overall_state"+++def test_multi_value_reveal_container_is_binder_inert_and_preserves_plan() -> None:+ base = _derive()+ with_reveal = _derive(+ packet=_packet_with_reveal(+ _reveal_relation("", value_child_index=1),+ _reveal_relation("", value_child_index=2, value_text="Billing period: Mar 1 - Mar 31, 2026"),+ )+ )++ assert base is not None+ assert with_reveal is not None+ assert with_reveal.identity == base.identity+ assert {binding.output_path for binding in with_reveal.live_reads} == {+ binding.output_path for binding in base.live_reads+ }+++def test_single_value_reveal_heading_absent_from_labels_does_not_poison_plan() -> None:+ base = _derive()+ with_reveal = _derive(packet=_packet_with_reveal(_reveal_relation("March 2026 statement", value_child_index=1)))++ assert base is not None+ assert with_reveal is not None+ assert with_reveal.identity == base.identity+++def test_single_value_reveal_heading_colliding_with_bound_label_yields_bounded_none() -> None:+ assert _derive(packet=_packet_with_reveal(_reveal_relation("Overall State", value_child_index=1))) is None+++def test_reveal_truncation_signal_voids_plan_without_pass_one_flag() -> None:+ base = _derive()+ warned = _packet_with_reveal(+ _reveal_relation("", value_child_index=1),+ _reveal_relation("", value_child_index=2, value_text="Billing period: Mar 1 - Mar 31, 2026"),+ )+ warned_evidence = warned["evidence"]+ assert isinstance(warned_evidence, dict)+ warned_evidence["inspection_warnings"] = ["reveal_relations_truncated"]++ assert base is not None+ assert warned_evidence["key_value_relations_truncated"] is False+ assert _derive(packet=warned) is None++def test_table_identity_coordinate_and_shape_ambiguity_fail_closed() -> None:nested = _flow_packet()nested["evidence"]["result_containers"][0]["nested_table_free"] = Falsediff --git a/tests/unit/test_copilot_result_evidence.py b/tests/unit/test_copilot_result_evidence.pyindex ef4a0e7a7..3eda32b9b 100644--- a/tests/unit/test_copilot_result_evidence.py+++ b/tests/unit/test_copilot_result_evidence.py@@ -3,7 +3,7 @@ from __future__ import annotationsimport jsonfrom dataclasses import replace-from skyvern.forge.sdk.copilot.composition_evidence import SCOUT_INTERACTION_EVIDENCE_TOOL+from skyvern.forge.sdk.copilot.composition_evidence import SCOUT_INTERACTION_EVIDENCE_TOOL, has_witnessed_value_contentfrom skyvern.forge.sdk.copilot.output_extraction_plan import ShapeExpectation, ValueCardinality, ValueShapefrom skyvern.forge.sdk.copilot.result_evidence import (_COMPOSITION_TARGET_SUMMARY_CHAR_BUDGET,@@ -531,6 +531,92 @@ def test_mint_excludes_ambiguous_path_with_two_candidates() -> None:assert contract is None+def _reveal_shape_relation(+ *, value_text: str, value_child_index: int, key_text: str = "March 2026 statement"+) -> dict[str, object]:+ return {+ "key_text": key_text,+ "value_text": value_text,+ "container_selector": "#result",+ "container_match_count": 1,+ "container_position": 0,+ "value_child_index": value_child_index,+ "direct_child_count": 3,+ "visible": True,+ "value_visible": True,+ }+++def _reveal_shape_page_evidence(relations: list[dict[str, object]]) -> dict[str, object]:+ return {+ "current_url": "https://portal.example.com/statement",+ "inspection_warnings": [],+ "result_containers_truncated": False,+ "key_value_relations_truncated": False,+ "key_value_relations": relations,+ "result_containers": [],+ }+++def test_mint_binds_single_reveal_shape_relation() -> None:+ evidence = _reveal_shape_page_evidence(+ [_reveal_shape_relation(value_text="Amount due: $3,927.75", value_child_index=1)]+ )+ contract = mint_scout_observation_contract(+ evidence,+ labels_by_path={"output.amount": ("March 2026 statement",)},+ url="https://portal.example.com/statement",+ has_bounded_page_schema=False,+ )++ assert contract is not None+ assert scout_observation_bound_paths(contract) == {"output.amount"}+ assert contract.bindings[0].kind == "key_value"+ assert contract.bindings[0].value_witness == "capture_nonempty_value"+++def test_mint_excludes_ambiguous_reveal_shape_siblings() -> None:+ evidence = _reveal_shape_page_evidence(+ [+ _reveal_shape_relation(value_text="Amount due: $3,927.75", value_child_index=1),+ _reveal_shape_relation(value_text="Billing period: Mar 1 - Mar 31, 2026", value_child_index=2),+ ]+ )+ contract = mint_scout_observation_contract(+ evidence,+ labels_by_path={"output.amount": ("March 2026 statement",)},+ url="https://portal.example.com/statement",+ has_bounded_page_schema=False,+ )++ assert contract is None+++def test_reveal_shape_relation_witnesses_value_content() -> None:+ evidence = _reveal_shape_page_evidence(+ [_reveal_shape_relation(value_text="Amount due: $3,927.75", value_child_index=1)]+ )+ assert has_witnessed_value_content(evidence) is True+++def test_mint_ignores_empty_key_reveal_siblings_but_still_witnesses() -> None:+ evidence = _reveal_shape_page_evidence(+ [+ _reveal_shape_relation(value_text="Amount due: $3,927.75", value_child_index=1, key_text=""),+ _reveal_shape_relation(value_text="Billing period: Mar 1 - Mar 31, 2026", value_child_index=2, key_text=""),+ ]+ )+ contract = mint_scout_observation_contract(+ evidence,+ labels_by_path={"output.amount": ("March 2026 statement",)},+ url="https://portal.example.com/statement",+ has_bounded_page_schema=False,+ )++ assert contract is None+ assert has_witnessed_value_content(evidence) is True++def test_mint_returns_none_on_truncated_or_warned_capture() -> None:truncated = _kv_page_evidence()truncated["result_containers_truncated"] = True@@ -552,6 +638,39 @@ def test_mint_returns_none_on_truncated_or_warned_capture() -> None:)+def test_mint_returns_none_on_reveal_truncation_signal() -> None:+ # A single-binding reveal packet binds non-None with no warning; the ONLY change below is the+ # reveal_relations_truncated signal, so this proves that token is load-bearing (a cap-drop on an+ # otherwise-clean reveal voids the bind) rather than the bind already being None for another reason.+ single_binding = [_reveal_shape_relation(value_text="Amount due: $3,927.75", value_child_index=1)]+ labels = {"output.amount": ("March 2026 statement",)}++ baseline = _reveal_shape_page_evidence(single_binding)+ baseline_contract = mint_scout_observation_contract(+ baseline,+ labels_by_path=labels,+ url="https://portal.example.com/statement",+ has_bounded_page_schema=False,+ )+ assert baseline_contract is not None+ assert scout_observation_bound_paths(baseline_contract) == {"output.amount"}+ assert has_witnessed_value_content(baseline) is True++ warned = _reveal_shape_page_evidence(single_binding)+ warned["inspection_warnings"] = ["reveal_relations_truncated"]+ assert warned["key_value_relations_truncated"] is False+ assert (+ mint_scout_observation_contract(+ warned,+ labels_by_path=labels,+ url="https://portal.example.com/statement",+ has_bounded_page_schema=False,+ )+ is None+ )+ assert has_witnessed_value_content(warned) is False++def test_mint_returns_none_without_url_or_labels() -> None:labels = {"output.overall_credentialing_result": ("Overall Credentialing Result",)}assert (