Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
SKY-12156: compile grounded Copilot output extraction recipes (#7310)
Andrew Neilson9793647dba2e
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.pyindex 0fa53fef9..03f99e7b6 100644--- a/skyvern/forge/sdk/copilot/agent.py+++ b/skyvern/forge/sdk/copilot/agent.py@@ -54,9 +54,11 @@ from skyvern.forge.sdk.copilot.build_test_outcome import ()from skyvern.forge.sdk.copilot.code_block_preflight import SANDBOX_UNRESOLVED_NAME_REASON_CODEfrom skyvern.forge.sdk.copilot.code_block_synthesis import (+ freeze_requested_output_extraction_candidate,is_optional_dismissal_only_trajectory,render_synthesized_offer_text,synthesize_code_block,+ synthesize_code_block_with_extraction,trajectory_has_browser_fill_interaction,)from skyvern.forge.sdk.copilot.completion_criteria_store import (@@ -64,6 +66,7 @@ from skyvern.forge.sdk.copilot.completion_criteria_store import (apply_requested_output_producer_floor,build_turn_state,reconcile_completion_criteria,+ requested_output_paths,)from skyvern.forge.sdk.copilot.completion_verification import only_structural_requested_output_abstentionsfrom skyvern.forge.sdk.copilot.config import (@@ -97,6 +100,8 @@ from skyvern.forge.sdk.copilot.enforcement import (log_scouted_spine_unresolved_at_turn_halt,outcome_fully_verified,recycle_admits_present_completion_contract_ask,+ requested_output_extraction_plan,+ requested_output_extraction_plan_changed,synthesized_persistence_reopened,synthesized_persistence_reopened_after_failed_run,synthesized_trajectory_is_goal_complete,@@ -1029,11 +1034,23 @@ def _synthesized_block_offer_prompt(ctx: CopilotContext | None) -> str:trajectory_len = len(ctx.scout_trajectory)previous_offer_len = ctx.synthesized_block_offered_trajectory_lentrajectory_goal_complete = synthesized_trajectory_is_goal_complete(ctx)+ extraction_plan = requested_output_extraction_plan(ctx)+ request_policy = getattr(ctx, "request_policy", None)+ if (+ request_policy is not None+ and requested_output_paths(request_policy.completion_criteria)+ and extraction_plan is None+ ):+ LOG.debug("copilot_synthesized_block_offer_skipped", reason="extraction_plan_unavailable")+ return ""+ plan_changed = requested_output_extraction_plan_changed(ctx, extraction_plan)+ reopened = reopened or plan_changedif (ctx.synthesized_block_offeredand trajectory_len < previous_offer_len + SYNTHESIZED_OFFER_REFRESH_STEP_THRESHOLDand (not trajectory_goal_complete or getattr(ctx, "synthesized_block_offered_goal_complete", False))and not reopened+ and not plan_changed):LOG.debug("copilot_synthesized_block_offer_skipped",@@ -1055,7 +1072,19 @@ def _synthesized_block_offer_prompt(ctx: CopilotContext | None) -> str:trajectory_len=len(ctx.scout_trajectory),fill_step_count=fill_step_count,)- synthesized = synthesize_code_block(ctx.scout_trajectory, reached_download_target=ctx.reached_download_target)+ synthesized = (+ synthesize_code_block_with_extraction(+ ctx.scout_trajectory,+ extraction_plan,+ strict_selectors=True,+ reached_download_target=getattr(ctx, "reached_download_target", None),+ )+ if extraction_plan is not None+ else synthesize_code_block(+ ctx.scout_trajectory,+ reached_download_target=getattr(ctx, "reached_download_target", None),+ )+ )if synthesized is None:LOG.debug("copilot_synthesized_block_offer_skipped",@@ -1063,6 +1092,15 @@ def _synthesized_block_offer_prompt(ctx: CopilotContext | None) -> str:trajectory_len=len(ctx.scout_trajectory),)return ""+ if extraction_plan is not None:+ candidate = freeze_requested_output_extraction_candidate(synthesized, extraction_plan, source="generated")+ if candidate is None:+ return ""+ existing_candidate = ctx.requested_output_extraction_candidate+ if existing_candidate is not None and existing_candidate != candidate and not reopened:+ LOG.warning("copilot_requested_output_extraction_candidate_mismatch")+ return ""+ ctx.requested_output_extraction_candidate = candidatectx.synthesized_block_offered = Truectx.synthesized_block_offered_trajectory_len = trajectory_lenctx.synthesized_block_offered_goal_complete = trajectory_goal_completediff --git a/skyvern/forge/sdk/copilot/code_block_synthesis.py b/skyvern/forge/sdk/copilot/code_block_synthesis.pyindex b239198d0..5ef20fc67 100644--- a/skyvern/forge/sdk/copilot/code_block_synthesis.py+++ b/skyvern/forge/sdk/copilot/code_block_synthesis.py@@ -9,6 +9,7 @@ that runs on the raw ``page`` object the copilot code block executes against.from __future__ import annotationsimport ast+import hashlibimport ioimport jsonimport keyword@@ -21,6 +22,13 @@ from typing import Anyfrom urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplitfrom skyvern.forge.sdk.copilot.composition_evidence import SCOUT_INTERACTION_EVIDENCE_TOOL+from skyvern.forge.sdk.copilot.output_extraction_plan import (+ FrozenRequestedOutputExtractionCandidate,+ LiveReadBinding,+ LiveReadKind,+ RequestedOutputExtractionPlan,+ output_path_segments,+)from skyvern.forge.sdk.copilot.reached_download_target import ReachedDownloadTargetfrom skyvern.utils.strings import escape_code_fences@@ -176,6 +184,22 @@ class SynthesizedCodeBlock:notes: list[str] = field(default_factory=list)diagnostics: SynthesisDiagnostics = field(default_factory=SynthesisDiagnostics)steps: list[dict[str, Any]] = field(default_factory=list)+ interaction_code: str = ""+ extraction_code: str = ""+ extraction_fingerprint: str = ""+ extraction_plan_identity: str = ""+++@dataclass(frozen=True, slots=True)+class SynthesizedExtractionSuffix:+ code: str+ fingerprint: str+++@dataclass+class _ExtractionReturnNode:+ children: dict[str, _ExtractionReturnNode] = field(default_factory=dict)+ value_expression: str = ""# str.splitlines() and several parsers treat these codepoints as line boundaries, so a raw one in a@@ -1235,6 +1259,188 @@ def render_missing_rung_call_sources(uncovered: Sequence[Mapping[str, Any]]) ->return "Missing rung source to reuse verbatim:\n```python\n" + "\n".join(sources) + "\n```"+def _return_node_expression(node: _ExtractionReturnNode) -> str:+ if node.value_expression:+ return node.value_expression+ items = ", ".join(+ f"{json.dumps(key)}: {_return_node_expression(child)}" for key, child in sorted(node.children.items())+ )+ return "{" + items + "}"+++def _set_return_expression(root: _ExtractionReturnNode, segments: Sequence[tuple[str, bool]], expression: str) -> None:+ current = root+ for name, _is_array in segments:+ current = current.children.setdefault(name, _ExtractionReturnNode())+ current.value_expression = expression+++def _array_prefix(binding: LiveReadBinding) -> tuple[tuple[str, bool], ...]:+ segments = output_path_segments(binding.output_path)+ for index, (_name, is_array) in enumerate(segments):+ if is_array:+ return segments[: index + 1]+ return ()+++def synthesize_extraction_suffix(plan: RequestedOutputExtractionPlan) -> SynthesizedExtractionSuffix | None:+ if not plan.live_reads:+ return None+ lines: list[str] = []+ return_root = _ExtractionReturnNode()+ scalar_bindings = [binding for binding in plan.live_reads if binding.kind == LiveReadKind.KEY_VALUE]+ for index, binding in enumerate(scalar_bindings):+ variable = f"_extraction_value_{index}"+ container = f"page.locator({json.dumps(binding.selector)})"+ target = f"{container}.nth({binding.selector_index})"+ children = f'{target}.locator(":scope > *")'+ lines.extend(+ [+ f"if await {container}.count() != {binding.selector_count}:",+ f'{_INDENT}raise ValueError("Observed scalar selector cardinality changed")',+ f"if not await {target}.is_visible():",+ f'{_INDENT}raise ValueError("Observed scalar relation is no longer visible")',+ f"if await {children}.count() != {binding.child_count}:",+ f'{_INDENT}raise ValueError("Observed scalar direct-child shape changed")',+ f"if not await {children}.nth(0).is_visible():",+ f'{_INDENT}raise ValueError("Observed scalar label is no longer visible")',+ f"if (await {children}.nth(0).inner_text()).strip() != {json.dumps(binding.relation_label)}:",+ f'{_INDENT}raise ValueError("Observed scalar label changed")',+ f"if not await {children}.nth({binding.child_index}).is_visible():",+ f'{_INDENT}raise ValueError("Observed scalar value is no longer visible")',+ f"{variable} = (await {children}.nth({binding.child_index}).inner_text()).strip()",+ ]+ )+ _set_return_expression(return_root, output_path_segments(binding.output_path), variable)++ table_groups: dict[tuple[str, int, tuple[tuple[str, bool], ...]], list[LiveReadBinding]] = {}+ for binding in plan.live_reads:+ if binding.kind != LiveReadKind.TABLE_COLUMN:+ continue+ prefix = _array_prefix(binding)+ if not prefix or not binding.row_selector or binding.row_count <= 0:+ return None+ table_groups.setdefault((binding.row_selector, binding.row_count, prefix), []).append(binding)+ for group_index, ((row_selector, row_count, prefix), bindings) in enumerate(sorted(table_groups.items())):+ records_variable = f"_extraction_records_{group_index}"+ record_root = _ExtractionReturnNode()+ exemplar = bindings[0]+ table = f"page.locator({json.dumps(exemplar.selector)})"+ selected_table = f"{table}.nth({exemplar.selector_index})"+ rows = f"page.locator({json.dumps(row_selector)})"+ headers = f'{table}.nth({exemplar.selector_index}).locator(":scope > thead > tr > th")'+ lines.append(f"if await {table}.count() != {exemplar.selector_count}:")+ lines.append(f'{_INDENT}raise ValueError("Observed table identity changed")')+ lines.append(f"if not await {table}.nth({exemplar.selector_index}).is_visible():")+ lines.append(f'{_INDENT}raise ValueError("Observed table is no longer visible")')+ lines.append(f'if await {selected_table}.locator(":scope table").count() != 0:')+ lines.append(f'{_INDENT}raise ValueError("Observed table gained a nested table")')+ lines.append(f'if await {table}.nth({exemplar.selector_index}).locator("[colspan], [rowspan]").count() != 0:')+ lines.append(f'{_INDENT}raise ValueError("Observed table gained spanning cells")')+ lines.append(f"if await {headers}.count() != {len(exemplar.headers)}:")+ lines.append(f'{_INDENT}raise ValueError("Observed table header cardinality changed")')+ for header_index, header_text in enumerate(exemplar.headers):+ lines.append(f"if (await {headers}.nth({header_index}).inner_text()).strip() != {json.dumps(header_text)}:")+ lines.append(f'{_INDENT}raise ValueError("Observed table header identity changed")')+ lines.append(f"if await {rows}.count() != {row_count}:")+ lines.append(f'{_INDENT}raise ValueError("Observed table row count changed")')+ lines.append(f"{records_variable} = []")+ for row_index in range(row_count):+ row = f"{rows}.nth({row_index})"+ cells = f'{row}.locator(":scope > th, :scope > td")'+ lines.append(f"if not await {row}.is_visible():")+ lines.append(f'{_INDENT}raise ValueError("Observed table row is no longer visible")')+ lines.append(f"if await {cells}.count() != {exemplar.row_cell_counts[row_index]}:")+ lines.append(f'{_INDENT}raise ValueError("Observed table direct-cell cardinality changed")')+ lines.append(f'if await {row}.locator(":scope > th").count() != 0:')+ lines.append(f'{_INDENT}raise ValueError("Observed table row gained a row header")')+ lines.append(+ f'if " ".join((await {row}.inner_text()).split()) != {json.dumps(exemplar.row_identities[row_index])}:'+ )+ lines.append(f'{_INDENT}raise ValueError("Observed table row identity changed")')+ for binding_index, binding in enumerate(sorted(bindings, key=lambda item: item.output_path)):+ value_variable = f"_extraction_cell_{group_index}_{row_index}_{binding_index}"+ lines.append(f"if not await {cells}.nth({binding.column_index}).is_visible():")+ lines.append(f'{_INDENT}raise ValueError("Observed table cell is no longer visible")')+ lines.append(f"{value_variable} = (await {cells}.nth({binding.column_index}).inner_text()).strip()")+ _set_return_expression(+ record_root, output_path_segments(binding.output_path)[len(prefix) :], value_variable+ )+ lines.append(f"{records_variable}.append({_return_node_expression(record_root)})")+ _set_return_expression(return_root, prefix, records_variable)++ lines.append(f"return {_return_node_expression(return_root)}")+ code = "\n".join(lines) + "\n"+ fingerprint_material = repr((plan.identity, plan.observation_identity, plan.reveal, code))+ return SynthesizedExtractionSuffix(code=code, fingerprint=hashlib.sha256(fingerprint_material.encode()).hexdigest())+++def _trajectory_contains_reveal(trajectory: Sequence[Mapping[str, Any]], plan: RequestedOutputExtractionPlan) -> bool:+ return any(+ str(interaction.get("tool_name") or "") == "click"+ and (+ (bool(plan.reveal.selector) and str(interaction.get("selector") or "") == plan.reveal.selector)+ or (+ bool(plan.reveal.role and plan.reveal.name)+ and str(interaction.get("role") or "") == plan.reveal.role+ and str(interaction.get("accessible_name") or "") == plan.reveal.name+ )+ )+ for interaction in trajectory+ )+++def synthesize_code_block_with_extraction(+ trajectory: Sequence[Mapping[str, Any]],+ extraction_plan: RequestedOutputExtractionPlan,+ *,+ strict_selectors: bool = False,+ reached_download_target: ReachedDownloadTarget | None = None,+) -> SynthesizedCodeBlock | None:+ if not _trajectory_contains_reveal(trajectory, extraction_plan):+ return None+ interaction = synthesize_code_block(+ trajectory,+ strict_selectors=strict_selectors,+ reached_download_target=reached_download_target,+ )+ suffix = synthesize_extraction_suffix(extraction_plan)+ if interaction is None or suffix is None:+ return None+ interaction_code = interaction.code.rstrip() + "\n"+ interaction.code = interaction_code + suffix.code+ interaction.interaction_code = interaction_code+ interaction.extraction_code = suffix.code+ interaction.extraction_fingerprint = suffix.fingerprint+ interaction.extraction_plan_identity = extraction_plan.identity+ return interaction+++def freeze_requested_output_extraction_candidate(+ synthesized: SynthesizedCodeBlock,+ plan: RequestedOutputExtractionPlan,+ *,+ source: str,+) -> FrozenRequestedOutputExtractionCandidate | None:+ if (+ not synthesized.extraction_code+ or not synthesized.extraction_fingerprint+ or synthesized.extraction_plan_identity != plan.identity+ ):+ return None+ return FrozenRequestedOutputExtractionCandidate(+ plan_identity=plan.identity,+ observation_identity=plan.observation_identity,+ requested_output_paths=plan.requested_output_paths,+ reveal=plan.reveal,+ interaction_code=synthesized.interaction_code,+ extraction_code=synthesized.extraction_code,+ source=source,+ admission_result="admitted",+ fingerprint=synthesized.extraction_fingerprint,+ )++# Model-owned slots the synthesizer cannot prove; the model fills these._FILL_DECLARED_GOAL = "<fill: the durable goal this block accomplishes>"_FILL_CLAIM_ID = "claim:<fill>"@@ -1372,16 +1578,25 @@ def render_synthesized_offer_text("""Render the offer body the copilot sees for a synthesized block (pure)."""param_keys = [p.get("key", "") for p in synthesized.parameters if p.get("key") and not p.get("credential_id")]credential_parameters = [p for p in synthesized.parameters if p.get("key") and p.get("credential_id")]+ extraction_instruction = (+ "The same interaction-reached evidence compiled the requested keyed extraction suffix; preserve both "+ "the reveal and extraction segments VERBATIM."+ if synthesized.extraction_code+ else (+ "Hand-author a contract-shaped keyed structure for any requested outputs, never a flat "+ "`page.inner_text(...)` / `text_content(...)` blob."+ )+ )parts = ["SYNTHESIZED CODE BLOCK (offered once). The page interactions you scouted were compiled into a ""deterministic Playwright snippet. Persist it VERBATIM as a `code` block labeled "- f"`{_SYNTHESIZED_BLOCK_LABEL}` via update_workflow / update_and_run_blocks; hand-author the "- "data-capture step it does not cover so the block `return`s a keyed structure (a dict, or an array "- "of objects for repeated records) — never a flat `page.inner_text(...)` / `text_content(...)` blob.",+ f"`{_SYNTHESIZED_BLOCK_LABEL}` via update_workflow / update_and_run_blocks. {extraction_instruction}","```python",synthesized.code.rstrip("\n"),"```",]+ if synthesized.extraction_plan_identity:+ parts.append(f"Extraction plan identity: `{synthesized.extraction_plan_identity}`.")if param_keys:default_keys = [str(p.get("key") or "")diff --git a/skyvern/forge/sdk/copilot/composition_browser_expressions.py b/skyvern/forge/sdk/copilot/composition_browser_expressions.pyindex 843393c45..4535b8a2c 100644--- a/skyvern/forge/sdk/copilot/composition_browser_expressions.py+++ b/skyvern/forge/sdk/copilot/composition_browser_expressions.py@@ -11,12 +11,15 @@ from skyvern.forge.sdk.copilot.composition_evidence import (_MAX_CLICKABLE_CONTROLS,_MAX_FIELDS_PER_FORM,_MAX_FORMS,+ _MAX_KEY_VALUE_RELATIONS,_MAX_MODAL_DISMISS_CONTROLS,_MAX_MODAL_OVERLAYS,_MAX_NAVIGATION_TARGETS,_MAX_PAGE_OBSTRUCTIONS,_MAX_RESULT_CONTAINERS,+ _MAX_RESULT_SAMPLE_ROWS,_MAX_SELECT_OPTIONS,+ _MAX_TABLE_HEADERS,_MAX_VISIBLE_TEXT_EXCERPT_CHARS,_MODAL_DISMISS_HINTS,_MODAL_DISMISS_SYMBOLS,@@ -198,6 +201,9 @@ _STRUCTURED_CONST_HEADER = (f"const MAX_FIELDS_PER_FORM={int(_MAX_FIELDS_PER_FORM)};"f"const MAX_NAVIGATION_TARGETS={int(_MAX_NAVIGATION_TARGETS)};"f"const MAX_RESULT_CONTAINERS={int(_MAX_RESULT_CONTAINERS)};"+ f"const MAX_KEY_VALUE_RELATIONS={int(_MAX_KEY_VALUE_RELATIONS)};"+ f"const MAX_TABLE_HEADERS={int(_MAX_TABLE_HEADERS)};"+ f"const MAX_RESULT_SAMPLE_ROWS={int(_MAX_RESULT_SAMPLE_ROWS)};"f"const MAX_SELECT_OPTIONS={int(_MAX_SELECT_OPTIONS)};"f"const MAX_CHALLENGE_CONTROLS={int(_MAX_CHALLENGE_CONTROLS)};"f"const MAX_CLICKABLE_CONTROLS={int(_MAX_CLICKABLE_CONTROLS)};"@@ -405,17 +411,43 @@ for (const el of document.querySelectorAll('button,[role="button"],[data-action]}const resultContainers = [];+let resultContainersTruncated = false;+const selectorMatchCount = (selector) => { if (!selector) return 0; try { return document.querySelectorAll(selector).length; } catch (e) { return 0; } };+const elementVisible = (node) => {+ if (!node || !node.getBoundingClientRect) return false;+ let style; try { style = window.getComputedStyle(node); } catch (e) { return false; }+ const rect = node.getBoundingClientRect();+ return style.display !== 'none' && style.visibility !== 'hidden' && Number.parseFloat(style.opacity || '1') > 0.05 && rect.width > 0 && rect.height > 0;+};const resultRowTextIsContent = (s) => {const text = lower(String(s || '').replace(/\s+/g, ' ').trim());return !!text && !['0 results', 'no matching records', 'no records found', 'no results', 'no results found', 'nothing found'].some((p) => text.includes(p));};const resultEntry = (node, tag) => {- const entry = { tag: tag, id: attr(node, 'id'), selector: selectorFor(node), is_table: tag === 'table' };+ const selector = selectorFor(node);+ const entry = { tag: tag, id: attr(node, 'id'), selector: selector, selector_match_count: selectorMatchCount(selector), visible: elementVisible(node), is_table: tag === 'table' };if (tag === 'table') {- let rows = Array.from(node.querySelectorAll('tbody tr')).filter((r) => r.querySelector('td'));- if (!rows.length) rows = Array.from(node.querySelectorAll('tr')).filter((r) => r.querySelector('td'));- const sampleRows = rows.map((r) => Array.from(r.children || []).map((c) => nodeText(c)).filter(Boolean).join(' ') || nodeText(r)).filter(resultRowTextIsContent).slice(0, 5);- if (sampleRows.length) { entry.row_count = sampleRows.length; entry.sample_rows = sampleRows; }+ let rows = Array.from(node.querySelectorAll(':scope > tbody > tr')).filter((r) => r.querySelector(':scope > td'));+ if (!rows.length) rows = Array.from(node.querySelectorAll(':scope > tr')).filter((r) => r.querySelector(':scope > td'));+ entry.row_count = rows.length;+ entry.rows_truncated = rows.length > MAX_RESULT_SAMPLE_ROWS;+ const headerNodes = Array.from(node.querySelectorAll(':scope > thead > tr > th'));+ const headers = headerNodes.slice(0, MAX_TABLE_HEADERS).map((h, i) => ({ text: nodeText(h), column_index: i })).filter((h) => !!h.text);+ if (headers.length) entry.headers = headers;+ entry.span_free = !node.querySelector('th[colspan],th[rowspan],td[colspan],td[rowspan]');+ entry.nested_table_free = !node.querySelector(':scope table');+ entry.row_selector = selector ? selector + ' > tbody > tr' : '';+ entry.rows = rows.slice(0, MAX_RESULT_SAMPLE_ROWS).map((row, rowIndex) => ({+ row_index: rowIndex,+ visible: elementVisible(row),+ has_row_header: !!row.querySelector(':scope > th'),+ cells: Array.from(row.querySelectorAll(':scope > th, :scope > td')).slice(0, MAX_TABLE_HEADERS).map((cell, columnIndex) => ({+ column_index: columnIndex,+ visible: elementVisible(cell),+ })),+ }));+ const sampleRows = rows.map((r) => Array.from(r.children || []).map((c) => nodeText(c)).filter(Boolean).join(' ') || nodeText(r)).filter(resultRowTextIsContent).slice(0, MAX_RESULT_SAMPLE_ROWS);+ if (sampleRows.length) entry.sample_rows = sampleRows;} else {const text = nodeText(node);if (text) entry.text_excerpt = text;@@ -423,15 +455,37 @@ const resultEntry = (node, tag) => {return entry;};for (const node of all) {- if (resultContainers.length >= MAX_RESULT_CONTAINERS) break;const tag = (node.tagName || '').toLowerCase();if (SKIP_TAGS.has(tag)) continue;const identity = (attr(node, 'id') + ' ' + classesFor(node).join(' ')).toLowerCase();if (tag === 'table' || RESULT_CONTAINER_HINTS.some((h) => identity.includes(h))) {+ if (resultContainers.length >= MAX_RESULT_CONTAINERS) { resultContainersTruncated = true; break; }resultContainers.push(resultEntry(node, tag));}}+const keyValueRelations = [];+let keyValueRelationsTruncated = false;+const keyValueSkipTags = new Set(['body', 'form', 'html', 'table', 'tbody', 'thead', 'tr']);+for (const node of all) {+ const tag = (node.tagName || '').toLowerCase();+ if (keyValueSkipTags.has(tag) || !elementVisible(node)) continue;+ const children = Array.from(node.children || []);+ if (children.length !== 2) continue;+ if (children[0].children && children[0].children.length > 0) continue;+ const keyText = nodeText(children[0]);+ const valueText = nodeText(children[1]);+ if (!keyText || keyText.length > 120 || !valueText || keyText === valueText) continue;+ if (keyValueRelations.length >= MAX_KEY_VALUE_RELATIONS) { keyValueRelationsTruncated = true; break; }+ const selector = selectorFor(node);+ const matches = selectorMatchCount(selector);+ if (!matches) continue;+ let position = -1;+ try { position = Array.from(document.querySelectorAll(selector)).indexOf(node); } catch (e) { position = -1; }+ if (position < 0) continue;+ keyValueRelations.push({ key_text: keyText, container_selector: selector, container_match_count: matches, container_position: position, value_child_index: 1, direct_child_count: children.length, visible: true, value_visible: elementVisible(children[1]) });+}+const challengeControls = [];const seenChallenge = new Set();for (const node of all) {@@ -511,6 +565,9 @@ return JSON.stringify({forms: forms,navigation_targets: navTargets,result_containers: resultContainers,+ result_containers_truncated: resultContainersTruncated,+ key_value_relations: keyValueRelations,+ key_value_relations_truncated: keyValueRelationsTruncated,clickable_controls: clickableControls,challenge_controls: challengeControls,modal_overlays: modalOverlays,diff --git a/skyvern/forge/sdk/copilot/composition_evidence.py b/skyvern/forge/sdk/copilot/composition_evidence.pyindex 683037039..207cf0f3f 100644--- a/skyvern/forge/sdk/copilot/composition_evidence.py+++ b/skyvern/forge/sdk/copilot/composition_evidence.py@@ -53,6 +53,9 @@ _RESULT_CONTAINER_HINTS: frozenset[str] = frozenset({"result", "results", "recor_MAX_FORMS = 5_MAX_FIELDS_PER_FORM = 20_MAX_RESULT_CONTAINERS = 8+_MAX_KEY_VALUE_RELATIONS = 24+_MAX_TABLE_HEADERS = 12+_MAX_RESULT_SAMPLE_ROWS = 5_MAX_NAVIGATION_TARGETS = 20_MAX_SELECT_OPTIONS = 30_MAX_CHALLENGE_CONTROLS = 8@@ -1124,6 +1127,9 @@ def _empty_evidence(inspected_url: str, current_url: str) -> dict[str, Any]:"forms": [],"navigation_targets": [],"result_containers": [],+ "result_containers_truncated": False,+ "key_value_relations": [],+ "key_value_relations_truncated": False,**_clickable_controls_channel([]),"visible_text_excerpt": "","anti_bot_indicators": [],@@ -1365,7 +1371,7 @@ def _clickable_controls_html(soup: Any, *, used_selectors: set[str]) -> list[dicfor node in candidates:if len(controls) >= _MAX_CLICKABLE_CONTROLS:break- tag_name = str(getattr(node, "name", "") or "").lower()+ tag_name = str(node.name or "").lower()if tag_name in {"script", "style", "noscript"}:continueif hasattr(node, "find_parent") and node.find_parent("form") is not None:@@ -1464,14 +1470,66 @@ def _result_row_text_is_content(text: str) -> bool:return bool(normalized) and not any(pattern in normalized for pattern in _EMPTY_RESULT_TEXT_PATTERNS)-def _result_container_entry(node: Any) -> dict[str, Any]:- tag_name = str(getattr(node, "name", "") or "").lower()+def _selector_match_count(soup: Any, selector: str) -> int:+ try:+ return len(soup.select(selector))+ except Exception:+ return 0+++def _key_value_relations(soup: Any) -> tuple[list[dict[str, Any]], bool]:+ relations: list[dict[str, Any]] = []+ truncated = False+ for node in soup.find_all(True):+ tag_name = str(getattr(node, "name", "") or "").lower()+ if tag_name in {"body", "form", "html", "table", "tbody", "thead", "tr"}:+ continue+ children = [child for child in node.find_all(recursive=False) if child.name]+ if len(children) != 2:+ continue+ if children[0].find(True) is not None:+ continue+ key_text = _schema_text(_node_text(children[0]), 120)+ value_text = _schema_text(_node_text(children[1]), 240)+ if not key_text or not value_text or key_text == value_text:+ continue+ if len(relations) >= _MAX_KEY_VALUE_RELATIONS:+ truncated = True+ break+ selector = _selector_for(node)[:160]+ match_count = _selector_match_count(soup, selector)+ if match_count <= 0:+ continue+ matches = soup.select(selector)+ try:+ position = matches.index(node)+ except ValueError:+ continue+ relations.append(+ {+ "key_text": key_text,+ "container_selector": selector,+ "container_match_count": match_count,+ "container_position": position,+ "value_child_index": 1,+ "direct_child_count": len(children),+ "visible": True,+ "value_visible": True,+ }+ )+ return relations, truncated+++def _result_container_entry(node: Any, *, soup: Any) -> dict[str, Any]:+ tag_name = str(node.name or "").lower()node_id = str(node.get("id") or "")selector = _selector_for(node)[:160]entry: dict[str, Any] = {"tag": tag_name,"id": node_id[:120],"selector": selector,+ "selector_match_count": _selector_match_count(soup, selector),+ "visible": True,}if tag_name == "table":entry["row_selector"] = f"{selector} tbody tr"@@ -1485,10 +1543,32 @@ def _result_container_entry(node: Any) -> dict[str, Any]:data_rows = [row for row in node.select("tbody tr") if row.find("td") is not None]if not data_rows:data_rows = [row for row in node.select("tr") if row.find("td") is not None]+ headers = [+ {"text": _schema_text(_node_text(header), 120), "column_index": index}+ for index, header in enumerate(node.select("thead th")[:_MAX_TABLE_HEADERS])+ if _schema_text(_node_text(header), 120)+ ]+ if headers:+ entry["headers"] = headers+ entry["row_count"] = len(data_rows)+ entry["rows_truncated"] = len(data_rows) > _MAX_RESULT_SAMPLE_ROWS+ entry["span_free"] = node.select_one("th[colspan], th[rowspan], td[colspan], td[rowspan]") is None+ entry["nested_table_free"] = node.find("table") is None+ entry["rows"] = [+ {+ "row_index": row_index,+ "visible": True,+ "has_row_header": row.select_one(":scope > th") is not None,+ "cells": [+ {"column_index": column_index, "visible": True}+ for column_index, _cell in enumerate(row.select(":scope > td")[:_MAX_TABLE_HEADERS])+ ],+ }+ for row_index, row in enumerate(data_rows[:_MAX_RESULT_SAMPLE_ROWS])+ ]sample_rows = [_schema_text(_node_text(row), 240) for row in data_rows]sample_rows = [row for row in sample_rows if _result_row_text_is_content(row)][:5]if sample_rows:- entry["row_count"] = len(sample_rows)entry["sample_rows"] = sample_rowselse:text_excerpt = _schema_text(_node_text(node), 240)@@ -1811,16 +1891,20 @@ def parse_composition_html(html: str, *, inspected_url: str, current_url: str) -navigation_targets.append(nav_entry)result_containers: list[dict[str, Any]] = []+ result_containers_truncated = Falsefor node in all_nodes:- if len(result_containers) >= _MAX_RESULT_CONTAINERS:- breaktag_name = str(getattr(node, "name", "") or "").lower()node_id = str(node.get("id") or "")class_value = node.get("class") or []class_text = " ".join(class_value) if isinstance(class_value, list) else str(class_value)result_identity = f"{node_id} {class_text}".lower()if tag_name == "table" or any(hint in result_identity for hint in _RESULT_CONTAINER_HINTS):- result_containers.append(_result_container_entry(node))+ if len(result_containers) >= _MAX_RESULT_CONTAINERS:+ result_containers_truncated = True+ break+ result_containers.append(_result_container_entry(node, soup=soup))++ key_value_relations, key_value_relations_truncated = _key_value_relations(soup)used_selectors: set[str] = set()for form in forms:@@ -1855,6 +1939,9 @@ def parse_composition_html(html: str, *, inspected_url: str, current_url: str) -"forms": forms,"navigation_targets": navigation_targets,"result_containers": result_containers,+ "result_containers_truncated": result_containers_truncated,+ "key_value_relations": key_value_relations,+ "key_value_relations_truncated": key_value_relations_truncated,**_clickable_controls_channel(clickable_controls),"visible_text_excerpt": _schema_text(visible_text, _MAX_VISIBLE_TEXT_EXCERPT_CHARS),"anti_bot_indicators": anti_bot_indicators,@@ -1993,15 +2080,65 @@ def _structured_result_containers(value: Any) -> list[dict[str, Any]]:"tag": tag_name,"id": _structured_str(node.get("id"))[:120],"selector": selector,+ "selector_match_count": node.get("selector_match_count")+ if isinstance(node.get("selector_match_count"), int)+ else 0,+ "visible": node.get("visible") is True,}+ headers: list[dict[str, Any]] = []+ raw_headers = node.get("headers")+ if isinstance(raw_headers, list):+ for header in raw_headers[:_MAX_TABLE_HEADERS]:+ if not isinstance(header, dict):+ continue+ text = _schema_text(_structured_str(header.get("text")), 120)+ column_index = header.get("column_index")+ if not text or not isinstance(column_index, int) or isinstance(column_index, bool) or column_index < 0:+ continue+ headers.append({"text": text, "column_index": column_index})+ if headers:+ entry["headers"] = headers+ row_count = node.get("row_count")+ if isinstance(row_count, int) and not isinstance(row_count, bool) and row_count >= 0:+ entry["row_count"] = row_count+ entry["rows_truncated"] = node.get("rows_truncated") is True+ entry["span_free"] = node.get("span_free") is True+ entry["nested_table_free"] = node.get("nested_table_free") is True+ rows: list[dict[str, Any]] = []+ raw_rows = node.get("rows")+ if isinstance(raw_rows, list):+ for raw_row in raw_rows[:_MAX_RESULT_SAMPLE_ROWS]:+ if not isinstance(raw_row, dict):+ continue+ row_index = raw_row.get("row_index")+ if not isinstance(row_index, int) or isinstance(row_index, bool) or row_index < 0:+ continue+ cells: list[dict[str, Any]] = []+ raw_cells = raw_row.get("cells")+ if isinstance(raw_cells, list):+ for raw_cell in raw_cells[:_MAX_TABLE_HEADERS]:+ if not isinstance(raw_cell, dict):+ continue+ column_index = raw_cell.get("column_index")+ if not isinstance(column_index, int) or isinstance(column_index, bool) or column_index < 0:+ continue+ cells.append({"column_index": column_index, "visible": raw_cell.get("visible") is True})+ rows.append(+ {+ "row_index": row_index,+ "visible": raw_row.get("visible") is True,+ "has_row_header": raw_row.get("has_row_header") is True,+ "cells": cells,+ }+ )+ entry["rows"] = rowssample_rows = [_schema_text(_structured_str(row), 240)for row in (node.get("sample_rows") or [])if isinstance(row, str) and _result_row_text_is_content(row)][:5]if sample_rows:- row_count = node.get("row_count")- entry["row_count"] = row_count if isinstance(row_count, int) and row_count > 0 else len(sample_rows)+ entry.setdefault("row_count", len(sample_rows))entry["sample_rows"] = sample_rowstext_excerpt = _schema_text(_structured_str(@@ -2012,7 +2149,8 @@ def _structured_result_containers(value: Any) -> list[dict[str, Any]]:if text_excerpt:entry["text_excerpt"] = text_excerptif tag_name == "table" or node.get("is_table") is True:- entry["row_selector"] = f"{selector} tbody tr"+ reported_row_selector = _structured_str(node.get("row_selector"))[:240]+ entry["row_selector"] = reported_row_selector or f"{selector} tbody tr"entry["expand_toggle_candidates"] = [f"{selector} tbody tr [aria-expanded]",f'{selector} tbody tr [role="button"]',@@ -2024,6 +2162,53 @@ def _structured_result_containers(value: Any) -> list[dict[str, Any]]:return containers+def _structured_key_value_relations(value: Any) -> list[dict[str, Any]]:+ relations: list[dict[str, Any]] = []+ if not isinstance(value, list):+ return relations+ for item in value[:_MAX_KEY_VALUE_RELATIONS]:+ if not isinstance(item, dict):+ continue+ key_text = _schema_text(_structured_str(item.get("key_text")), 120)+ selector = _structured_str(item.get("container_selector"))[:160]+ match_count = item.get("container_match_count")+ position = item.get("container_position")+ child_index = item.get("value_child_index")+ child_count = item.get("direct_child_count")+ if not key_text or not selector:+ continue+ if (+ not isinstance(match_count, int)+ or isinstance(match_count, bool)+ or match_count < 0+ or not isinstance(position, int)+ or isinstance(position, bool)+ or position < 0+ or not isinstance(child_index, int)+ or isinstance(child_index, bool)+ or child_index < 0+ or not isinstance(child_count, int)+ or isinstance(child_count, bool)+ or child_count <= child_index+ ):+ continue+ if match_count <= position:+ continue+ relations.append(+ {+ "key_text": key_text,+ "container_selector": selector,+ "container_match_count": match_count,+ "container_position": position,+ "value_child_index": child_index,+ "direct_child_count": child_count,+ "visible": item.get("visible") is True,+ "value_visible": item.get("value_visible") is True,+ }+ )+ return relations++def _structured_clickable_controls(value: Any) -> list[dict[str, Any]]:controls: list[dict[str, Any]] = []if not isinstance(value, list):@@ -2145,6 +2330,7 @@ def parse_composition_structured(data: Any, *, inspected_url: str, current_url:forms = forms[:_MAX_FORMS]navigation_targets = _structured_navigation_targets(data.get("navigation_targets"), base_url=base_url)result_containers = _structured_result_containers(data.get("result_containers"))+ key_value_relations = _structured_key_value_relations(data.get("key_value_relations"))clickable_controls = _structured_clickable_controls(data.get("clickable_controls"))challenge_controls = _structured_challenge_controls(data.get("challenge_controls"))modal_overlays = _structured_modal_overlays(data.get("modal_overlays"))@@ -2177,6 +2363,9 @@ def parse_composition_structured(data: Any, *, inspected_url: str, current_url:"forms": forms,"navigation_targets": navigation_targets,"result_containers": result_containers,+ "result_containers_truncated": data.get("result_containers_truncated") is True,+ "key_value_relations": key_value_relations,+ "key_value_relations_truncated": data.get("key_value_relations_truncated") is True,**_clickable_controls_channel(clickable_controls),"visible_text_excerpt": visible_text,"anti_bot_indicators": anti_bot_indicators,diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.pyindex 1250b4cab..facaf9f45 100644--- a/skyvern/forge/sdk/copilot/context.py+++ b/skyvern/forge/sdk/copilot/context.py@@ -110,6 +110,7 @@ if TYPE_CHECKING:from skyvern.forge.sdk.copilot.completion_criteria_store import CompletionCriteriaTurnStatefrom skyvern.forge.sdk.copilot.diagnosis_repair_contract import DiagnosisRepairContractfrom skyvern.forge.sdk.copilot.narration import NarratorState+ from skyvern.forge.sdk.copilot.output_extraction_plan import FrozenRequestedOutputExtractionCandidatefrom skyvern.forge.sdk.copilot.request_policy import RequestPolicyfrom skyvern.forge.sdk.copilot.schema_incompatibility import SchemaIncompatibilityfrom skyvern.forge.sdk.copilot.turn_context import TurnContextPacket@@ -657,6 +658,7 @@ class CopilotContext(AgentContext):prior_turn_outcome: TurnOutcome | None = Nonelatest_diagnosis_repair_contract: DiagnosisRepairContract | None = Noneblocked_reply_signatures: list[str] = field(default_factory=list)+ requested_output_extraction_candidate: FrozenRequestedOutputExtractionCandidate | None = None# Tool trackingconsecutive_tool_tracker: list[str] = field(default_factory=list)diff --git a/skyvern/forge/sdk/copilot/enforcement.py b/skyvern/forge/sdk/copilot/enforcement.pyindex 927eba23a..677f1cfc2 100644--- a/skyvern/forge/sdk/copilot/enforcement.py+++ b/skyvern/forge/sdk/copilot/enforcement.py@@ -38,6 +38,7 @@ from skyvern.forge.sdk.copilot.build_test_outcome import (from skyvern.forge.sdk.copilot.challenge_evidence import composition_challenge_carrierfrom skyvern.forge.sdk.copilot.code_block_synthesis import (SCOUTED_SPINE_UNDER_BUILD_REASON_CODE,+ freeze_requested_output_extraction_candidate,is_durable_fallback_entry_target,is_generic_entry_opener_click,is_optional_dismissal_only_trajectory,@@ -45,6 +46,7 @@ from skyvern.forge.sdk.copilot.code_block_synthesis import (render_missing_rung_call_sources,render_synthesized_offer_text,synthesize_code_block,+ synthesize_code_block_with_extraction,trajectory_has_browser_fill_interaction,uncovered_required_emitted_interactions,)@@ -87,6 +89,10 @@ from skyvern.forge.sdk.copilot.diagnosis_repair_contract import (from skyvern.forge.sdk.copilot.failure_tracking import PER_TOOL_BUDGET_FAILURE_CATEGORY, normalize_failure_reasonfrom skyvern.forge.sdk.copilot.narration import TransitionKindfrom skyvern.forge.sdk.copilot.output_contracts import OutputContractAdvisoryState+from skyvern.forge.sdk.copilot.output_extraction_plan import (+ RequestedOutputExtractionPlan,+ derive_requested_output_extraction_plan,+)from skyvern.forge.sdk.copilot.output_policy import (completion_criterion_requires_browser_fill_delivery,normalize_response_scaffolding,@@ -2226,8 +2232,13 @@ def _maybe_synthesized_block_offer_msg(ctx: Any) -> dict[str, Any] | None:near-duplicate repeats, but a materially longer scout trajectory can refreshthe deterministic code before the model authors the workflow."""+ extraction_plan = requested_output_extraction_plan(ctx)+ requested_extraction = bool(_requested_output_paths_for_ctx(ctx))+ if requested_extraction and extraction_plan is None:+ return None+ plan_changed = requested_output_extraction_plan_changed(ctx, extraction_plan)reopened_after_failed_run = synthesized_persistence_reopened_after_failed_run(ctx)- reopened = synthesized_persistence_reopened(ctx)+ reopened = synthesized_persistence_reopened(ctx) or plan_changedif getattr(ctx, "update_workflow_called", False) and not reopened:return Noneif normalize_block_authoring_policy(getattr(ctx, "block_authoring_policy", None)) != (@@ -2249,11 +2260,29 @@ def _maybe_synthesized_block_offer_msg(ctx: Any) -> dict[str, Any] | None:and not reopened):return None- synthesized = synthesize_code_block(- trajectory, reached_download_target=getattr(ctx, "reached_download_target", None)+ synthesized = (+ synthesize_code_block_with_extraction(+ trajectory,+ extraction_plan,+ strict_selectors=True,+ reached_download_target=getattr(ctx, "reached_download_target", None),+ )+ if extraction_plan is not None+ else synthesize_code_block(+ trajectory,+ reached_download_target=getattr(ctx, "reached_download_target", None),+ ))if synthesized is None:return None+ if extraction_plan is not None:+ candidate = freeze_requested_output_extraction_candidate(synthesized, extraction_plan, source="generated")+ if candidate is None:+ return None+ existing_candidate = getattr(ctx, "requested_output_extraction_candidate", None)+ if existing_candidate is not None and existing_candidate != candidate and not reopened:+ return None+ ctx.requested_output_extraction_candidate = candidatectx.synthesized_block_offered = Truectx.synthesized_block_offered_trajectory_len = trajectory_len@@ -2400,6 +2429,39 @@ def uncovered_requested_output_paths(ctx: AgentContext) -> set[str]:return {path for path in requested if path not in covered and tokens_by_path.get(path)}+def requested_output_extraction_plan(ctx: AgentContext) -> RequestedOutputExtractionPlan | None:+ requested_paths = _requested_output_paths_for_ctx(ctx)+ if not requested_paths:+ return None+ labels_by_path: dict[str, tuple[str, ...]] = {}+ for criterion in _pre_run_gated_completion_criteria(ctx):+ if criterion.output_path in requested_paths and criterion.outcome.strip():+ labels_by_path.setdefault(criterion.output_path, ())+ labels_by_path[criterion.output_path] += (criterion.outcome.strip(),)+ if set(labels_by_path) != requested_paths:+ return None+ return derive_requested_output_extraction_plan(+ flow_evidence=ctx.flow_evidence,+ labels_by_path=labels_by_path,+ )+++def requested_output_extraction_plan_changed(ctx: AgentContext, current: RequestedOutputExtractionPlan | None) -> bool:+ if current is None or len(ctx.flow_evidence) < 2:+ return False+ labels_by_path: dict[str, tuple[str, ...]] = {}+ requested_paths = _requested_output_paths_for_ctx(ctx)+ for criterion in _pre_run_gated_completion_criteria(ctx):+ if criterion.output_path in requested_paths and criterion.outcome.strip():+ labels_by_path.setdefault(criterion.output_path, ())+ labels_by_path[criterion.output_path] += (criterion.outcome.strip(),)+ previous = derive_requested_output_extraction_plan(+ flow_evidence=ctx.flow_evidence[:-1],+ labels_by_path=labels_by_path,+ )+ return previous is not None and previous.identity != current.identity++def record_scouted_output_coverage(ctx: AgentContext, page_evidence: dict[str, Any]) -> None:coverage_tokens = _requested_output_coverage_tokens(ctx)if not coverage_tokens:@@ -2421,6 +2483,10 @@ def synthesized_trajectory_is_goal_complete(ctx: Any) -> bool:byte-identically, so an entry ``synthesize_code_block`` would drop never counts as complete."""if uncovered_requested_output_paths(ctx):return False+ requested_paths = _requested_output_paths_for_ctx(ctx)+ plan = requested_output_extraction_plan(ctx)+ if requested_paths and (plan is None or not requested_paths.issubset(set(plan.requested_output_paths))):+ return Falsetrajectory = getattr(ctx, "scout_trajectory", None)if not isinstance(trajectory, list) or not trajectory:return Falsediff --git a/skyvern/forge/sdk/copilot/mcp_adapter.py b/skyvern/forge/sdk/copilot/mcp_adapter.pyindex 26875ceda..dca6cbacf 100644--- a/skyvern/forge/sdk/copilot/mcp_adapter.py+++ b/skyvern/forge/sdk/copilot/mcp_adapter.py@@ -79,6 +79,7 @@ _POST_HOOK_CONTEXT_ROLLBACK_FIELDS = ("synthesized_block_offered_trajectory_len","synthesized_block_offered_goal_complete","scouted_output_covered_paths",+ "requested_output_extraction_candidate","synthesized_block_reopened_for_output_coverage","uncovered_output_rescout_context_key","uncovered_output_rescout_steer_key",diff --git a/skyvern/forge/sdk/copilot/output_extraction_plan.py b/skyvern/forge/sdk/copilot/output_extraction_plan.pynew file mode 100644index 000000000..5a6aba2f1--- /dev/null+++ b/skyvern/forge/sdk/copilot/output_extraction_plan.py@@ -0,0 +1,304 @@+"""Contract-owned JIT structural-read plans for requested Copilot outputs."""++from __future__ import annotations++import hashlib+from dataclasses import dataclass+from enum import StrEnum+from typing import Any, TypeGuard+++class LiveReadKind(StrEnum):+ KEY_VALUE = "key_value"+ TABLE_COLUMN = "table_column"+++@dataclass(frozen=True, slots=True)+class RevealAnchor:+ selector: str = ""+ role: str = ""+ name: str = ""++ def __post_init__(self) -> None:+ if bool(self.selector) == bool(self.role and self.name):+ raise ValueError("Reveal anchor must contain exactly one selector or role/name pair")+++@dataclass(frozen=True, slots=True)+class LiveReadBinding:+ output_path: str+ kind: LiveReadKind+ selector: str+ selector_count: int+ selector_index: int+ child_index: int = 0+ child_count: int = 0+ row_selector: str = ""+ row_count: int = 0+ column_index: int = 0+ relation_label: str = ""+ headers: tuple[str, ...] = ()+ row_cell_counts: tuple[int, ...] = ()+ row_identities: tuple[str, ...] = ()+++@dataclass(frozen=True, slots=True)+class RequestedOutputExtractionPlan:+ requested_output_paths: tuple[str, ...]+ observation_step: int+ observation_identity: str+ reveal: RevealAnchor+ live_reads: tuple[LiveReadBinding, ...]+ identity: str+++@dataclass(frozen=True, slots=True)+class FrozenRequestedOutputExtractionCandidate:+ plan_identity: str+ observation_identity: str+ requested_output_paths: tuple[str, ...]+ reveal: RevealAnchor+ interaction_code: str+ extraction_code: str+ source: str+ admission_result: str+ fingerprint: str+++def _stable_identity(value: str) -> str:+ return hashlib.sha256(value.encode("utf-8")).hexdigest()+++def _is_int(value: Any) -> TypeGuard[int]:+ return isinstance(value, int) and not isinstance(value, bool)+++def _leaf_paths(paths: set[str]) -> set[str]:+ return {+ path+ for path in paths+ if path.startswith("output.")+ and not any(+ other != path and (other.startswith(f"{path}.") or other.startswith(f"{path}[]")) for other in paths+ )+ }+++def output_path_segments(path: str) -> tuple[tuple[str, bool], ...]:+ segments: list[tuple[str, bool]] = []+ for raw_part in path.split("."):+ part = raw_part.strip()+ if not part:+ continue+ is_array = "[]" in part+ name = part.replace("[]", "")+ if name:+ segments.append((name, is_array))+ return tuple(segments)+++def _exact_path(label: str, labels_by_path: dict[str, tuple[str, ...]]) -> str | None:+ matches = [path for path, labels in labels_by_path.items() if label in labels]+ return matches[0] if len(matches) == 1 else None+++def _key_value_bindings(packet: dict[str, Any], labels_by_path: dict[str, tuple[str, ...]]) -> list[LiveReadBinding]:+ relations = packet.get("key_value_relations")+ if not isinstance(relations, list):+ return []+ bindings: list[LiveReadBinding] = []+ for relation in relations:+ if (+ not isinstance(relation, dict)+ or relation.get("visible") is not True+ or relation.get("value_visible") is not True+ ):+ continue+ label = relation.get("key_text")+ selector = relation.get("container_selector")+ match_count = relation.get("container_match_count")+ position = relation.get("container_position")+ child_index = relation.get("value_child_index")+ child_count = relation.get("direct_child_count")+ if not isinstance(label, str) or not isinstance(selector, str):+ continue+ if not _is_int(match_count) or not _is_int(position) or not _is_int(child_index) or not _is_int(child_count):+ continue+ if match_count <= position or position < 0 or child_index < 0 or child_count <= child_index:+ continue+ output_path = _exact_path(label, labels_by_path)+ if output_path is None:+ continue+ bindings.append(+ LiveReadBinding(+ output_path,+ LiveReadKind.KEY_VALUE,+ selector,+ match_count,+ position,+ child_index,+ child_count,+ relation_label=label,+ )+ )+ return bindings+++def _table_bindings(packet: dict[str, Any], labels_by_path: dict[str, tuple[str, ...]]) -> list[LiveReadBinding]:+ containers = packet.get("result_containers")+ if not isinstance(containers, list):+ return []+ bindings: list[LiveReadBinding] = []+ for container in containers:+ if (+ not isinstance(container, dict)+ or container.get("visible") is not True+ or container.get("span_free") is not True+ or container.get("nested_table_free") is not True+ ):+ continue+ selector, row_selector = container.get("selector"), container.get("row_selector")+ match_count, row_count = container.get("selector_match_count"), container.get("row_count")+ headers, rows, sample_rows = container.get("headers"), container.get("rows"), container.get("sample_rows")+ if (+ not isinstance(selector, str)+ or not isinstance(row_selector, str)+ or not _is_int(match_count)+ or match_count != 1+ ):+ continue+ if not _is_int(row_count) or row_count <= 0 or container.get("rows_truncated") is not False:+ continue+ if not isinstance(headers, list) or not isinstance(rows, list) or len(rows) != row_count:+ continue+ if (+ not isinstance(sample_rows, list)+ or len(sample_rows) != row_count+ or not all(isinstance(value, str) for value in sample_rows)+ ):+ continue+ header_label_list: list[str] = []+ for header in headers:+ if isinstance(header, dict) and isinstance(header.get("text"), str):+ header_label_list.append(header["text"])+ header_labels = tuple(header_label_list)+ if len(header_labels) != len(headers):+ continue+ row_cell_counts: list[int] = []+ for row_index, row in enumerate(rows):+ if (+ not isinstance(row, dict)+ or row.get("row_index") != row_index+ or row.get("visible") is not True+ or row.get("has_row_header") is not False+ ):+ break+ cells = row.get("cells")+ if (+ not isinstance(cells, list)+ or len(cells) != len(headers)+ or any(+ not isinstance(cell, dict)+ or cell.get("column_index") != column_index+ or cell.get("visible") is not True+ for column_index, cell in enumerate(cells)+ )+ ):+ break+ row_cell_counts.append(len(cells))+ if len(row_cell_counts) != row_count:+ continue+ for header in headers:+ column_index = header.get("column_index") if isinstance(header, dict) else None+ if not isinstance(header, dict) or not isinstance(header.get("text"), str) or not _is_int(column_index):+ continue+ output_path = _exact_path(header["text"], labels_by_path)+ if output_path is None:+ continue+ bindings.append(+ LiveReadBinding(+ output_path,+ LiveReadKind.TABLE_COLUMN,+ selector,+ match_count,+ 0,+ row_selector=row_selector,+ row_count=row_count,+ column_index=column_index,+ relation_label=header["text"],+ headers=header_labels,+ row_cell_counts=tuple(row_cell_counts),+ row_identities=tuple(sample_rows),+ )+ )+ return bindings+++def _array_prefix(path: str) -> tuple[tuple[str, bool], ...]:+ segments = output_path_segments(path)+ for index, (_, is_array) in enumerate(segments):+ if is_array:+ return segments[: index + 1]+ return ()+++def _plan_from_entry(+ entry: dict[str, Any], *, labels_by_path: dict[str, tuple[str, ...]]+) -> RequestedOutputExtractionPlan | None:+ if entry.get("reached_via") != "interaction" or entry.get("had_bounded_schema") is not True:+ return None+ step, packet = entry.get("step"), entry.get("evidence")+ if not _is_int(step) or not isinstance(packet, dict) or packet.get("source_tool") != "scout_interaction":+ return None+ if (+ packet.get("result_containers_truncated") is not False+ or packet.get("key_value_relations_truncated") is not False+ ):+ return None+ if isinstance(packet.get("inspection_warnings"), list) and packet["inspection_warnings"]:+ return None+ selector, role, name = (+ packet.get("interaction_selector"),+ packet.get("interaction_role"),+ packet.get("interaction_accessible_name"),+ )+ if isinstance(selector, str) and selector:+ reveal = RevealAnchor(selector=selector)+ elif isinstance(role, str) and role and isinstance(name, str) and name:+ reveal = RevealAnchor(role=role, name=name)+ else:+ return None+ leaf_paths = _leaf_paths(set(labels_by_path))+ live_reads = _key_value_bindings(packet, labels_by_path) + _table_bindings(packet, labels_by_path)+ by_path: dict[str, list[LiveReadBinding]] = {}+ for binding in live_reads:+ by_path.setdefault(binding.output_path, []).append(binding)+ if any(len(by_path.get(path, [])) != 1 for path in leaf_paths):+ return None+ ordered_reads = tuple(by_path[path][0] for path in sorted(leaf_paths))+ tables_by_array: dict[tuple[tuple[str, bool], ...], set[tuple[str, int]]] = {}+ for binding in ordered_reads:+ prefix = _array_prefix(binding.output_path)+ if prefix:+ tables_by_array.setdefault(prefix, set()).add((binding.selector, binding.selector_index))+ if any(len(tables) != 1 for tables in tables_by_array.values()):+ return None+ # The step is provenance, not structure: an identical re-observation must not+ # invalidate the candidate frozen from the earlier offer.+ observation_identity = _stable_identity(repr((reveal, ordered_reads)))+ identity = _stable_identity(repr((tuple(sorted(labels_by_path)), observation_identity)))+ return RequestedOutputExtractionPlan(+ tuple(sorted(labels_by_path)), step, observation_identity, reveal, ordered_reads, identity+ )+++def derive_requested_output_extraction_plan(+ *, flow_evidence: list[dict[str, Any]], labels_by_path: dict[str, tuple[str, ...]]+) -> RequestedOutputExtractionPlan | None:+ """Derive from one rollback-owned packet; never combine partial observations."""+ if not labels_by_path:+ return None+ for entry in reversed(flow_evidence):+ if isinstance(entry, dict) and entry.get("reached_via") == "interaction":+ return _plan_from_entry(entry, labels_by_path=labels_by_path)+ return Nonediff --git a/skyvern/forge/sdk/copilot/runtime.py b/skyvern/forge/sdk/copilot/runtime.pyindex c90dd4453..8e2e0a166 100644--- a/skyvern/forge/sdk/copilot/runtime.py+++ b/skyvern/forge/sdk/copilot/runtime.py@@ -54,6 +54,7 @@ if TYPE_CHECKING:from skyvern.forge.sdk.copilot.completion_criteria_store import CompletionCriteriaTurnStatefrom skyvern.forge.sdk.copilot.completion_verification import CompletionVerificationResultfrom skyvern.forge.sdk.copilot.context import CodeAuthoringRepairContext+ from skyvern.forge.sdk.copilot.output_extraction_plan import FrozenRequestedOutputExtractionCandidatefrom skyvern.forge.sdk.copilot.reached_download_target import ReachedDownloadTargetfrom skyvern.forge.sdk.copilot.request_policy import RequestPolicyfrom skyvern.forge.sdk.copilot.result_evidence import LoadedResultCompositionEvidence@@ -428,6 +429,7 @@ class AgentContext:synthesized_block_offered: bool = Falsesynthesized_block_offered_trajectory_len: int = 0synthesized_block_offered_goal_complete: bool = False+ requested_output_extraction_candidate: FrozenRequestedOutputExtractionCandidate | None = Nonesynthesized_block_reopened_after_failed_run: bool = Falsesynthesized_block_reopened_for_output_coverage: bool = Falsescouted_output_covered_paths: set[str] = field(default_factory=set)diff --git a/skyvern/forge/sdk/copilot/tools/workflow_update.py b/skyvern/forge/sdk/copilot/tools/workflow_update.pyindex 08decf662..c1714ceff 100644--- a/skyvern/forge/sdk/copilot/tools/workflow_update.py+++ b/skyvern/forge/sdk/copilot/tools/workflow_update.py@@ -13,6 +13,7 @@ from collections.abc import Callable, Iterable, Iterator, Mappingfrom dataclasses import dataclassfrom dataclasses import field as dataclass_fieldfrom dataclasses import replace+from enum import StrEnumfrom typing import Annotated, Any, Literal, NamedTuple, castfrom urllib.parse import urlsplit@@ -52,6 +53,7 @@ from skyvern.forge.sdk.copilot.code_block_security import CodeBlockSecurityErrorfrom skyvern.forge.sdk.copilot.code_block_steps import apply_derived_code_block_steps, fill_code_block_prompts_in_yamlfrom skyvern.forge.sdk.copilot.code_block_synthesis import (_BARE_TAG_RE,+ _INTERNAL_SCOUT_VARS,_SYNTHESIZED_BLOCK_LABEL,CREDENTIAL_FILL_TOOL_NAME,SCOUTED_SPINE_UNDER_BUILD_REASON_CODE,@@ -62,11 +64,13 @@ from skyvern.forge.sdk.copilot.code_block_synthesis import (_parse_role_name,artifact_dependency_id,artifact_observation_ref_id,+ freeze_requested_output_extraction_candidate,locator_selector_literals,missing_rung_text,normalized_locator_expr,render_missing_rung_call_sources,synthesize_code_block,+ synthesize_code_block_with_extraction,uncovered_required_emitted_interactions,)from skyvern.forge.sdk.copilot.composition_evidence import (@@ -86,6 +90,9 @@ from skyvern.forge.sdk.copilot.enforcement import (_CHURN_REASON_CODES,_record_code_authoring_guardrail_reject,_scouted_spine_open_obligation,+ requested_output_extraction_plan,+ requested_output_extraction_plan_changed,+ synthesized_persistence_reopened,synthesized_persistence_reopened_after_failed_run,)from skyvern.forge.sdk.copilot.loop_detection import clear_failed_step_tracker_for_tools_in_ctx@@ -5673,6 +5680,12 @@ class _BrowserBindings(NamedTuple):method_aliases: set[str]+class _BrowserExpressionKind(StrEnum):+ LOCATOR = "locator"+ SCALAR = "scalar"+ OTHER = "other"++def _code_block_label(block: Mapping[str, Any]) -> str:return str(block.get("label") or "").strip() or "unlabeled code block"@@ -5703,6 +5716,15 @@ def _direct_page_method_signature(node: ast.AST) -> str | None:return "page"+def _bounded_nth_constant(node: ast.AST) -> bool:+ return (+ isinstance(node, ast.Constant)+ and isinstance(node.value, int)+ and not isinstance(node.value, bool)+ and 0 <= node.value <= 10_000+ )++def _browser_mutation_signature_for_call(node: ast.Call) -> _BrowserMutationSignature | None:func = node.funcif not isinstance(func, ast.Attribute):@@ -5828,18 +5850,104 @@ def _is_dynamic_browser_dispatch(node: ast.Call, bindings: _BrowserBindings) ->return _expr_contains_browser_receiver(func.args[0], bindings)+def _browser_expression_kind(node: ast.AST, bindings: _BrowserBindings) -> _BrowserExpressionKind:+ # Alias trust is intentionally limited to compiler-owned scout variables;+ # submitted-code aliases never gain browser-read authority by assignment.+ if isinstance(node, ast.Name) and node.id in bindings.locator_aliases and node.id in _INTERNAL_SCOUT_VARS:+ return _BrowserExpressionKind.LOCATOR+ if _direct_locator_receiver_signature(node) is not None:+ return _BrowserExpressionKind.LOCATOR+ if isinstance(node, ast.Attribute) and node.attr in {"first", "last"}:+ if _browser_expression_kind(node.value, bindings) == _BrowserExpressionKind.LOCATOR:+ return _BrowserExpressionKind.LOCATOR+ if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute):+ receiver_kind = _browser_expression_kind(node.func.value, bindings)+ if (+ node.func.attr in {"locator", "get_by_role", "frame_locator"}+ and receiver_kind == _BrowserExpressionKind.LOCATOR+ ):+ return _BrowserExpressionKind.LOCATOR+ if node.func.attr == "nth" and receiver_kind == _BrowserExpressionKind.LOCATOR and len(node.args) == 1:+ index = node.args[0]+ if _bounded_nth_constant(index):+ return _BrowserExpressionKind.LOCATOR+ if node.func.attr in _LOCATOR_READ_METHODS and receiver_kind == _BrowserExpressionKind.LOCATOR:+ return _BrowserExpressionKind.SCALAR+ if (+ node.func.attr in _PAGE_READ_METHODS+ and isinstance(node.func.value, ast.Name)+ and node.func.value.id == "page"+ ):+ return _BrowserExpressionKind.SCALAR+ if (+ node.func.attr+ in {+ "casefold",+ "endswith",+ "join",+ "lower",+ "lstrip",+ "replace",+ "rsplit",+ "rstrip",+ "split",+ "startswith",+ "strip",+ "upper",+ }+ and receiver_kind == _BrowserExpressionKind.SCALAR+ ):+ return _BrowserExpressionKind.SCALAR+ if isinstance(node, ast.Await):+ return _browser_expression_kind(node.value, bindings)+ if (+ isinstance(node, ast.Call)+ and isinstance(node.func, ast.Name)+ and node.func.id+ in {+ "all",+ "any",+ "bool",+ "dict",+ "enumerate",+ "float",+ "int",+ "len",+ "list",+ "max",+ "min",+ "range",+ "set",+ "sorted",+ "str",+ "sum",+ "tuple",+ }+ ):+ values = [*node.args, *(keyword.value for keyword in node.keywords)]+ kinds = [_browser_expression_kind(value, bindings) for value in values]+ if (+ any(kind == _BrowserExpressionKind.SCALAR for kind in kinds)+ and all(kind != _BrowserExpressionKind.LOCATOR for kind in kinds)+ and all(+ kind == _BrowserExpressionKind.SCALAR or not _expr_contains_browser_receiver(value, bindings)+ for value, kind in zip(values, kinds, strict=True)+ )+ ):+ return _BrowserExpressionKind.SCALAR+ return _BrowserExpressionKind.OTHER++def _is_allowed_browser_read_call(node: ast.Call, bindings: _BrowserBindings) -> bool:func = node.funcif not isinstance(func, ast.Attribute):return Falseif func.attr in _PAGE_FACTORY_METHODS:- return isinstance(func.value, ast.Name) and func.value.id == "page"+ return _browser_expression_kind(node, bindings) == _BrowserExpressionKind.LOCATORif func.attr in _PAGE_READ_METHODS and isinstance(func.value, ast.Name) and func.value.id == "page":return Trueif func.attr in _LOCATOR_READ_METHODS:- return _direct_locator_receiver_signature(func.value) is not None or (- isinstance(func.value, ast.Name) and func.value.id in bindings.locator_aliases- )+ return _browser_expression_kind(func.value, bindings) == _BrowserExpressionKind.LOCATORreturn False@@ -5853,7 +5961,11 @@ def _browser_surface_for_code(code: str) -> tuple[list[_BrowserMutationSignaturepage_aliases = _collect_page_aliases(tree)locator_aliases = _collect_locator_aliases(tree)method_aliases = _collect_browser_method_aliases(tree, page_aliases, locator_aliases)- bindings = _BrowserBindings(page_aliases, locator_aliases, method_aliases)+ bindings = _BrowserBindings(+ page_aliases,+ locator_aliases,+ method_aliases,+ )# ast.walk yields breadth-first; the coverage scan (uncovered_required_emitted_interactions)# matches draft calls as an ordered subsequence, so enumerate Call nodes in source order or a# nested-but-present rung is falsely reported uncovered.@@ -5875,6 +5987,11 @@ def _browser_surface_for_code(code: str) -> tuple[list[_BrowserMutationSignatureif _is_dynamic_browser_dispatch(node, bindings):ambiguous.append(_call_name(node))continue+ if _browser_expression_kind(node, bindings) in {+ _BrowserExpressionKind.LOCATOR,+ _BrowserExpressionKind.SCALAR,+ }:+ continueif _is_allowed_browser_read_call(node, bindings):continuefunc = node.func@@ -6351,7 +6468,6 @@ def _selector_refines(bare: str, candidate: str) -> bool:and bool(candidate_name)and not candidate_suffix)-if not _BARE_TAG_RE.match(bare):return Falseif not candidate.startswith(bare) or _is_positional_selector(candidate):@@ -7479,18 +7595,50 @@ def _maybe_impose_synthesized_code_block(workflow_yaml: str, ctx: AgentContext)if not _submitted_code_block_changed(code_block, prior_yaml):return _SynthesizedCodeImpositionResult(workflow_yaml=workflow_yaml)- synthesized = synthesize_code_block(- scout_trajectory,- strict_selectors=True,- reached_download_target=ctx.reached_download_target,+ extraction_plan = requested_output_extraction_plan(ctx)+ extraction_candidate_refresh_allowed = synthesized_persistence_reopened(+ ctx+ ) or requested_output_extraction_plan_changed(ctx, extraction_plan)+ synthesized = (+ synthesize_code_block_with_extraction(+ scout_trajectory,+ extraction_plan,+ strict_selectors=True,+ reached_download_target=ctx.reached_download_target,+ )+ if extraction_plan is not None+ else synthesize_code_block(+ scout_trajectory,+ strict_selectors=True,+ reached_download_target=ctx.reached_download_target,+ ))if synthesized is None:return _SynthesizedCodeImpositionResult(workflow_yaml=workflow_yaml,violations=["Unable to impose synthesized code block: scout trajectory produced no runnable code."],)+ if extraction_plan is not None:+ candidate = freeze_requested_output_extraction_candidate(synthesized, extraction_plan, source="generated")+ if candidate is None:+ return _SynthesizedCodeImpositionResult(+ workflow_yaml=workflow_yaml,+ violations=["Unable to impose synthesized code block: extraction candidate is incomplete."],+ )+ existing_candidate = ctx.requested_output_extraction_candidate+ if (+ existing_candidate is not None+ and existing_candidate != candidate+ and not extraction_candidate_refresh_allowed+ ):+ return _SynthesizedCodeImpositionResult(+ workflow_yaml=workflow_yaml,+ violations=["Unable to impose synthesized code block: extraction candidate identity changed."],+ )+ ctx.requested_output_extraction_candidate = candidate- if _separated_spine_already_imposed(code_blocks, code_block, synthesized.code):+ synthesized_spine_code = synthesized.interaction_code or synthesized.code+ if _separated_spine_already_imposed(code_blocks, code_block, synthesized_spine_code):return _SynthesizedCodeImpositionResult(workflow_yaml=workflow_yaml)diagnostics = synthesized.diagnostics@@ -7530,13 +7678,27 @@ def _maybe_impose_synthesized_code_block(workflow_yaml: str, ctx: AgentContext)code_blocks=code_blocks,selected_code_block=code_block,submitted_selected_code=submitted_code,- synthesized_code=synthesized.code,+ synthesized_code=synthesized_spine_code,prior_yaml=prior_yaml,synthesized_diagnostics=diagnostics,)violations.extend(surface_validation.violations)ambiguous_reject_present = any(record.kind == "ambiguous" for record in surface_validation.provenance)- extraction_suffix = _submitted_suffix_after_synthesized_code(submitted_code, synthesized.code)+ submitted_extraction_suffix = _submitted_suffix_after_synthesized_code(submitted_code, synthesized_spine_code)+ extraction_suffix = submitted_extraction_suffix or synthesized.extraction_code+ extraction_candidate_source = "submitted" if submitted_extraction_suffix else "generated"+ if (+ submitted_extraction_suffix+ and synthesized.extraction_code+ and submitted_extraction_suffix.strip() != synthesized.extraction_code.strip()+ ):+ return _SynthesizedCodeImpositionResult(+ workflow_yaml=workflow_yaml,+ violations=[+ "Unable to impose synthesized code block: submitted extraction candidate does not execute the "+ "captured requested-output plan recipe."+ ],+ )if extraction_suffix:suffix_mutations, _, suffix_ambiguous = _browser_surface_for_code(extraction_suffix)selected_label = _code_block_label(code_block)@@ -7616,7 +7778,7 @@ def _maybe_impose_synthesized_code_block(workflow_yaml: str, ctx: AgentContext)textwrap.dedent(submitted_code), parameter_reconciliation)reconciled_synthesized_code = _apply_parameter_reconciliation_to_code(- textwrap.dedent(synthesized.code), parameter_reconciliation+ textwrap.dedent(synthesized_spine_code), parameter_reconciliation)submitted_contains_full_spine = _browser_surface_contains_full_action_spine(reconciled_submitted_code, reconciled_synthesized_code@@ -7640,7 +7802,7 @@ def _maybe_impose_synthesized_code_block(workflow_yaml: str, ctx: AgentContext)metadata_declares_goal_valuesand submitted_contains_full_spineand not extraction_suffix- and not _is_submitted_code_synthesized_only(submitted_code, synthesized.code)+ and not _is_submitted_code_synthesized_only(submitted_code, synthesized_spine_code))append_selected_extraction = (metadata_declares_goal_values@@ -7667,6 +7829,14 @@ def _maybe_impose_synthesized_code_block(workflow_yaml: str, ctx: AgentContext)"selector_provenance": diagnostics.locator_provenance,"prior_source": prior_source,}+ if extraction_plan is not None:+ substitutions.update(+ {+ "extraction_plan_identity": extraction_plan.identity,+ "extraction_candidate_fingerprint": synthesized.extraction_fingerprint,+ "extraction_candidate_source": extraction_candidate_source,+ }+ )reconciled_extraction_suffix = (_submitted_suffix_after_synthesized_code(reconciled_submitted_code, reconciled_synthesized_code)if preserve_submitted_extraction@@ -7734,9 +7904,9 @@ def _maybe_impose_synthesized_code_block(workflow_yaml: str, ctx: AgentContext)return split_under_buildsubstitutions["separated_browser_stage_count"] = len(durable_stage_codes)else:- imposed_code = textwrap.dedent(submitted_code if preserve_submitted_extraction else synthesized.code).lstrip(- "\n"- )+ imposed_code = textwrap.dedent(+ submitted_code if preserve_submitted_extraction else synthesized_spine_code+ ).lstrip("\n")if extraction_suffix:imposed_code = imposed_code.rstrip() + "\n" + extraction_suffix.rstrip() + "\n"elif append_selected_extraction:@@ -7769,6 +7939,15 @@ def _maybe_impose_synthesized_code_block(workflow_yaml: str, ctx: AgentContext)selected_extraction_metadata_disposition = "self_authored_extraction_preserved"elif extraction_suffix or append_selected_extraction:selected_extraction_metadata_disposition = "sibling_or_suffix_extraction_preserved"+ if extraction_plan is not None:+ LOG.info(+ "copilot_requested_output_extraction_candidate_imposed",+ canonical_paths=list(extraction_plan.requested_output_paths),+ extraction_plan_identity=extraction_plan.identity,+ observation_step=extraction_plan.observation_step,+ candidate_source=extraction_candidate_source,+ candidate_fingerprint=synthesized.extraction_fingerprint,+ )return _SynthesizedCodeImpositionResult(workflow_yaml=yaml.safe_dump(parsed, sort_keys=False),substitutions=substitutions,diff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.pyindex dbcb96807..ed3962076 100644--- a/tests/unit/test_copilot_agent_helpers.py+++ b/tests/unit/test_copilot_agent_helpers.py@@ -23,6 +23,7 @@ from skyvern.forge.sdk.copilot.agent import (_build_built_unverified_exit_result,_build_goal_satisfied_exit_result,_resolve_wrapped_exception_exit_result,+ _synthesized_block_offer_prompt,_verified_workflow_or_none,)from skyvern.forge.sdk.copilot.blocker_signal import CopilotToolBlockerSignal@@ -113,6 +114,61 @@ from tests.unit.copilot_test_helpers import make_verified_goal_contract as _veri_HISTORY_SENTINEL_TS = datetime(2026, 1, 1, tzinfo=timezone.utc)+def test_prompt_offer_compiles_plan_and_refreshes_when_observation_changes() -> None:+ criterion = CompletionCriterion(+ id="record_id",+ outcome="Record Identifier",+ output_path="output.record_id",+ )+ ctx = _ctx()+ ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER+ ctx.request_policy = RequestPolicy(completion_criteria=[criterion])+ ctx.completion_criteria_turn_state = SimpleNamespace(decision=SimpleNamespace(criteria=(criterion,)))+ ctx.copilot_config = CopilotConfig(requested_output_path_aliases={"record identifier": "output.record_id"})+ ctx.scout_trajectory = [+ {"tool_name": "click", "selector": "#show-details", "source_url": "https://example.com/provider"}+ ]++ def packet(step: int) -> dict[str, object]:+ return {+ "step": step,+ "reached_via": "interaction",+ "had_bounded_schema": True,+ "evidence": {+ "source_tool": "scout_interaction",+ "interaction_tool": "click",+ "interaction_selector": "#show-details",+ "inspection_warnings": [],+ "result_containers_truncated": False,+ "key_value_relations_truncated": False,+ "key_value_relations": [+ {+ "key_text": "Record Identifier",+ "container_selector": ".record-kv",+ "container_match_count": 1,+ "container_position": 0,+ "value_child_index": 1,+ "direct_child_count": 2,+ "visible": True,+ "value_visible": True,+ }+ ],+ "result_containers": [],+ },+ }++ ctx.flow_evidence = [packet(2)]+ first = _synthesized_block_offer_prompt(ctx)+ ctx.flow_evidence.append(packet(3))+ second = _synthesized_block_offer_prompt(ctx)++ assert 'page.locator(".record-kv").nth(0)' in first+ assert 'return {"output": {"record_id": _extraction_value_0}}' in first+ assert second == ""+ assert "Extraction plan identity" in first+ assert ctx.requested_output_extraction_candidate is not None++def _history(*pairs: tuple[str, str]) -> list[WorkflowCopilotChatHistoryMessage]:return [WorkflowCopilotChatHistoryMessage(diff --git a/tests/unit/test_copilot_code_block_persist_seam.py b/tests/unit/test_copilot_code_block_persist_seam.pyindex 0a455c029..26e5a9094 100644--- a/tests/unit/test_copilot_code_block_persist_seam.py+++ b/tests/unit/test_copilot_code_block_persist_seam.py@@ -47,7 +47,7 @@ from skyvern.forge.sdk.copilot.code_block_synthesis import (_get_by_role_expr_strict,)from skyvern.forge.sdk.copilot.completion_verification import CompletionVerificationResult, CriterionVerdict-from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy+from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy, CopilotConfigfrom skyvern.forge.sdk.copilot.context import CodeAuthoringRepairContext, CopilotContext, FillCarryfrom skyvern.forge.sdk.copilot.enforcement import (MAX_CODE_AUTHORING_GUARDRAIL_REJECTS,@@ -427,6 +427,114 @@ def _single_code_block(parsed: dict[str, object]) -> dict[str, object]:return blocks[0]+ "code",+ [+ 'value = (await page.locator("#record").inner_text()).strip()',+ 'parts = (await page.locator("#record").inner_text()).split("|")',+ 'value = await page.locator("#rows").nth(2).locator("td").nth(1).inner_text()',+ ],+)+def test_browser_surface_admits_awaited_read_postprocessing_and_bounded_nth(code: str) -> None:+ mutations, unscouted, ambiguous = workflow_update_module._browser_surface_for_code(code)++ assert mutations == []+ assert unscouted == []+ assert ambiguous == []+++ "code",+ [+ 'locator = page.locator("#record")\nvalue = await locator.inner_text()',+ 'read = page.locator("#record").inner_text\nvalue = await read()',+ 'await getattr(page.locator("#record"), "inner_text")()',+ 'await page.locator("#record").nth(-1).inner_text()',+ 'await page.locator("#record").nth(10001).inner_text()',+ 'for index in range(await page.locator("#rows").count()):\n value = await page.locator("#rows").nth(index).inner_text()',+ 'await page.locator("#record").fill("changed")',+ 'values = sorted(await page.locator("#rows").all_text_contents(), key=page.set_default_timeout)',+ ],+)+def test_browser_surface_rejects_alias_dynamic_unbounded_and_mutation_forms(code: str) -> None:+ mutations, unscouted, ambiguous = workflow_update_module._browser_surface_for_code(code)++ assert mutations or unscouted or ambiguous+++def test_plan_backed_imposition_executes_generated_and_submitted_live_read_recipe() -> None:+ ctx = _code_only_ctx()+ _enable_imposition(ctx)+ ctx.request_policy = RequestPolicy(+ completion_criteria=[+ CompletionCriterion(+ id="record_id",+ outcome="Record Identifier",+ output_path="output.record_id",+ )+ ]+ )+ ctx.completion_criteria_turn_state = SimpleNamespace(+ decision=SimpleNamespace(criteria=tuple(ctx.request_policy.completion_criteria))+ )+ ctx.copilot_config = CopilotConfig(requested_output_path_aliases={"record identifier": "output.record_id"})+ ctx.flow_evidence = [+ {+ "step": 2,+ "reached_via": "interaction",+ "had_bounded_schema": True,+ "evidence": {+ "source_tool": "scout_interaction",+ "interaction_tool": "click",+ "interaction_selector": "#search-submit",+ "inspection_warnings": [],+ "result_containers_truncated": False,+ "key_value_relations_truncated": False,+ "key_value_relations": [+ {+ "key_text": "Record Identifier",+ "container_selector": ".record-kv",+ "container_match_count": 1,+ "container_position": 0,+ "value_child_index": 1,+ "direct_child_count": 2,+ "visible": True,+ "value_visible": True,+ }+ ],+ "result_containers": [],+ },+ }+ ]+ submitted = _yaml(+ """+ title: Record lookup+ workflow_definition:+ blocks:+ - block_type: code+ label: extract_record+ code: |+ await page.locator("#search-submit").click()+ """+ )++ result = workflow_update_module._maybe_impose_synthesized_code_block(submitted, ctx)++ assert result.violations == []+ code = str(_single_code_block(parse_workflow_yaml(result.workflow_yaml))["code"])+ assert 'page.locator(".record-kv").nth(0)' in code+ assert code.count('return {"output": {"record_id": _extraction_value_0}}') == 1+ assert result.substitutions["extraction_candidate_source"] == "generated"+ assert result.substitutions["extraction_plan_identity"]++ submitted_result = workflow_update_module._maybe_impose_synthesized_code_block(result.workflow_yaml, ctx)++ assert submitted_result.violations == []+ assert submitted_result.substitutions["extraction_candidate_source"] == "submitted"+ submitted_code = str(_single_code_block(parse_workflow_yaml(submitted_result.workflow_yaml))["code"])+ assert submitted_code == code++def _credential_code_yaml(*, code: str, credential_id: str = "cred_missing") -> str:indented_code = textwrap.indent(textwrap.dedent(code).strip(), " " * 8)return (diff --git a/tests/unit/test_copilot_code_block_synthesis.py b/tests/unit/test_copilot_code_block_synthesis.pyindex 520de8b65..7821ed636 100644--- a/tests/unit/test_copilot_code_block_synthesis.py+++ b/tests/unit/test_copilot_code_block_synthesis.py@@ -33,6 +33,8 @@ from skyvern.forge.sdk.copilot.code_block_synthesis import (is_optional_dismissal_only_trajectory,render_synthesized_offer_text,synthesize_code_block,+ synthesize_code_block_with_extraction,+ synthesize_extraction_suffix,uncovered_required_emitted_interactions,)from skyvern.forge.sdk.copilot.context import (@@ -40,6 +42,12 @@ from skyvern.forge.sdk.copilot.context import (StructuredContext,_fill_carry_from_scout_trajectory,)+from skyvern.forge.sdk.copilot.output_extraction_plan import (+ LiveReadBinding,+ LiveReadKind,+ RequestedOutputExtractionPlan,+ RevealAnchor,+)from skyvern.forge.sdk.copilot.reached_download_target import ReachedDownloadTargetfrom skyvern.forge.sdk.copilot.tools import _normalize_code_artifact_metadatafrom skyvern.forge.sdk.copilot.tools.scouting import _fill_carry_to_interaction@@ -68,6 +76,186 @@ def _interaction(tool_name: str, **fields: Any) -> dict[str, Any]:return {"tool_name": tool_name, **fields}+def _extraction_plan() -> RequestedOutputExtractionPlan:+ return RequestedOutputExtractionPlan(+ requested_output_paths=(+ "output.records[].detail",+ "output.records[].state",+ "output.record_id",+ "output.overall_state",+ ),+ observation_step=9,+ observation_identity="observation-identity",+ reveal=RevealAnchor(selector="#show-details"),+ live_reads=(+ LiveReadBinding(+ output_path="output.record_id",+ kind=LiveReadKind.KEY_VALUE,+ selector=".kv",+ selector_count=2,+ selector_index=0,+ child_index=1,+ child_count=2,+ relation_label="Record Identifier",+ ),+ LiveReadBinding(+ output_path="output.records[].detail",+ kind=LiveReadKind.TABLE_COLUMN,+ selector="#records",+ selector_count=1,+ selector_index=0,+ row_selector="#records > tbody > tr",+ row_count=3,+ column_index=1,+ headers=("Record", "Detail", "State"),+ row_cell_counts=(3, 3, 3),+ row_identities=("One Detail State", "Two Detail State", "Three Detail State"),+ ),+ LiveReadBinding(+ output_path="output.records[].state",+ kind=LiveReadKind.TABLE_COLUMN,+ selector="#records",+ selector_count=1,+ selector_index=0,+ row_selector="#records > tbody > tr",+ row_count=3,+ column_index=2,+ headers=("Record", "Detail", "State"),+ row_cell_counts=(3, 3, 3),+ row_identities=("One Detail State", "Two Detail State", "Three Detail State"),+ ),+ LiveReadBinding(+ output_path="output.overall_state",+ kind=LiveReadKind.KEY_VALUE,+ selector=".kv",+ selector_count=2,+ selector_index=1,+ child_index=1,+ child_count=2,+ relation_label="Overall State",+ ),+ ),+ identity="plan-identity",+ )+++def test_extraction_suffix_compiles_direct_guarded_live_reads() -> None:+ suffix = synthesize_extraction_suffix(_extraction_plan())++ assert suffix is not None+ assert 'page.locator(".kv").nth(0).locator(":scope > *").nth(1).inner_text()' in suffix.code+ assert 'page.locator("#records > tbody > tr").count() != 3' in suffix.code+ assert "for _extraction_index" not in suffix.code+ assert 'page.locator("#records > tbody > tr").nth(2)' in suffix.code+ assert '"overall_state": _extraction_value_1' in suffix.code+ assert '"detail"' in suffix.code+ assert '"state"' in suffix.code+++class _RecipeLocator:+ def __init__(self, page: _RecipePage, selector: str, indices: tuple[int, ...] = ()) -> None:+ self.page = page+ self.selector = selector+ self.indices = indices++ def nth(self, index: int) -> _RecipeLocator:+ return _RecipeLocator(self.page, self.selector, (*self.indices, index))++ def locator(self, selector: str) -> _RecipeLocator:+ return _RecipeLocator(self.page, f"{self.selector}|{selector}", self.indices)++ async def count(self) -> int:+ return self.page.counts[(self.selector, self.indices)]++ async def is_visible(self) -> bool:+ return self.page.visibility.get((self.selector, self.indices), True)++ async def inner_text(self) -> str:+ return self.page.text[(self.selector, self.indices)]+++class _RecipePage:+ def __init__(self) -> None:+ self.visibility: dict[tuple[str, tuple[int, ...]], bool] = {}+ self.counts: dict[tuple[str, tuple[int, ...]], int] = {+ (".kv", ()): 2,+ (".kv|:scope > *", (0,)): 2,+ (".kv|:scope > *", (1,)): 2,+ ("#records", ()): 1,+ ("#records|:scope > thead > tr > th", (0,)): 3,+ ("#records|[colspan], [rowspan]", (0,)): 0,+ ("#records > tbody > tr", ()): 3,+ ("#records|:scope table", (0,)): 0,+ }+ self.text: dict[tuple[str, tuple[int, ...]], str] = {+ (".kv|:scope > *", (0, 0)): "Record Identifier",+ (".kv|:scope > *", (0, 1)): "record-123",+ (".kv|:scope > *", (1, 0)): "Overall State",+ (".kv|:scope > *", (1, 1)): "Ready",+ }+ for index, header in enumerate(("Record", "Detail", "State")):+ self.text[("#records|:scope > thead > tr > th", (0, index))] = header+ for row, identity in enumerate(("One Detail State", "Two Detail State", "Three Detail State")):+ self.counts[("#records > tbody > tr|:scope > th, :scope > td", (row,))] = 3+ self.counts[("#records > tbody > tr|:scope > th", (row,))] = 0+ self.text[("#records > tbody > tr", (row,))] = identity+ self.text[("#records > tbody > tr|:scope > th, :scope > td", (row, 1))] = f"Detail {row}"+ self.text[("#records > tbody > tr|:scope > th, :scope > td", (row, 2))] = "Ready"++ def locator(self, selector: str) -> _RecipeLocator:+ return _RecipeLocator(self, selector)+++async def _execute_recipe(page: _RecipePage) -> dict[str, object]:+ suffix = synthesize_extraction_suffix(_extraction_plan())+ assert suffix is not None+ namespace: dict[str, object] = {}+ exec("async def recipe(page):\n" + textwrap.indent(suffix.code, " "), namespace)+ recipe = namespace["recipe"]+ assert callable(recipe)+ return await recipe(page)+++async def test_generated_recipe_executes_and_fails_closed_on_runtime_drift() -> None:+ page = _RecipePage()+ result = await _execute_recipe(page)+ assert result["output"]["record_id"] == "record-123"+ assert len(result["output"]["records"]) == 3++ page.counts[(".kv", ())] = 3+ with pytest.raises(ValueError, match="scalar selector cardinality"):+ await _execute_recipe(page)++ page.counts[(".kv", ())] = 2+ page.visibility[("#records > tbody > tr|:scope > th, :scope > td", (1, 2))] = False+ with pytest.raises(ValueError, match="cell is no longer visible"):+ await _execute_recipe(page)+++def test_plan_compiler_requires_exact_reveal_and_is_idempotent() -> None:+ trajectory = [+ _interaction(+ "click",+ selector="#show-details",+ role="button",+ accessible_name="Show details",+ source_url="https://example.com/records",+ )+ ]++ first = synthesize_code_block_with_extraction(trajectory, _extraction_plan())+ second = synthesize_code_block_with_extraction(trajectory, _extraction_plan())++ assert first is not None+ assert second is not None+ assert first.code == second.code+ assert first.code.count(".click()") == 1+ assert first.extraction_plan_identity == "plan-identity"+ assert first.extraction_fingerprint == second.extraction_fingerprint+ assert synthesize_code_block_with_extraction([], _extraction_plan()) is None++class TestLocatorSynthesis:def test_role_selector_emits_get_by_role(self) -> None:# A `role=...` selector is an ARIA anchor (ref_to_selector form), not a native CSS path.diff --git a/tests/unit/test_copilot_composition_evidence.py b/tests/unit/test_copilot_composition_evidence.pyindex 358497963..9cb47b0f6 100644--- a/tests/unit/test_copilot_composition_evidence.py+++ b/tests/unit/test_copilot_composition_evidence.py@@ -2217,6 +2217,128 @@ def test_structured_preserves_populated_result_container_content() -> None:assert records["text_excerpt"] == "Record A ready"+def test_structured_preserves_live_key_and_table_binding_shape() -> None:+ payload = {+ "page_title": "Records",+ "forms": [],+ "navigation_targets": [],+ "result_containers": [+ {+ "tag": "table",+ "selector": "#records",+ "selector_match_count": 1,+ "visible": True,+ "span_free": True,+ "nested_table_free": True,+ "row_selector": "#records > tbody > tr",+ "headers": [+ {"text": "Address", "column_index": 0},+ {"text": "Status", "column_index": 1},+ ],+ "row_count": 2,+ "rows_truncated": False,+ "rows": [+ {+ "row_index": row_index,+ "visible": True,+ "has_row_header": False,+ "cells": [+ {"column_index": 0, "visible": True},+ {"column_index": 1, "visible": True},+ ],+ }+ for row_index in range(2)+ ],+ "sample_rows": ["Record One Ready", "Record Two Pending"],+ }+ ],+ "result_containers_truncated": False,+ "key_value_relations": [+ {+ "key_text": "Record Identifier",+ "container_selector": ".kv",+ "container_match_count": 1,+ "container_position": 0,+ "value_child_index": 1,+ "direct_child_count": 2,+ "visible": True,+ "value_visible": True,+ }+ ],+ "key_value_relations_truncated": False,+ "challenge_controls": [],+ "modal_overlays": [],+ "visual_obstruction_candidates": [],+ "visible_text_excerpt": "Record details",+ "anti_bot_indicators": [],+ }++ parsed = parse_composition_structured(+ payload, inspected_url="https://example.com/records", current_url="https://example.com/records"+ )++ assert parsed is not None+ assert parsed["key_value_relations"] == payload["key_value_relations"]+ assert parsed["key_value_relations_truncated"] is False+ assert parsed["result_containers_truncated"] is False+ assert parsed["result_containers"][0]["headers"] == payload["result_containers"][0]["headers"]+ assert parsed["result_containers"][0]["selector_match_count"] == 1+ assert parsed["result_containers"][0]["rows_truncated"] is False+ assert parsed["result_containers"][0]["nested_table_free"] is True+ assert parsed["result_containers"][0]["row_selector"] == "#records > tbody > tr"+ assert parsed["result_containers"][0]["rows"][1]["has_row_header"] is False+ assert parsed["result_containers"][0]["rows"][1]["cells"][1] == {+ "column_index": 1,+ "visible": True,+ }+++def test_html_packet_excludes_hidden_bindings_and_preserves_revealed_structure() -> None:+ details = """+ <div id="details" STYLE>+ <div class="kv"><div>Record Identifier</div><div>record-123</div></div>+ <table id="records">+ <thead><tr><th>Address</th><th>Status</th></tr></thead>+ <tbody><tr><td>Record One</td><td>Ready</td></tr></tbody>+ </table>+ </div>+ """+ hidden = parse_composition_html(+ details.replace("STYLE", 'style="display:none"'),+ inspected_url="https://example.com/records",+ current_url="https://example.com/records",+ )+ revealed = parse_composition_html(+ details.replace("STYLE", ""),+ inspected_url="https://example.com/records",+ current_url="https://example.com/records",+ )++ assert hidden["key_value_relations"] == []+ assert hidden["result_containers"] == []+ assert revealed["key_value_relations"][0]["key_text"] == "Record Identifier"+ table = revealed["result_containers"][0]+ assert table["selector"] == "#records"+ assert table["selector_match_count"] == 1+ assert table["headers"] == [+ {"text": "Address", "column_index": 0},+ {"text": "Status", "column_index": 1},+ ]+ assert table["row_count"] == 1+ assert table["rows_truncated"] is False+ assert table["rows"] == [+ {+ "row_index": 0,+ "visible": True,+ "has_row_header": False,+ "cells": [+ {"column_index": 0, "visible": True},+ {"column_index": 1, "visible": True},+ ],+ }+ ]++def test_structured_detects_modal_overlay_with_dismiss_controls() -> None:payload = {"page_title": "",diff --git a/tests/unit/test_copilot_enforcement_pruning.py b/tests/unit/test_copilot_enforcement_pruning.pyindex 1ebfd75b0..931ae2e41 100644--- a/tests/unit/test_copilot_enforcement_pruning.py+++ b/tests/unit/test_copilot_enforcement_pruning.py@@ -29,7 +29,11 @@ from skyvern.forge.sdk.copilot.build_test_outcome import ()from skyvern.forge.sdk.copilot.code_block_synthesis import SynthesizedCodeBlockfrom skyvern.forge.sdk.copilot.completion_verification import CompletionVerificationResult, CriterionVerdict-from skyvern.forge.sdk.copilot.config import SYNTHESIZED_OFFER_REFRESH_STEP_THRESHOLD, BlockAuthoringPolicy+from skyvern.forge.sdk.copilot.config import (+ SYNTHESIZED_OFFER_REFRESH_STEP_THRESHOLD,+ BlockAuthoringPolicy,+ CopilotConfig,+)from skyvern.forge.sdk.copilot.context import CodeAuthoringRepairContextfrom skyvern.forge.sdk.copilot.enforcement import (KEEP_RECENT_TOOL_OUTPUTS,@@ -123,6 +127,8 @@ class _Ctx:self.synthesized_block_reopened_after_failed_run = Falseself.synthesized_block_reopened_for_output_coverage = Falseself.scouted_output_covered_paths: set[str] = set()+ self.flow_evidence: list[dict[str, object]] = []+ self.copilot_config: CopilotConfig | None = Noneself.uncovered_output_rescout_context_key = Noneself.uncovered_output_rescout_steer_key = Noneself.latest_recorded_build_test_outcome = None@@ -2254,6 +2260,49 @@ class TestScoutOutputCoverageGate:ctx.synthesized_block_offered_goal_complete = synthesized_trajectory_is_goal_complete(ctx)return ctx+ @staticmethod+ def _attach_document_plan(ctx: _Ctx, *, step: int) -> None:+ ctx.copilot_config = CopilotConfig(requested_output_path_aliases={"document name": "output.document_name"})+ ctx.flow_evidence = [+ {+ "step": step,+ "reached_via": "interaction",+ "had_bounded_schema": True,+ "evidence": {+ "source_tool": "scout_interaction",+ "interaction_tool": "click",+ "interaction_selector": "button[data-action='search']",+ "inspection_warnings": [],+ "result_containers_truncated": False,+ "key_value_relations_truncated": False,+ "key_value_relations": [+ {+ "key_text": "Document Name",+ "container_selector": ".document-kv",+ "container_match_count": 1,+ "container_position": 0,+ "value_child_index": 1,+ "direct_child_count": 2,+ "visible": True,+ "value_visible": True,+ }+ ],+ "result_containers": [],+ },+ }+ ]++ def test_post_turn_offer_compiles_plan_recipe(self) -> None:+ ctx = self._authoring_ctx(_criterion("output.document_name", "Document Name"))+ self._attach_document_plan(ctx, step=6)+ ctx.synthesized_block_offered = False++ message = _maybe_synthesized_block_offer_msg(ctx)++ assert message is not None+ assert 'page.locator(".document-kv").nth(0)' in str(message["content"])+ assert 'return {"output": {"document_name": _extraction_value_0}}' in str(message["content"])+def test_empty_output_set_falls_through_to_shape_heuristic(self) -> None:ctx = self._authoring_ctx()assert uncovered_requested_output_paths(ctx) == set()@@ -2291,7 +2340,7 @@ class TestScoutOutputCoverageGate:assert _should_block_mutating_tool_after_synthesized_offer(ctx, "click") is Falseassert synthesized_block_persistence_signal(ctx, "click") is None- def test_value_bearing_container_covers_path_and_force_fires(self) -> None:+ def test_value_bearing_container_coverage_without_plan_does_not_force(self) -> None:ctx = self._authoring_ctx(_criterion("output.document_name", "the order status document name is captured"))page_evidence = {"result_containers": [@@ -2302,8 +2351,8 @@ class TestScoutOutputCoverageGate:assert ctx.scouted_output_covered_paths == {"output.document_name"}assert uncovered_requested_output_paths(ctx) == set()ctx.synthesized_block_offered_goal_complete = synthesized_trajectory_is_goal_complete(ctx)- assert synthesized_trajectory_is_goal_complete(ctx) is True- assert _should_force_synthesized_block_persistence(ctx) is True+ assert synthesized_trajectory_is_goal_complete(ctx) is False+ assert _should_force_synthesized_block_persistence(ctx) is Falsedef test_empty_shell_selector_tokens_do_not_credit(self) -> None:ctx = self._authoring_ctx(_criterion("output.document_name", "the order status document name is captured"))@@ -2345,12 +2394,12 @@ class TestScoutOutputCoverageGate:ctx.completion_criteria_turn_state = SimpleNamespace(decision=None)assert uncovered_requested_output_paths(ctx) == set()- def test_all_generic_token_path_is_exempt_and_falls_through_to_shape(self) -> None:+ def test_all_generic_token_path_still_requires_producer_plan(self) -> None:ctx = self._authoring_ctx(_criterion("output.data", "the data is captured"))assert uncovered_requested_output_paths(ctx) == set()ctx.synthesized_block_offered_goal_complete = synthesized_trajectory_is_goal_complete(ctx)- assert synthesized_trajectory_is_goal_complete(ctx) is True- assert _should_force_synthesized_block_persistence(ctx) is True+ assert synthesized_trajectory_is_goal_complete(ctx) is False+ assert _should_force_synthesized_block_persistence(ctx) is Falsedef test_generic_path_exemption_keeps_specific_path_gating(self) -> None:ctx = self._authoring_ctx(@@ -2542,7 +2591,7 @@ class TestScoutOutputCoverageGate:assert _uncovered_output_reject_admits_evaluate(ctx, "evaluate") is Falseassert consume_uncovered_output_reopen_event(ctx) is False- def test_coverage_reopen_refreshes_synthesized_offer_after_authoring(self, monkeypatch: pytest.MonkeyPatch) -> None:+ def test_coverage_reopen_without_plan_does_not_refresh_offer(self, monkeypatch: pytest.MonkeyPatch) -> None:monkeypatch.setattr("skyvern.forge.sdk.copilot.enforcement.synthesize_code_block",lambda *args, **kwargs: SynthesizedCodeBlock(code="await page.click('button')"),@@ -2551,7 +2600,7 @@ class TestScoutOutputCoverageGate:ctx.update_workflow_called = Trueassert _maybe_synthesized_block_offer_msg(ctx) is Nonectx.synthesized_block_reopened_for_output_coverage = True- assert _maybe_synthesized_block_offer_msg(ctx) is not None+ assert _maybe_synthesized_block_offer_msg(ctx) is Nonedef test_post_hook_failure_rolls_back_coverage_credit(self) -> None:assert "scouted_output_covered_paths" in _POST_HOOK_CONTEXT_ROLLBACK_FIELDSdiff --git a/tests/unit/test_copilot_output_extraction_plan.py b/tests/unit/test_copilot_output_extraction_plan.pynew file mode 100644index 000000000..536846ca3--- /dev/null+++ b/tests/unit/test_copilot_output_extraction_plan.py@@ -0,0 +1,223 @@+from __future__ import annotations++from types import SimpleNamespace++from skyvern.forge.sdk.copilot.composition_evidence import parse_composition_structured+from skyvern.forge.sdk.copilot.mcp_adapter import _restore_post_hook_context, _snapshot_post_hook_context+from skyvern.forge.sdk.copilot.output_extraction_plan import (+ LiveReadKind,+ derive_requested_output_extraction_plan,+)++LABELS_BY_PATH = {+ "output.record_id": ("Record Identifier",),+ "output.records[].detail": ("Detail",),+ "output.records[].state": ("State",),+ "output.overall_state": ("Overall State",),+}+++def _flow_packet(*, visible: bool = True, ambiguous_id: bool = False, truncated: bool = False) -> dict[str, object]:+ id_relation = {+ "key_text": "Record Identifier",+ "container_selector": ".kv",+ "container_match_count": 7,+ "container_position": 0,+ "value_child_index": 1,+ "direct_child_count": 2,+ "visible": visible,+ "value_visible": visible,+ }+ overall_relation = dict(id_relation, key_text="Overall State", container_position=2)+ relations = [id_relation, overall_relation]+ if ambiguous_id:+ relations.append(dict(id_relation, container_position=1))+ return {+ "step": 4,+ "reached_via": "interaction",+ "had_bounded_schema": True,+ "evidence": {+ "source_tool": "scout_interaction",+ "interaction_tool": "click",+ "interaction_selector": "#show-details",+ "inspection_warnings": [],+ "result_containers_truncated": truncated,+ "key_value_relations_truncated": False,+ "key_value_relations": relations,+ "result_containers": [+ {+ "tag": "table",+ "selector": "#records",+ "selector_match_count": 1,+ "visible": visible,+ "span_free": True,+ "nested_table_free": True,+ "headers": [+ {"text": "Record", "column_index": 0},+ {"text": "Detail", "column_index": 1},+ {"text": "State", "column_index": 2},+ ],+ "row_selector": "#records tbody tr",+ "row_count": 3,+ "rows_truncated": False,+ "sample_rows": [f"Record {row_index} Detail State" for row_index in range(3)],+ "rows": [+ {+ "row_index": row_index,+ "visible": True,+ "has_row_header": False,+ "cells": [+ {"column_index": 0, "visible": True},+ {"column_index": 1, "visible": True},+ {"column_index": 2, "visible": True},+ ],+ }+ for row_index in range(3)+ ],+ }+ ],+ },+ }+++def _derive(*, packet: dict[str, object] | None = None):+ return derive_requested_output_extraction_plan(+ flow_evidence=[packet or _flow_packet()],+ labels_by_path=LABELS_BY_PATH,+ )+++def test_derives_complete_plan_from_one_visible_interaction_packet() -> None:+ plan = _derive()++ assert plan is not None+ assert plan.observation_step == 4+ assert plan.reveal.selector == "#show-details"+ assert {binding.output_path for binding in plan.live_reads} == {+ "output.record_id",+ "output.records[].detail",+ "output.records[].state",+ "output.overall_state",+ }+ assert {binding.kind for binding in plan.live_reads} == {LiveReadKind.KEY_VALUE, LiveReadKind.TABLE_COLUMN}+ assert plan.identity+++def test_identical_reobservation_keeps_structural_candidate_identity() -> None:+ first = _flow_packet()+ second = _flow_packet()+ second["step"] = 5++ first_plan = _derive(packet=first)+ second_plan = _derive(packet=second)++ assert first_plan is not None+ assert second_plan is not None+ assert first_plan.observation_step != second_plan.observation_step+ assert first_plan.observation_identity == second_plan.observation_identity+ assert first_plan.identity == second_plan.identity+++def test_derives_table_plan_through_production_structured_normalizer() -> None:+ raw_evidence = _flow_packet()["evidence"]+ parsed = parse_composition_structured(+ raw_evidence,+ inspected_url="https://example.com/records",+ current_url="https://example.com/records",+ )+ assert parsed is not None+ parsed.update(+ source_tool="scout_interaction",+ interaction_tool="click",+ interaction_selector="#show-details",+ )++ plan = derive_requested_output_extraction_plan(+ flow_evidence=[+ {+ "step": 4,+ "reached_via": "interaction",+ "had_bounded_schema": True,+ "evidence": parsed,+ }+ ],+ labels_by_path=LABELS_BY_PATH,+ )++ assert plan is not None+ assert {binding.output_path for binding in plan.live_reads if binding.kind == LiveReadKind.TABLE_COLUMN} == {+ "output.records[].detail",+ "output.records[].state",+ }+++def test_hidden_ambiguous_truncated_and_mixed_packets_fail_closed() -> None:+ assert _derive(packet=_flow_packet(visible=False)) is None+ assert _derive(packet=_flow_packet(ambiguous_id=True)) is None+ assert _derive(packet=_flow_packet(truncated=True)) is None++ hidden_row = _flow_packet()+ hidden_row["evidence"]["result_containers"][0]["rows"][1]["visible"] = False+ assert _derive(packet=hidden_row) is None++ partial_id = _flow_packet()+ partial_id["evidence"] = dict(partial_id["evidence"], result_containers=[])+ partial_table = _flow_packet()+ partial_table["step"] = 5+ partial_table["evidence"] = dict(partial_table["evidence"], key_value_relations=[])+ assert (+ derive_requested_output_extraction_plan(+ flow_evidence=[partial_id, partial_table],+ labels_by_path={key: value for key, value in LABELS_BY_PATH.items() if key != "output.overall_state"},+ )+ is None+ )+ assert (+ derive_requested_output_extraction_plan(+ flow_evidence=[_flow_packet(), partial_table],+ labels_by_path={key: value for key, value in LABELS_BY_PATH.items() if key != "output.overall_state"},+ )+ is None+ )+++def test_only_exact_configured_aliases_bind_observed_labels() -> None:+ fuzzy = _flow_packet()+ fuzzy["evidence"]["key_value_relations"][0]["key_text"] = "Approximate record value"++ assert _derive(packet=fuzzy) is None+++def test_table_identity_coordinate_and_shape_ambiguity_fail_closed() -> None:+ nested = _flow_packet()+ nested["evidence"]["result_containers"][0]["nested_table_free"] = False+ assert _derive(packet=nested) is None++ row_header = _flow_packet()+ row_header["evidence"]["result_containers"][0]["rows"][0]["has_row_header"] = True+ assert _derive(packet=row_header) is None++ shifted_cell = _flow_packet()+ shifted_cell["evidence"]["result_containers"][0]["rows"][1]["cells"][1]["column_index"] = 2+ assert _derive(packet=shifted_cell) is None++ span = _flow_packet()+ span["evidence"]["result_containers"][0]["span_free"] = False+ assert _derive(packet=span) is None+++def test_jit_plan_cannot_see_flow_evidence_rolled_back_after_failed_hook() -> None:+ ctx = SimpleNamespace(flow_evidence=[])+ snapshot = _snapshot_post_hook_context(ctx)+ ctx.flow_evidence.append(_flow_packet())+ assert _derive(packet=ctx.flow_evidence[0]) is not None++ _restore_post_hook_context(ctx, snapshot)++ assert (+ derive_requested_output_extraction_plan(+ flow_evidence=ctx.flow_evidence,+ labels_by_path={"output.record_id": ("Record Identifier",)},+ )+ is None+ )