Skip to content

Automate browser based workflows with AI

SKY-12156: compile grounded Copilot output extraction recipes (#7310)

Andrew Neilson9793647dba2e
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.py
index 0fa53fef9..03f99e7b6 100644
--- a/skyvern/forge/sdk/copilot/agent.py
+++ b/skyvern/forge/sdk/copilot/agent.py
@@ -54,9 +54,11 @@ from skyvern.forge.sdk.copilot.build_test_outcome import (
)
from skyvern.forge.sdk.copilot.code_block_preflight import SANDBOX_UNRESOLVED_NAME_REASON_CODE
from skyvern.forge.sdk.copilot.code_block_synthesis import (
+ freeze_requested_output_extraction_candidate,
is_optional_dismissal_only_trajectory,
render_synthesized_offer_text,
synthesize_code_block,
+ synthesize_code_block_with_extraction,
trajectory_has_browser_fill_interaction,
)
from skyvern.forge.sdk.copilot.completion_criteria_store import (
@@ -64,6 +66,7 @@ from skyvern.forge.sdk.copilot.completion_criteria_store import (
apply_requested_output_producer_floor,
build_turn_state,
reconcile_completion_criteria,
+ requested_output_paths,
)
from skyvern.forge.sdk.copilot.completion_verification import only_structural_requested_output_abstentions
from skyvern.forge.sdk.copilot.config import (
@@ -97,6 +100,8 @@ from skyvern.forge.sdk.copilot.enforcement import (
log_scouted_spine_unresolved_at_turn_halt,
outcome_fully_verified,
recycle_admits_present_completion_contract_ask,
+ requested_output_extraction_plan,
+ requested_output_extraction_plan_changed,
synthesized_persistence_reopened,
synthesized_persistence_reopened_after_failed_run,
synthesized_trajectory_is_goal_complete,
@@ -1029,11 +1034,23 @@ def _synthesized_block_offer_prompt(ctx: CopilotContext | None) -> str:
trajectory_len = len(ctx.scout_trajectory)
previous_offer_len = ctx.synthesized_block_offered_trajectory_len
trajectory_goal_complete = synthesized_trajectory_is_goal_complete(ctx)
+ extraction_plan = requested_output_extraction_plan(ctx)
+ request_policy = getattr(ctx, "request_policy", None)
+ if (
+ request_policy is not None
+ and requested_output_paths(request_policy.completion_criteria)
+ and extraction_plan is None
+ ):
+ LOG.debug("copilot_synthesized_block_offer_skipped", reason="extraction_plan_unavailable")
+ return ""
+ plan_changed = requested_output_extraction_plan_changed(ctx, extraction_plan)
+ reopened = reopened or plan_changed
if (
ctx.synthesized_block_offered
and trajectory_len < previous_offer_len + SYNTHESIZED_OFFER_REFRESH_STEP_THRESHOLD
and (not trajectory_goal_complete or getattr(ctx, "synthesized_block_offered_goal_complete", False))
and not reopened
+ and not plan_changed
):
LOG.debug(
"copilot_synthesized_block_offer_skipped",
@@ -1055,7 +1072,19 @@ def _synthesized_block_offer_prompt(ctx: CopilotContext | None) -> str:
trajectory_len=len(ctx.scout_trajectory),
fill_step_count=fill_step_count,
)
- synthesized = synthesize_code_block(ctx.scout_trajectory, reached_download_target=ctx.reached_download_target)
+ synthesized = (
+ synthesize_code_block_with_extraction(
+ ctx.scout_trajectory,
+ extraction_plan,
+ strict_selectors=True,
+ reached_download_target=getattr(ctx, "reached_download_target", None),
+ )
+ if extraction_plan is not None
+ else synthesize_code_block(
+ ctx.scout_trajectory,
+ reached_download_target=getattr(ctx, "reached_download_target", None),
+ )
+ )
if synthesized is None:
LOG.debug(
"copilot_synthesized_block_offer_skipped",
@@ -1063,6 +1092,15 @@ def _synthesized_block_offer_prompt(ctx: CopilotContext | None) -> str:
trajectory_len=len(ctx.scout_trajectory),
)
return ""
+ if extraction_plan is not None:
+ candidate = freeze_requested_output_extraction_candidate(synthesized, extraction_plan, source="generated")
+ if candidate is None:
+ return ""
+ existing_candidate = ctx.requested_output_extraction_candidate
+ if existing_candidate is not None and existing_candidate != candidate and not reopened:
+ LOG.warning("copilot_requested_output_extraction_candidate_mismatch")
+ return ""
+ ctx.requested_output_extraction_candidate = candidate
ctx.synthesized_block_offered = True
ctx.synthesized_block_offered_trajectory_len = trajectory_len
ctx.synthesized_block_offered_goal_complete = trajectory_goal_complete
diff --git a/skyvern/forge/sdk/copilot/code_block_synthesis.py b/skyvern/forge/sdk/copilot/code_block_synthesis.py
index b239198d0..5ef20fc67 100644
--- a/skyvern/forge/sdk/copilot/code_block_synthesis.py
+++ b/skyvern/forge/sdk/copilot/code_block_synthesis.py
@@ -9,6 +9,7 @@ that runs on the raw ``page`` object the copilot code block executes against.
from __future__ import annotations
import ast
+import hashlib
import io
import json
import keyword
@@ -21,6 +22,13 @@ from typing import Any
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
from skyvern.forge.sdk.copilot.composition_evidence import SCOUT_INTERACTION_EVIDENCE_TOOL
+from skyvern.forge.sdk.copilot.output_extraction_plan import (
+ FrozenRequestedOutputExtractionCandidate,
+ LiveReadBinding,
+ LiveReadKind,
+ RequestedOutputExtractionPlan,
+ output_path_segments,
+)
from skyvern.forge.sdk.copilot.reached_download_target import ReachedDownloadTarget
from skyvern.utils.strings import escape_code_fences
@@ -176,6 +184,22 @@ class SynthesizedCodeBlock:
notes: list[str] = field(default_factory=list)
diagnostics: SynthesisDiagnostics = field(default_factory=SynthesisDiagnostics)
steps: list[dict[str, Any]] = field(default_factory=list)
+ interaction_code: str = ""
+ extraction_code: str = ""
+ extraction_fingerprint: str = ""
+ extraction_plan_identity: str = ""
+
+
+@dataclass(frozen=True, slots=True)
+class SynthesizedExtractionSuffix:
+ code: str
+ fingerprint: str
+
+
+@dataclass
+class _ExtractionReturnNode:
+ children: dict[str, _ExtractionReturnNode] = field(default_factory=dict)
+ value_expression: str = ""
# str.splitlines() and several parsers treat these codepoints as line boundaries, so a raw one in a
@@ -1235,6 +1259,188 @@ def render_missing_rung_call_sources(uncovered: Sequence[Mapping[str, Any]]) ->
return "Missing rung source to reuse verbatim:\n```python\n" + "\n".join(sources) + "\n```"
+def _return_node_expression(node: _ExtractionReturnNode) -> str:
+ if node.value_expression:
+ return node.value_expression
+ items = ", ".join(
+ f"{json.dumps(key)}: {_return_node_expression(child)}" for key, child in sorted(node.children.items())
+ )
+ return "{" + items + "}"
+
+
+def _set_return_expression(root: _ExtractionReturnNode, segments: Sequence[tuple[str, bool]], expression: str) -> None:
+ current = root
+ for name, _is_array in segments:
+ current = current.children.setdefault(name, _ExtractionReturnNode())
+ current.value_expression = expression
+
+
+def _array_prefix(binding: LiveReadBinding) -> tuple[tuple[str, bool], ...]:
+ segments = output_path_segments(binding.output_path)
+ for index, (_name, is_array) in enumerate(segments):
+ if is_array:
+ return segments[: index + 1]
+ return ()
+
+
+def synthesize_extraction_suffix(plan: RequestedOutputExtractionPlan) -> SynthesizedExtractionSuffix | None:
+ if not plan.live_reads:
+ return None
+ lines: list[str] = []
+ return_root = _ExtractionReturnNode()
+ scalar_bindings = [binding for binding in plan.live_reads if binding.kind == LiveReadKind.KEY_VALUE]
+ for index, binding in enumerate(scalar_bindings):
+ variable = f"_extraction_value_{index}"
+ container = f"page.locator({json.dumps(binding.selector)})"
+ target = f"{container}.nth({binding.selector_index})"
+ children = f'{target}.locator(":scope > *")'
+ lines.extend(
+ [
+ f"if await {container}.count() != {binding.selector_count}:",
+ f'{_INDENT}raise ValueError("Observed scalar selector cardinality changed")',
+ f"if not await {target}.is_visible():",
+ f'{_INDENT}raise ValueError("Observed scalar relation is no longer visible")',
+ f"if await {children}.count() != {binding.child_count}:",
+ f'{_INDENT}raise ValueError("Observed scalar direct-child shape changed")',
+ f"if not await {children}.nth(0).is_visible():",
+ f'{_INDENT}raise ValueError("Observed scalar label is no longer visible")',
+ f"if (await {children}.nth(0).inner_text()).strip() != {json.dumps(binding.relation_label)}:",
+ f'{_INDENT}raise ValueError("Observed scalar label changed")',
+ f"if not await {children}.nth({binding.child_index}).is_visible():",
+ f'{_INDENT}raise ValueError("Observed scalar value is no longer visible")',
+ f"{variable} = (await {children}.nth({binding.child_index}).inner_text()).strip()",
+ ]
+ )
+ _set_return_expression(return_root, output_path_segments(binding.output_path), variable)
+
+ table_groups: dict[tuple[str, int, tuple[tuple[str, bool], ...]], list[LiveReadBinding]] = {}
+ for binding in plan.live_reads:
+ if binding.kind != LiveReadKind.TABLE_COLUMN:
+ continue
+ prefix = _array_prefix(binding)
+ if not prefix or not binding.row_selector or binding.row_count <= 0:
+ return None
+ table_groups.setdefault((binding.row_selector, binding.row_count, prefix), []).append(binding)
+ for group_index, ((row_selector, row_count, prefix), bindings) in enumerate(sorted(table_groups.items())):
+ records_variable = f"_extraction_records_{group_index}"
+ record_root = _ExtractionReturnNode()
+ exemplar = bindings[0]
+ table = f"page.locator({json.dumps(exemplar.selector)})"
+ selected_table = f"{table}.nth({exemplar.selector_index})"
+ rows = f"page.locator({json.dumps(row_selector)})"
+ headers = f'{table}.nth({exemplar.selector_index}).locator(":scope > thead > tr > th")'
+ lines.append(f"if await {table}.count() != {exemplar.selector_count}:")
+ lines.append(f'{_INDENT}raise ValueError("Observed table identity changed")')
+ lines.append(f"if not await {table}.nth({exemplar.selector_index}).is_visible():")
+ lines.append(f'{_INDENT}raise ValueError("Observed table is no longer visible")')
+ lines.append(f'if await {selected_table}.locator(":scope table").count() != 0:')
+ lines.append(f'{_INDENT}raise ValueError("Observed table gained a nested table")')
+ lines.append(f'if await {table}.nth({exemplar.selector_index}).locator("[colspan], [rowspan]").count() != 0:')
+ lines.append(f'{_INDENT}raise ValueError("Observed table gained spanning cells")')
+ lines.append(f"if await {headers}.count() != {len(exemplar.headers)}:")
+ lines.append(f'{_INDENT}raise ValueError("Observed table header cardinality changed")')
+ for header_index, header_text in enumerate(exemplar.headers):
+ lines.append(f"if (await {headers}.nth({header_index}).inner_text()).strip() != {json.dumps(header_text)}:")
+ lines.append(f'{_INDENT}raise ValueError("Observed table header identity changed")')
+ lines.append(f"if await {rows}.count() != {row_count}:")
+ lines.append(f'{_INDENT}raise ValueError("Observed table row count changed")')
+ lines.append(f"{records_variable} = []")
+ for row_index in range(row_count):
+ row = f"{rows}.nth({row_index})"
+ cells = f'{row}.locator(":scope > th, :scope > td")'
+ lines.append(f"if not await {row}.is_visible():")
+ lines.append(f'{_INDENT}raise ValueError("Observed table row is no longer visible")')
+ lines.append(f"if await {cells}.count() != {exemplar.row_cell_counts[row_index]}:")
+ lines.append(f'{_INDENT}raise ValueError("Observed table direct-cell cardinality changed")')
+ lines.append(f'if await {row}.locator(":scope > th").count() != 0:')
+ lines.append(f'{_INDENT}raise ValueError("Observed table row gained a row header")')
+ lines.append(
+ f'if " ".join((await {row}.inner_text()).split()) != {json.dumps(exemplar.row_identities[row_index])}:'
+ )
+ lines.append(f'{_INDENT}raise ValueError("Observed table row identity changed")')
+ for binding_index, binding in enumerate(sorted(bindings, key=lambda item: item.output_path)):
+ value_variable = f"_extraction_cell_{group_index}_{row_index}_{binding_index}"
+ lines.append(f"if not await {cells}.nth({binding.column_index}).is_visible():")
+ lines.append(f'{_INDENT}raise ValueError("Observed table cell is no longer visible")')
+ lines.append(f"{value_variable} = (await {cells}.nth({binding.column_index}).inner_text()).strip()")
+ _set_return_expression(
+ record_root, output_path_segments(binding.output_path)[len(prefix) :], value_variable
+ )
+ lines.append(f"{records_variable}.append({_return_node_expression(record_root)})")
+ _set_return_expression(return_root, prefix, records_variable)
+
+ lines.append(f"return {_return_node_expression(return_root)}")
+ code = "\n".join(lines) + "\n"
+ fingerprint_material = repr((plan.identity, plan.observation_identity, plan.reveal, code))
+ return SynthesizedExtractionSuffix(code=code, fingerprint=hashlib.sha256(fingerprint_material.encode()).hexdigest())
+
+
+def _trajectory_contains_reveal(trajectory: Sequence[Mapping[str, Any]], plan: RequestedOutputExtractionPlan) -> bool:
+ return any(
+ str(interaction.get("tool_name") or "") == "click"
+ and (
+ (bool(plan.reveal.selector) and str(interaction.get("selector") or "") == plan.reveal.selector)
+ or (
+ bool(plan.reveal.role and plan.reveal.name)
+ and str(interaction.get("role") or "") == plan.reveal.role
+ and str(interaction.get("accessible_name") or "") == plan.reveal.name
+ )
+ )
+ for interaction in trajectory
+ )
+
+
+def synthesize_code_block_with_extraction(
+ trajectory: Sequence[Mapping[str, Any]],
+ extraction_plan: RequestedOutputExtractionPlan,
+ *,
+ strict_selectors: bool = False,
+ reached_download_target: ReachedDownloadTarget | None = None,
+) -> SynthesizedCodeBlock | None:
+ if not _trajectory_contains_reveal(trajectory, extraction_plan):
+ return None
+ interaction = synthesize_code_block(
+ trajectory,
+ strict_selectors=strict_selectors,
+ reached_download_target=reached_download_target,
+ )
+ suffix = synthesize_extraction_suffix(extraction_plan)
+ if interaction is None or suffix is None:
+ return None
+ interaction_code = interaction.code.rstrip() + "\n"
+ interaction.code = interaction_code + suffix.code
+ interaction.interaction_code = interaction_code
+ interaction.extraction_code = suffix.code
+ interaction.extraction_fingerprint = suffix.fingerprint
+ interaction.extraction_plan_identity = extraction_plan.identity
+ return interaction
+
+
+def freeze_requested_output_extraction_candidate(
+ synthesized: SynthesizedCodeBlock,
+ plan: RequestedOutputExtractionPlan,
+ *,
+ source: str,
+) -> FrozenRequestedOutputExtractionCandidate | None:
+ if (
+ not synthesized.extraction_code
+ or not synthesized.extraction_fingerprint
+ or synthesized.extraction_plan_identity != plan.identity
+ ):
+ return None
+ return FrozenRequestedOutputExtractionCandidate(
+ plan_identity=plan.identity,
+ observation_identity=plan.observation_identity,
+ requested_output_paths=plan.requested_output_paths,
+ reveal=plan.reveal,
+ interaction_code=synthesized.interaction_code,
+ extraction_code=synthesized.extraction_code,
+ source=source,
+ admission_result="admitted",
+ fingerprint=synthesized.extraction_fingerprint,
+ )
+
+
# Model-owned slots the synthesizer cannot prove; the model fills these.
_FILL_DECLARED_GOAL = "<fill: the durable goal this block accomplishes>"
_FILL_CLAIM_ID = "claim:<fill>"
@@ -1372,16 +1578,25 @@ def render_synthesized_offer_text(
"""Render the offer body the copilot sees for a synthesized block (pure)."""
param_keys = [p.get("key", "") for p in synthesized.parameters if p.get("key") and not p.get("credential_id")]
credential_parameters = [p for p in synthesized.parameters if p.get("key") and p.get("credential_id")]
+ extraction_instruction = (
+ "The same interaction-reached evidence compiled the requested keyed extraction suffix; preserve both "
+ "the reveal and extraction segments VERBATIM."
+ if synthesized.extraction_code
+ else (
+ "Hand-author a contract-shaped keyed structure for any requested outputs, never a flat "
+ "`page.inner_text(...)` / `text_content(...)` blob."
+ )
+ )
parts = [
"SYNTHESIZED CODE BLOCK (offered once). The page interactions you scouted were compiled into a "
"deterministic Playwright snippet. Persist it VERBATIM as a `code` block labeled "
- f"`{_SYNTHESIZED_BLOCK_LABEL}` via update_workflow / update_and_run_blocks; hand-author the "
- "data-capture step it does not cover so the block `return`s a keyed structure (a dict, or an array "
- "of objects for repeated records) — never a flat `page.inner_text(...)` / `text_content(...)` blob.",
+ f"`{_SYNTHESIZED_BLOCK_LABEL}` via update_workflow / update_and_run_blocks. {extraction_instruction}",
"```python",
synthesized.code.rstrip("\n"),
"```",
]
+ if synthesized.extraction_plan_identity:
+ parts.append(f"Extraction plan identity: `{synthesized.extraction_plan_identity}`.")
if param_keys:
default_keys = [
str(p.get("key") or "")
diff --git a/skyvern/forge/sdk/copilot/composition_browser_expressions.py b/skyvern/forge/sdk/copilot/composition_browser_expressions.py
index 843393c45..4535b8a2c 100644
--- a/skyvern/forge/sdk/copilot/composition_browser_expressions.py
+++ b/skyvern/forge/sdk/copilot/composition_browser_expressions.py
@@ -11,12 +11,15 @@ from skyvern.forge.sdk.copilot.composition_evidence import (
_MAX_CLICKABLE_CONTROLS,
_MAX_FIELDS_PER_FORM,
_MAX_FORMS,
+ _MAX_KEY_VALUE_RELATIONS,
_MAX_MODAL_DISMISS_CONTROLS,
_MAX_MODAL_OVERLAYS,
_MAX_NAVIGATION_TARGETS,
_MAX_PAGE_OBSTRUCTIONS,
_MAX_RESULT_CONTAINERS,
+ _MAX_RESULT_SAMPLE_ROWS,
_MAX_SELECT_OPTIONS,
+ _MAX_TABLE_HEADERS,
_MAX_VISIBLE_TEXT_EXCERPT_CHARS,
_MODAL_DISMISS_HINTS,
_MODAL_DISMISS_SYMBOLS,
@@ -198,6 +201,9 @@ _STRUCTURED_CONST_HEADER = (
f"const MAX_FIELDS_PER_FORM={int(_MAX_FIELDS_PER_FORM)};"
f"const MAX_NAVIGATION_TARGETS={int(_MAX_NAVIGATION_TARGETS)};"
f"const MAX_RESULT_CONTAINERS={int(_MAX_RESULT_CONTAINERS)};"
+ f"const MAX_KEY_VALUE_RELATIONS={int(_MAX_KEY_VALUE_RELATIONS)};"
+ f"const MAX_TABLE_HEADERS={int(_MAX_TABLE_HEADERS)};"
+ f"const MAX_RESULT_SAMPLE_ROWS={int(_MAX_RESULT_SAMPLE_ROWS)};"
f"const MAX_SELECT_OPTIONS={int(_MAX_SELECT_OPTIONS)};"
f"const MAX_CHALLENGE_CONTROLS={int(_MAX_CHALLENGE_CONTROLS)};"
f"const MAX_CLICKABLE_CONTROLS={int(_MAX_CLICKABLE_CONTROLS)};"
@@ -405,17 +411,43 @@ for (const el of document.querySelectorAll('button,[role="button"],[data-action]
}
const resultContainers = [];
+let resultContainersTruncated = false;
+const selectorMatchCount = (selector) => { if (!selector) return 0; try { return document.querySelectorAll(selector).length; } catch (e) { return 0; } };
+const elementVisible = (node) => {
+ if (!node || !node.getBoundingClientRect) return false;
+ let style; try { style = window.getComputedStyle(node); } catch (e) { return false; }
+ const rect = node.getBoundingClientRect();
+ return style.display !== 'none' && style.visibility !== 'hidden' && Number.parseFloat(style.opacity || '1') > 0.05 && rect.width > 0 && rect.height > 0;
+};
const resultRowTextIsContent = (s) => {
const text = lower(String(s || '').replace(/\s+/g, ' ').trim());
return !!text && !['0 results', 'no matching records', 'no records found', 'no results', 'no results found', 'nothing found'].some((p) => text.includes(p));
};
const resultEntry = (node, tag) => {
- const entry = { tag: tag, id: attr(node, 'id'), selector: selectorFor(node), is_table: tag === 'table' };
+ const selector = selectorFor(node);
+ const entry = { tag: tag, id: attr(node, 'id'), selector: selector, selector_match_count: selectorMatchCount(selector), visible: elementVisible(node), is_table: tag === 'table' };
if (tag === 'table') {
- let rows = Array.from(node.querySelectorAll('tbody tr')).filter((r) => r.querySelector('td'));
- if (!rows.length) rows = Array.from(node.querySelectorAll('tr')).filter((r) => r.querySelector('td'));
- const sampleRows = rows.map((r) => Array.from(r.children || []).map((c) => nodeText(c)).filter(Boolean).join(' ') || nodeText(r)).filter(resultRowTextIsContent).slice(0, 5);
- if (sampleRows.length) { entry.row_count = sampleRows.length; entry.sample_rows = sampleRows; }
+ let rows = Array.from(node.querySelectorAll(':scope > tbody > tr')).filter((r) => r.querySelector(':scope > td'));
+ if (!rows.length) rows = Array.from(node.querySelectorAll(':scope > tr')).filter((r) => r.querySelector(':scope > td'));
+ entry.row_count = rows.length;
+ entry.rows_truncated = rows.length > MAX_RESULT_SAMPLE_ROWS;
+ const headerNodes = Array.from(node.querySelectorAll(':scope > thead > tr > th'));
+ const headers = headerNodes.slice(0, MAX_TABLE_HEADERS).map((h, i) => ({ text: nodeText(h), column_index: i })).filter((h) => !!h.text);
+ if (headers.length) entry.headers = headers;
+ entry.span_free = !node.querySelector('th[colspan],th[rowspan],td[colspan],td[rowspan]');
+ entry.nested_table_free = !node.querySelector(':scope table');
+ entry.row_selector = selector ? selector + ' > tbody > tr' : '';
+ entry.rows = rows.slice(0, MAX_RESULT_SAMPLE_ROWS).map((row, rowIndex) => ({
+ row_index: rowIndex,
+ visible: elementVisible(row),
+ has_row_header: !!row.querySelector(':scope > th'),
+ cells: Array.from(row.querySelectorAll(':scope > th, :scope > td')).slice(0, MAX_TABLE_HEADERS).map((cell, columnIndex) => ({
+ column_index: columnIndex,
+ visible: elementVisible(cell),
+ })),
+ }));
+ const sampleRows = rows.map((r) => Array.from(r.children || []).map((c) => nodeText(c)).filter(Boolean).join(' ') || nodeText(r)).filter(resultRowTextIsContent).slice(0, MAX_RESULT_SAMPLE_ROWS);
+ if (sampleRows.length) entry.sample_rows = sampleRows;
} else {
const text = nodeText(node);
if (text) entry.text_excerpt = text;
@@ -423,15 +455,37 @@ const resultEntry = (node, tag) => {
return entry;
};
for (const node of all) {
- if (resultContainers.length >= MAX_RESULT_CONTAINERS) break;
const tag = (node.tagName || '').toLowerCase();
if (SKIP_TAGS.has(tag)) continue;
const identity = (attr(node, 'id') + ' ' + classesFor(node).join(' ')).toLowerCase();
if (tag === 'table' || RESULT_CONTAINER_HINTS.some((h) => identity.includes(h))) {
+ if (resultContainers.length >= MAX_RESULT_CONTAINERS) { resultContainersTruncated = true; break; }
resultContainers.push(resultEntry(node, tag));
}
}
+const keyValueRelations = [];
+let keyValueRelationsTruncated = false;
+const keyValueSkipTags = new Set(['body', 'form', 'html', 'table', 'tbody', 'thead', 'tr']);
+for (const node of all) {
+ const tag = (node.tagName || '').toLowerCase();
+ if (keyValueSkipTags.has(tag) || !elementVisible(node)) continue;
+ const children = Array.from(node.children || []);
+ if (children.length !== 2) continue;
+ if (children[0].children && children[0].children.length > 0) continue;
+ const keyText = nodeText(children[0]);
+ const valueText = nodeText(children[1]);
+ if (!keyText || keyText.length > 120 || !valueText || keyText === valueText) continue;
+ if (keyValueRelations.length >= MAX_KEY_VALUE_RELATIONS) { keyValueRelationsTruncated = true; break; }
+ const selector = selectorFor(node);
+ const matches = selectorMatchCount(selector);
+ if (!matches) continue;
+ let position = -1;
+ try { position = Array.from(document.querySelectorAll(selector)).indexOf(node); } catch (e) { position = -1; }
+ if (position < 0) continue;
+ keyValueRelations.push({ key_text: keyText, container_selector: selector, container_match_count: matches, container_position: position, value_child_index: 1, direct_child_count: children.length, visible: true, value_visible: elementVisible(children[1]) });
+}
+
const challengeControls = [];
const seenChallenge = new Set();
for (const node of all) {
@@ -511,6 +565,9 @@ return JSON.stringify({
forms: forms,
navigation_targets: navTargets,
result_containers: resultContainers,
+ result_containers_truncated: resultContainersTruncated,
+ key_value_relations: keyValueRelations,
+ key_value_relations_truncated: keyValueRelationsTruncated,
clickable_controls: clickableControls,
challenge_controls: challengeControls,
modal_overlays: modalOverlays,
diff --git a/skyvern/forge/sdk/copilot/composition_evidence.py b/skyvern/forge/sdk/copilot/composition_evidence.py
index 683037039..207cf0f3f 100644
--- a/skyvern/forge/sdk/copilot/composition_evidence.py
+++ b/skyvern/forge/sdk/copilot/composition_evidence.py
@@ -53,6 +53,9 @@ _RESULT_CONTAINER_HINTS: frozenset[str] = frozenset({"result", "results", "recor
_MAX_FORMS = 5
_MAX_FIELDS_PER_FORM = 20
_MAX_RESULT_CONTAINERS = 8
+_MAX_KEY_VALUE_RELATIONS = 24
+_MAX_TABLE_HEADERS = 12
+_MAX_RESULT_SAMPLE_ROWS = 5
_MAX_NAVIGATION_TARGETS = 20
_MAX_SELECT_OPTIONS = 30
_MAX_CHALLENGE_CONTROLS = 8
@@ -1124,6 +1127,9 @@ def _empty_evidence(inspected_url: str, current_url: str) -> dict[str, Any]:
"forms": [],
"navigation_targets": [],
"result_containers": [],
+ "result_containers_truncated": False,
+ "key_value_relations": [],
+ "key_value_relations_truncated": False,
**_clickable_controls_channel([]),
"visible_text_excerpt": "",
"anti_bot_indicators": [],
@@ -1365,7 +1371,7 @@ def _clickable_controls_html(soup: Any, *, used_selectors: set[str]) -> list[dic
for node in candidates:
if len(controls) >= _MAX_CLICKABLE_CONTROLS:
break
- tag_name = str(getattr(node, "name", "") or "").lower()
+ tag_name = str(node.name or "").lower()
if tag_name in {"script", "style", "noscript"}:
continue
if hasattr(node, "find_parent") and node.find_parent("form") is not None:
@@ -1464,14 +1470,66 @@ def _result_row_text_is_content(text: str) -> bool:
return bool(normalized) and not any(pattern in normalized for pattern in _EMPTY_RESULT_TEXT_PATTERNS)
-def _result_container_entry(node: Any) -> dict[str, Any]:
- tag_name = str(getattr(node, "name", "") or "").lower()
+def _selector_match_count(soup: Any, selector: str) -> int:
+ try:
+ return len(soup.select(selector))
+ except Exception:
+ return 0
+
+
+def _key_value_relations(soup: Any) -> tuple[list[dict[str, Any]], bool]:
+ relations: list[dict[str, Any]] = []
+ truncated = False
+ for node in soup.find_all(True):
+ tag_name = str(getattr(node, "name", "") or "").lower()
+ if tag_name in {"body", "form", "html", "table", "tbody", "thead", "tr"}:
+ continue
+ children = [child for child in node.find_all(recursive=False) if child.name]
+ if len(children) != 2:
+ continue
+ if children[0].find(True) is not None:
+ continue
+ key_text = _schema_text(_node_text(children[0]), 120)
+ value_text = _schema_text(_node_text(children[1]), 240)
+ if not key_text or not value_text or key_text == value_text:
+ continue
+ if len(relations) >= _MAX_KEY_VALUE_RELATIONS:
+ truncated = True
+ break
+ selector = _selector_for(node)[:160]
+ match_count = _selector_match_count(soup, selector)
+ if match_count <= 0:
+ continue
+ matches = soup.select(selector)
+ try:
+ position = matches.index(node)
+ except ValueError:
+ continue
+ relations.append(
+ {
+ "key_text": key_text,
+ "container_selector": selector,
+ "container_match_count": match_count,
+ "container_position": position,
+ "value_child_index": 1,
+ "direct_child_count": len(children),
+ "visible": True,
+ "value_visible": True,
+ }
+ )
+ return relations, truncated
+
+
+def _result_container_entry(node: Any, *, soup: Any) -> dict[str, Any]:
+ tag_name = str(node.name or "").lower()
node_id = str(node.get("id") or "")
selector = _selector_for(node)[:160]
entry: dict[str, Any] = {
"tag": tag_name,
"id": node_id[:120],
"selector": selector,
+ "selector_match_count": _selector_match_count(soup, selector),
+ "visible": True,
}
if tag_name == "table":
entry["row_selector"] = f"{selector} tbody tr"
@@ -1485,10 +1543,32 @@ def _result_container_entry(node: Any) -> dict[str, Any]:
data_rows = [row for row in node.select("tbody tr") if row.find("td") is not None]
if not data_rows:
data_rows = [row for row in node.select("tr") if row.find("td") is not None]
+ headers = [
+ {"text": _schema_text(_node_text(header), 120), "column_index": index}
+ for index, header in enumerate(node.select("thead th")[:_MAX_TABLE_HEADERS])
+ if _schema_text(_node_text(header), 120)
+ ]
+ if headers:
+ entry["headers"] = headers
+ entry["row_count"] = len(data_rows)
+ entry["rows_truncated"] = len(data_rows) > _MAX_RESULT_SAMPLE_ROWS
+ entry["span_free"] = node.select_one("th[colspan], th[rowspan], td[colspan], td[rowspan]") is None
+ entry["nested_table_free"] = node.find("table") is None
+ entry["rows"] = [
+ {
+ "row_index": row_index,
+ "visible": True,
+ "has_row_header": row.select_one(":scope > th") is not None,
+ "cells": [
+ {"column_index": column_index, "visible": True}
+ for column_index, _cell in enumerate(row.select(":scope > td")[:_MAX_TABLE_HEADERS])
+ ],
+ }
+ for row_index, row in enumerate(data_rows[:_MAX_RESULT_SAMPLE_ROWS])
+ ]
sample_rows = [_schema_text(_node_text(row), 240) for row in data_rows]
sample_rows = [row for row in sample_rows if _result_row_text_is_content(row)][:5]
if sample_rows:
- entry["row_count"] = len(sample_rows)
entry["sample_rows"] = sample_rows
else:
text_excerpt = _schema_text(_node_text(node), 240)
@@ -1811,16 +1891,20 @@ def parse_composition_html(html: str, *, inspected_url: str, current_url: str) -
navigation_targets.append(nav_entry)
result_containers: list[dict[str, Any]] = []
+ result_containers_truncated = False
for node in all_nodes:
- if len(result_containers) >= _MAX_RESULT_CONTAINERS:
- break
tag_name = str(getattr(node, "name", "") or "").lower()
node_id = str(node.get("id") or "")
class_value = node.get("class") or []
class_text = " ".join(class_value) if isinstance(class_value, list) else str(class_value)
result_identity = f"{node_id} {class_text}".lower()
if tag_name == "table" or any(hint in result_identity for hint in _RESULT_CONTAINER_HINTS):
- result_containers.append(_result_container_entry(node))
+ if len(result_containers) >= _MAX_RESULT_CONTAINERS:
+ result_containers_truncated = True
+ break
+ result_containers.append(_result_container_entry(node, soup=soup))
+
+ key_value_relations, key_value_relations_truncated = _key_value_relations(soup)
used_selectors: set[str] = set()
for form in forms:
@@ -1855,6 +1939,9 @@ def parse_composition_html(html: str, *, inspected_url: str, current_url: str) -
"forms": forms,
"navigation_targets": navigation_targets,
"result_containers": result_containers,
+ "result_containers_truncated": result_containers_truncated,
+ "key_value_relations": key_value_relations,
+ "key_value_relations_truncated": key_value_relations_truncated,
**_clickable_controls_channel(clickable_controls),
"visible_text_excerpt": _schema_text(visible_text, _MAX_VISIBLE_TEXT_EXCERPT_CHARS),
"anti_bot_indicators": anti_bot_indicators,
@@ -1993,15 +2080,65 @@ def _structured_result_containers(value: Any) -> list[dict[str, Any]]:
"tag": tag_name,
"id": _structured_str(node.get("id"))[:120],
"selector": selector,
+ "selector_match_count": node.get("selector_match_count")
+ if isinstance(node.get("selector_match_count"), int)
+ else 0,
+ "visible": node.get("visible") is True,
}
+ headers: list[dict[str, Any]] = []
+ raw_headers = node.get("headers")
+ if isinstance(raw_headers, list):
+ for header in raw_headers[:_MAX_TABLE_HEADERS]:
+ if not isinstance(header, dict):
+ continue
+ text = _schema_text(_structured_str(header.get("text")), 120)
+ column_index = header.get("column_index")
+ if not text or not isinstance(column_index, int) or isinstance(column_index, bool) or column_index < 0:
+ continue
+ headers.append({"text": text, "column_index": column_index})
+ if headers:
+ entry["headers"] = headers
+ row_count = node.get("row_count")
+ if isinstance(row_count, int) and not isinstance(row_count, bool) and row_count >= 0:
+ entry["row_count"] = row_count
+ entry["rows_truncated"] = node.get("rows_truncated") is True
+ entry["span_free"] = node.get("span_free") is True
+ entry["nested_table_free"] = node.get("nested_table_free") is True
+ rows: list[dict[str, Any]] = []
+ raw_rows = node.get("rows")
+ if isinstance(raw_rows, list):
+ for raw_row in raw_rows[:_MAX_RESULT_SAMPLE_ROWS]:
+ if not isinstance(raw_row, dict):
+ continue
+ row_index = raw_row.get("row_index")
+ if not isinstance(row_index, int) or isinstance(row_index, bool) or row_index < 0:
+ continue
+ cells: list[dict[str, Any]] = []
+ raw_cells = raw_row.get("cells")
+ if isinstance(raw_cells, list):
+ for raw_cell in raw_cells[:_MAX_TABLE_HEADERS]:
+ if not isinstance(raw_cell, dict):
+ continue
+ column_index = raw_cell.get("column_index")
+ if not isinstance(column_index, int) or isinstance(column_index, bool) or column_index < 0:
+ continue
+ cells.append({"column_index": column_index, "visible": raw_cell.get("visible") is True})
+ rows.append(
+ {
+ "row_index": row_index,
+ "visible": raw_row.get("visible") is True,
+ "has_row_header": raw_row.get("has_row_header") is True,
+ "cells": cells,
+ }
+ )
+ entry["rows"] = rows
sample_rows = [
_schema_text(_structured_str(row), 240)
for row in (node.get("sample_rows") or [])
if isinstance(row, str) and _result_row_text_is_content(row)
][:5]
if sample_rows:
- row_count = node.get("row_count")
- entry["row_count"] = row_count if isinstance(row_count, int) and row_count > 0 else len(sample_rows)
+ entry.setdefault("row_count", len(sample_rows))
entry["sample_rows"] = sample_rows
text_excerpt = _schema_text(
_structured_str(
@@ -2012,7 +2149,8 @@ def _structured_result_containers(value: Any) -> list[dict[str, Any]]:
if text_excerpt:
entry["text_excerpt"] = text_excerpt
if tag_name == "table" or node.get("is_table") is True:
- entry["row_selector"] = f"{selector} tbody tr"
+ reported_row_selector = _structured_str(node.get("row_selector"))[:240]
+ entry["row_selector"] = reported_row_selector or f"{selector} tbody tr"
entry["expand_toggle_candidates"] = [
f"{selector} tbody tr [aria-expanded]",
f'{selector} tbody tr [role="button"]',
@@ -2024,6 +2162,53 @@ def _structured_result_containers(value: Any) -> list[dict[str, Any]]:
return containers
+def _structured_key_value_relations(value: Any) -> list[dict[str, Any]]:
+ relations: list[dict[str, Any]] = []
+ if not isinstance(value, list):
+ return relations
+ for item in value[:_MAX_KEY_VALUE_RELATIONS]:
+ if not isinstance(item, dict):
+ continue
+ key_text = _schema_text(_structured_str(item.get("key_text")), 120)
+ selector = _structured_str(item.get("container_selector"))[:160]
+ match_count = item.get("container_match_count")
+ position = item.get("container_position")
+ child_index = item.get("value_child_index")
+ child_count = item.get("direct_child_count")
+ if not key_text or not selector:
+ continue
+ if (
+ not isinstance(match_count, int)
+ or isinstance(match_count, bool)
+ or match_count < 0
+ or not isinstance(position, int)
+ or isinstance(position, bool)
+ or position < 0
+ or not isinstance(child_index, int)
+ or isinstance(child_index, bool)
+ or child_index < 0
+ or not isinstance(child_count, int)
+ or isinstance(child_count, bool)
+ or child_count <= child_index
+ ):
+ continue
+ if match_count <= position:
+ continue
+ relations.append(
+ {
+ "key_text": key_text,
+ "container_selector": selector,
+ "container_match_count": match_count,
+ "container_position": position,
+ "value_child_index": child_index,
+ "direct_child_count": child_count,
+ "visible": item.get("visible") is True,
+ "value_visible": item.get("value_visible") is True,
+ }
+ )
+ return relations
+
+
def _structured_clickable_controls(value: Any) -> list[dict[str, Any]]:
controls: list[dict[str, Any]] = []
if not isinstance(value, list):
@@ -2145,6 +2330,7 @@ def parse_composition_structured(data: Any, *, inspected_url: str, current_url:
forms = forms[:_MAX_FORMS]
navigation_targets = _structured_navigation_targets(data.get("navigation_targets"), base_url=base_url)
result_containers = _structured_result_containers(data.get("result_containers"))
+ key_value_relations = _structured_key_value_relations(data.get("key_value_relations"))
clickable_controls = _structured_clickable_controls(data.get("clickable_controls"))
challenge_controls = _structured_challenge_controls(data.get("challenge_controls"))
modal_overlays = _structured_modal_overlays(data.get("modal_overlays"))
@@ -2177,6 +2363,9 @@ def parse_composition_structured(data: Any, *, inspected_url: str, current_url:
"forms": forms,
"navigation_targets": navigation_targets,
"result_containers": result_containers,
+ "result_containers_truncated": data.get("result_containers_truncated") is True,
+ "key_value_relations": key_value_relations,
+ "key_value_relations_truncated": data.get("key_value_relations_truncated") is True,
**_clickable_controls_channel(clickable_controls),
"visible_text_excerpt": visible_text,
"anti_bot_indicators": anti_bot_indicators,
diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.py
index 1250b4cab..facaf9f45 100644
--- a/skyvern/forge/sdk/copilot/context.py
+++ b/skyvern/forge/sdk/copilot/context.py
@@ -110,6 +110,7 @@ if TYPE_CHECKING:
from skyvern.forge.sdk.copilot.completion_criteria_store import CompletionCriteriaTurnState
from skyvern.forge.sdk.copilot.diagnosis_repair_contract import DiagnosisRepairContract
from skyvern.forge.sdk.copilot.narration import NarratorState
+ from skyvern.forge.sdk.copilot.output_extraction_plan import FrozenRequestedOutputExtractionCandidate
from skyvern.forge.sdk.copilot.request_policy import RequestPolicy
from skyvern.forge.sdk.copilot.schema_incompatibility import SchemaIncompatibility
from skyvern.forge.sdk.copilot.turn_context import TurnContextPacket
@@ -657,6 +658,7 @@ class CopilotContext(AgentContext):
prior_turn_outcome: TurnOutcome | None = None
latest_diagnosis_repair_contract: DiagnosisRepairContract | None = None
blocked_reply_signatures: list[str] = field(default_factory=list)
+ requested_output_extraction_candidate: FrozenRequestedOutputExtractionCandidate | None = None
# Tool tracking
consecutive_tool_tracker: list[str] = field(default_factory=list)
diff --git a/skyvern/forge/sdk/copilot/enforcement.py b/skyvern/forge/sdk/copilot/enforcement.py
index 927eba23a..677f1cfc2 100644
--- a/skyvern/forge/sdk/copilot/enforcement.py
+++ b/skyvern/forge/sdk/copilot/enforcement.py
@@ -38,6 +38,7 @@ from skyvern.forge.sdk.copilot.build_test_outcome import (
from skyvern.forge.sdk.copilot.challenge_evidence import composition_challenge_carrier
from skyvern.forge.sdk.copilot.code_block_synthesis import (
SCOUTED_SPINE_UNDER_BUILD_REASON_CODE,
+ freeze_requested_output_extraction_candidate,
is_durable_fallback_entry_target,
is_generic_entry_opener_click,
is_optional_dismissal_only_trajectory,
@@ -45,6 +46,7 @@ from skyvern.forge.sdk.copilot.code_block_synthesis import (
render_missing_rung_call_sources,
render_synthesized_offer_text,
synthesize_code_block,
+ synthesize_code_block_with_extraction,
trajectory_has_browser_fill_interaction,
uncovered_required_emitted_interactions,
)
@@ -87,6 +89,10 @@ from skyvern.forge.sdk.copilot.diagnosis_repair_contract import (
from skyvern.forge.sdk.copilot.failure_tracking import PER_TOOL_BUDGET_FAILURE_CATEGORY, normalize_failure_reason
from skyvern.forge.sdk.copilot.narration import TransitionKind
from skyvern.forge.sdk.copilot.output_contracts import OutputContractAdvisoryState
+from skyvern.forge.sdk.copilot.output_extraction_plan import (
+ RequestedOutputExtractionPlan,
+ derive_requested_output_extraction_plan,
+)
from skyvern.forge.sdk.copilot.output_policy import (
completion_criterion_requires_browser_fill_delivery,
normalize_response_scaffolding,
@@ -2226,8 +2232,13 @@ def _maybe_synthesized_block_offer_msg(ctx: Any) -> dict[str, Any] | None:
near-duplicate repeats, but a materially longer scout trajectory can refresh
the deterministic code before the model authors the workflow.
"""
+ extraction_plan = requested_output_extraction_plan(ctx)
+ requested_extraction = bool(_requested_output_paths_for_ctx(ctx))
+ if requested_extraction and extraction_plan is None:
+ return None
+ plan_changed = requested_output_extraction_plan_changed(ctx, extraction_plan)
reopened_after_failed_run = synthesized_persistence_reopened_after_failed_run(ctx)
- reopened = synthesized_persistence_reopened(ctx)
+ reopened = synthesized_persistence_reopened(ctx) or plan_changed
if getattr(ctx, "update_workflow_called", False) and not reopened:
return None
if normalize_block_authoring_policy(getattr(ctx, "block_authoring_policy", None)) != (
@@ -2249,11 +2260,29 @@ def _maybe_synthesized_block_offer_msg(ctx: Any) -> dict[str, Any] | None:
and not reopened
):
return None
- synthesized = synthesize_code_block(
- trajectory, reached_download_target=getattr(ctx, "reached_download_target", None)
+ synthesized = (
+ synthesize_code_block_with_extraction(
+ trajectory,
+ extraction_plan,
+ strict_selectors=True,
+ reached_download_target=getattr(ctx, "reached_download_target", None),
+ )
+ if extraction_plan is not None
+ else synthesize_code_block(
+ trajectory,
+ reached_download_target=getattr(ctx, "reached_download_target", None),
+ )
)
if synthesized is None:
return None
+ if extraction_plan is not None:
+ candidate = freeze_requested_output_extraction_candidate(synthesized, extraction_plan, source="generated")
+ if candidate is None:
+ return None
+ existing_candidate = getattr(ctx, "requested_output_extraction_candidate", None)
+ if existing_candidate is not None and existing_candidate != candidate and not reopened:
+ return None
+ ctx.requested_output_extraction_candidate = candidate
ctx.synthesized_block_offered = True
ctx.synthesized_block_offered_trajectory_len = trajectory_len
@@ -2400,6 +2429,39 @@ def uncovered_requested_output_paths(ctx: AgentContext) -> set[str]:
return {path for path in requested if path not in covered and tokens_by_path.get(path)}
+def requested_output_extraction_plan(ctx: AgentContext) -> RequestedOutputExtractionPlan | None:
+ requested_paths = _requested_output_paths_for_ctx(ctx)
+ if not requested_paths:
+ return None
+ labels_by_path: dict[str, tuple[str, ...]] = {}
+ for criterion in _pre_run_gated_completion_criteria(ctx):
+ if criterion.output_path in requested_paths and criterion.outcome.strip():
+ labels_by_path.setdefault(criterion.output_path, ())
+ labels_by_path[criterion.output_path] += (criterion.outcome.strip(),)
+ if set(labels_by_path) != requested_paths:
+ return None
+ return derive_requested_output_extraction_plan(
+ flow_evidence=ctx.flow_evidence,
+ labels_by_path=labels_by_path,
+ )
+
+
+def requested_output_extraction_plan_changed(ctx: AgentContext, current: RequestedOutputExtractionPlan | None) -> bool:
+ if current is None or len(ctx.flow_evidence) < 2:
+ return False
+ labels_by_path: dict[str, tuple[str, ...]] = {}
+ requested_paths = _requested_output_paths_for_ctx(ctx)
+ for criterion in _pre_run_gated_completion_criteria(ctx):
+ if criterion.output_path in requested_paths and criterion.outcome.strip():
+ labels_by_path.setdefault(criterion.output_path, ())
+ labels_by_path[criterion.output_path] += (criterion.outcome.strip(),)
+ previous = derive_requested_output_extraction_plan(
+ flow_evidence=ctx.flow_evidence[:-1],
+ labels_by_path=labels_by_path,
+ )
+ return previous is not None and previous.identity != current.identity
+
+
def record_scouted_output_coverage(ctx: AgentContext, page_evidence: dict[str, Any]) -> None:
coverage_tokens = _requested_output_coverage_tokens(ctx)
if not coverage_tokens:
@@ -2421,6 +2483,10 @@ def synthesized_trajectory_is_goal_complete(ctx: Any) -> bool:
byte-identically, so an entry ``synthesize_code_block`` would drop never counts as complete."""
if uncovered_requested_output_paths(ctx):
return False
+ requested_paths = _requested_output_paths_for_ctx(ctx)
+ plan = requested_output_extraction_plan(ctx)
+ if requested_paths and (plan is None or not requested_paths.issubset(set(plan.requested_output_paths))):
+ return False
trajectory = getattr(ctx, "scout_trajectory", None)
if not isinstance(trajectory, list) or not trajectory:
return False
diff --git a/skyvern/forge/sdk/copilot/mcp_adapter.py b/skyvern/forge/sdk/copilot/mcp_adapter.py
index 26875ceda..dca6cbacf 100644
--- a/skyvern/forge/sdk/copilot/mcp_adapter.py
+++ b/skyvern/forge/sdk/copilot/mcp_adapter.py
@@ -79,6 +79,7 @@ _POST_HOOK_CONTEXT_ROLLBACK_FIELDS = (
"synthesized_block_offered_trajectory_len",
"synthesized_block_offered_goal_complete",
"scouted_output_covered_paths",
+ "requested_output_extraction_candidate",
"synthesized_block_reopened_for_output_coverage",
"uncovered_output_rescout_context_key",
"uncovered_output_rescout_steer_key",
diff --git a/skyvern/forge/sdk/copilot/output_extraction_plan.py b/skyvern/forge/sdk/copilot/output_extraction_plan.py
new file mode 100644
index 000000000..5a6aba2f1
--- /dev/null
+++ b/skyvern/forge/sdk/copilot/output_extraction_plan.py
@@ -0,0 +1,304 @@
+"""Contract-owned JIT structural-read plans for requested Copilot outputs."""
+
+from __future__ import annotations
+
+import hashlib
+from dataclasses import dataclass
+from enum import StrEnum
+from typing import Any, TypeGuard
+
+
+class LiveReadKind(StrEnum):
+ KEY_VALUE = "key_value"
+ TABLE_COLUMN = "table_column"
+
+
+@dataclass(frozen=True, slots=True)
+class RevealAnchor:
+ selector: str = ""
+ role: str = ""
+ name: str = ""
+
+ def __post_init__(self) -> None:
+ if bool(self.selector) == bool(self.role and self.name):
+ raise ValueError("Reveal anchor must contain exactly one selector or role/name pair")
+
+
+@dataclass(frozen=True, slots=True)
+class LiveReadBinding:
+ output_path: str
+ kind: LiveReadKind
+ selector: str
+ selector_count: int
+ selector_index: int
+ child_index: int = 0
+ child_count: int = 0
+ row_selector: str = ""
+ row_count: int = 0
+ column_index: int = 0
+ relation_label: str = ""
+ headers: tuple[str, ...] = ()
+ row_cell_counts: tuple[int, ...] = ()
+ row_identities: tuple[str, ...] = ()
+
+
+@dataclass(frozen=True, slots=True)
+class RequestedOutputExtractionPlan:
+ requested_output_paths: tuple[str, ...]
+ observation_step: int
+ observation_identity: str
+ reveal: RevealAnchor
+ live_reads: tuple[LiveReadBinding, ...]
+ identity: str
+
+
+@dataclass(frozen=True, slots=True)
+class FrozenRequestedOutputExtractionCandidate:
+ plan_identity: str
+ observation_identity: str
+ requested_output_paths: tuple[str, ...]
+ reveal: RevealAnchor
+ interaction_code: str
+ extraction_code: str
+ source: str
+ admission_result: str
+ fingerprint: str
+
+
+def _stable_identity(value: str) -> str:
+ return hashlib.sha256(value.encode("utf-8")).hexdigest()
+
+
+def _is_int(value: Any) -> TypeGuard[int]:
+ return isinstance(value, int) and not isinstance(value, bool)
+
+
+def _leaf_paths(paths: set[str]) -> set[str]:
+ return {
+ path
+ for path in paths
+ if path.startswith("output.")
+ and not any(
+ other != path and (other.startswith(f"{path}.") or other.startswith(f"{path}[]")) for other in paths
+ )
+ }
+
+
+def output_path_segments(path: str) -> tuple[tuple[str, bool], ...]:
+ segments: list[tuple[str, bool]] = []
+ for raw_part in path.split("."):
+ part = raw_part.strip()
+ if not part:
+ continue
+ is_array = "[]" in part
+ name = part.replace("[]", "")
+ if name:
+ segments.append((name, is_array))
+ return tuple(segments)
+
+
+def _exact_path(label: str, labels_by_path: dict[str, tuple[str, ...]]) -> str | None:
+ matches = [path for path, labels in labels_by_path.items() if label in labels]
+ return matches[0] if len(matches) == 1 else None
+
+
+def _key_value_bindings(packet: dict[str, Any], labels_by_path: dict[str, tuple[str, ...]]) -> list[LiveReadBinding]:
+ relations = packet.get("key_value_relations")
+ if not isinstance(relations, list):
+ return []
+ bindings: list[LiveReadBinding] = []
+ for relation in relations:
+ if (
+ not isinstance(relation, dict)
+ or relation.get("visible") is not True
+ or relation.get("value_visible") is not True
+ ):
+ continue
+ label = relation.get("key_text")
+ selector = relation.get("container_selector")
+ match_count = relation.get("container_match_count")
+ position = relation.get("container_position")
+ child_index = relation.get("value_child_index")
+ child_count = relation.get("direct_child_count")
+ if not isinstance(label, str) or not isinstance(selector, str):
+ continue
+ if not _is_int(match_count) or not _is_int(position) or not _is_int(child_index) or not _is_int(child_count):
+ continue
+ if match_count <= position or position < 0 or child_index < 0 or child_count <= child_index:
+ continue
+ output_path = _exact_path(label, labels_by_path)
+ if output_path is None:
+ continue
+ bindings.append(
+ LiveReadBinding(
+ output_path,
+ LiveReadKind.KEY_VALUE,
+ selector,
+ match_count,
+ position,
+ child_index,
+ child_count,
+ relation_label=label,
+ )
+ )
+ return bindings
+
+
+def _table_bindings(packet: dict[str, Any], labels_by_path: dict[str, tuple[str, ...]]) -> list[LiveReadBinding]:
+ containers = packet.get("result_containers")
+ if not isinstance(containers, list):
+ return []
+ bindings: list[LiveReadBinding] = []
+ for container in containers:
+ if (
+ not isinstance(container, dict)
+ or container.get("visible") is not True
+ or container.get("span_free") is not True
+ or container.get("nested_table_free") is not True
+ ):
+ continue
+ selector, row_selector = container.get("selector"), container.get("row_selector")
+ match_count, row_count = container.get("selector_match_count"), container.get("row_count")
+ headers, rows, sample_rows = container.get("headers"), container.get("rows"), container.get("sample_rows")
+ if (
+ not isinstance(selector, str)
+ or not isinstance(row_selector, str)
+ or not _is_int(match_count)
+ or match_count != 1
+ ):
+ continue
+ if not _is_int(row_count) or row_count <= 0 or container.get("rows_truncated") is not False:
+ continue
+ if not isinstance(headers, list) or not isinstance(rows, list) or len(rows) != row_count:
+ continue
+ if (
+ not isinstance(sample_rows, list)
+ or len(sample_rows) != row_count
+ or not all(isinstance(value, str) for value in sample_rows)
+ ):
+ continue
+ header_label_list: list[str] = []
+ for header in headers:
+ if isinstance(header, dict) and isinstance(header.get("text"), str):
+ header_label_list.append(header["text"])
+ header_labels = tuple(header_label_list)
+ if len(header_labels) != len(headers):
+ continue
+ row_cell_counts: list[int] = []
+ for row_index, row in enumerate(rows):
+ if (
+ not isinstance(row, dict)
+ or row.get("row_index") != row_index
+ or row.get("visible") is not True
+ or row.get("has_row_header") is not False
+ ):
+ break
+ cells = row.get("cells")
+ if (
+ not isinstance(cells, list)
+ or len(cells) != len(headers)
+ or any(
+ not isinstance(cell, dict)
+ or cell.get("column_index") != column_index
+ or cell.get("visible") is not True
+ for column_index, cell in enumerate(cells)
+ )
+ ):
+ break
+ row_cell_counts.append(len(cells))
+ if len(row_cell_counts) != row_count:
+ continue
+ for header in headers:
+ column_index = header.get("column_index") if isinstance(header, dict) else None
+ if not isinstance(header, dict) or not isinstance(header.get("text"), str) or not _is_int(column_index):
+ continue
+ output_path = _exact_path(header["text"], labels_by_path)
+ if output_path is None:
+ continue
+ bindings.append(
+ LiveReadBinding(
+ output_path,
+ LiveReadKind.TABLE_COLUMN,
+ selector,
+ match_count,
+ 0,
+ row_selector=row_selector,
+ row_count=row_count,
+ column_index=column_index,
+ relation_label=header["text"],
+ headers=header_labels,
+ row_cell_counts=tuple(row_cell_counts),
+ row_identities=tuple(sample_rows),
+ )
+ )
+ return bindings
+
+
+def _array_prefix(path: str) -> tuple[tuple[str, bool], ...]:
+ segments = output_path_segments(path)
+ for index, (_, is_array) in enumerate(segments):
+ if is_array:
+ return segments[: index + 1]
+ return ()
+
+
+def _plan_from_entry(
+ entry: dict[str, Any], *, labels_by_path: dict[str, tuple[str, ...]]
+) -> RequestedOutputExtractionPlan | None:
+ if entry.get("reached_via") != "interaction" or entry.get("had_bounded_schema") is not True:
+ return None
+ step, packet = entry.get("step"), entry.get("evidence")
+ if not _is_int(step) or not isinstance(packet, dict) or packet.get("source_tool") != "scout_interaction":
+ return None
+ if (
+ packet.get("result_containers_truncated") is not False
+ or packet.get("key_value_relations_truncated") is not False
+ ):
+ return None
+ if isinstance(packet.get("inspection_warnings"), list) and packet["inspection_warnings"]:
+ return None
+ selector, role, name = (
+ packet.get("interaction_selector"),
+ packet.get("interaction_role"),
+ packet.get("interaction_accessible_name"),
+ )
+ if isinstance(selector, str) and selector:
+ reveal = RevealAnchor(selector=selector)
+ elif isinstance(role, str) and role and isinstance(name, str) and name:
+ reveal = RevealAnchor(role=role, name=name)
+ else:
+ return None
+ leaf_paths = _leaf_paths(set(labels_by_path))
+ live_reads = _key_value_bindings(packet, labels_by_path) + _table_bindings(packet, labels_by_path)
+ by_path: dict[str, list[LiveReadBinding]] = {}
+ for binding in live_reads:
+ by_path.setdefault(binding.output_path, []).append(binding)
+ if any(len(by_path.get(path, [])) != 1 for path in leaf_paths):
+ return None
+ ordered_reads = tuple(by_path[path][0] for path in sorted(leaf_paths))
+ tables_by_array: dict[tuple[tuple[str, bool], ...], set[tuple[str, int]]] = {}
+ for binding in ordered_reads:
+ prefix = _array_prefix(binding.output_path)
+ if prefix:
+ tables_by_array.setdefault(prefix, set()).add((binding.selector, binding.selector_index))
+ if any(len(tables) != 1 for tables in tables_by_array.values()):
+ return None
+ # The step is provenance, not structure: an identical re-observation must not
+ # invalidate the candidate frozen from the earlier offer.
+ observation_identity = _stable_identity(repr((reveal, ordered_reads)))
+ identity = _stable_identity(repr((tuple(sorted(labels_by_path)), observation_identity)))
+ return RequestedOutputExtractionPlan(
+ tuple(sorted(labels_by_path)), step, observation_identity, reveal, ordered_reads, identity
+ )
+
+
+def derive_requested_output_extraction_plan(
+ *, flow_evidence: list[dict[str, Any]], labels_by_path: dict[str, tuple[str, ...]]
+) -> RequestedOutputExtractionPlan | None:
+ """Derive from one rollback-owned packet; never combine partial observations."""
+ if not labels_by_path:
+ return None
+ for entry in reversed(flow_evidence):
+ if isinstance(entry, dict) and entry.get("reached_via") == "interaction":
+ return _plan_from_entry(entry, labels_by_path=labels_by_path)
+ return None
diff --git a/skyvern/forge/sdk/copilot/runtime.py b/skyvern/forge/sdk/copilot/runtime.py
index c90dd4453..8e2e0a166 100644
--- a/skyvern/forge/sdk/copilot/runtime.py
+++ b/skyvern/forge/sdk/copilot/runtime.py
@@ -54,6 +54,7 @@ if TYPE_CHECKING:
from skyvern.forge.sdk.copilot.completion_criteria_store import CompletionCriteriaTurnState
from skyvern.forge.sdk.copilot.completion_verification import CompletionVerificationResult
from skyvern.forge.sdk.copilot.context import CodeAuthoringRepairContext
+ from skyvern.forge.sdk.copilot.output_extraction_plan import FrozenRequestedOutputExtractionCandidate
from skyvern.forge.sdk.copilot.reached_download_target import ReachedDownloadTarget
from skyvern.forge.sdk.copilot.request_policy import RequestPolicy
from skyvern.forge.sdk.copilot.result_evidence import LoadedResultCompositionEvidence
@@ -428,6 +429,7 @@ class AgentContext:
synthesized_block_offered: bool = False
synthesized_block_offered_trajectory_len: int = 0
synthesized_block_offered_goal_complete: bool = False
+ requested_output_extraction_candidate: FrozenRequestedOutputExtractionCandidate | None = None
synthesized_block_reopened_after_failed_run: bool = False
synthesized_block_reopened_for_output_coverage: bool = False
scouted_output_covered_paths: set[str] = field(default_factory=set)
diff --git a/skyvern/forge/sdk/copilot/tools/workflow_update.py b/skyvern/forge/sdk/copilot/tools/workflow_update.py
index 08decf662..c1714ceff 100644
--- a/skyvern/forge/sdk/copilot/tools/workflow_update.py
+++ b/skyvern/forge/sdk/copilot/tools/workflow_update.py
@@ -13,6 +13,7 @@ from collections.abc import Callable, Iterable, Iterator, Mapping
from dataclasses import dataclass
from dataclasses import field as dataclass_field
from dataclasses import replace
+from enum import StrEnum
from typing import Annotated, Any, Literal, NamedTuple, cast
from urllib.parse import urlsplit
@@ -52,6 +53,7 @@ from skyvern.forge.sdk.copilot.code_block_security import CodeBlockSecurityError
from skyvern.forge.sdk.copilot.code_block_steps import apply_derived_code_block_steps, fill_code_block_prompts_in_yaml
from skyvern.forge.sdk.copilot.code_block_synthesis import (
_BARE_TAG_RE,
+ _INTERNAL_SCOUT_VARS,
_SYNTHESIZED_BLOCK_LABEL,
CREDENTIAL_FILL_TOOL_NAME,
SCOUTED_SPINE_UNDER_BUILD_REASON_CODE,
@@ -62,11 +64,13 @@ from skyvern.forge.sdk.copilot.code_block_synthesis import (
_parse_role_name,
artifact_dependency_id,
artifact_observation_ref_id,
+ freeze_requested_output_extraction_candidate,
locator_selector_literals,
missing_rung_text,
normalized_locator_expr,
render_missing_rung_call_sources,
synthesize_code_block,
+ synthesize_code_block_with_extraction,
uncovered_required_emitted_interactions,
)
from skyvern.forge.sdk.copilot.composition_evidence import (
@@ -86,6 +90,9 @@ from skyvern.forge.sdk.copilot.enforcement import (
_CHURN_REASON_CODES,
_record_code_authoring_guardrail_reject,
_scouted_spine_open_obligation,
+ requested_output_extraction_plan,
+ requested_output_extraction_plan_changed,
+ synthesized_persistence_reopened,
synthesized_persistence_reopened_after_failed_run,
)
from skyvern.forge.sdk.copilot.loop_detection import clear_failed_step_tracker_for_tools_in_ctx
@@ -5673,6 +5680,12 @@ class _BrowserBindings(NamedTuple):
method_aliases: set[str]
+class _BrowserExpressionKind(StrEnum):
+ LOCATOR = "locator"
+ SCALAR = "scalar"
+ OTHER = "other"
+
+
def _code_block_label(block: Mapping[str, Any]) -> str:
return str(block.get("label") or "").strip() or "unlabeled code block"
@@ -5703,6 +5716,15 @@ def _direct_page_method_signature(node: ast.AST) -> str | None:
return "page"
+def _bounded_nth_constant(node: ast.AST) -> bool:
+ return (
+ isinstance(node, ast.Constant)
+ and isinstance(node.value, int)
+ and not isinstance(node.value, bool)
+ and 0 <= node.value <= 10_000
+ )
+
+
def _browser_mutation_signature_for_call(node: ast.Call) -> _BrowserMutationSignature | None:
func = node.func
if not isinstance(func, ast.Attribute):
@@ -5828,18 +5850,104 @@ def _is_dynamic_browser_dispatch(node: ast.Call, bindings: _BrowserBindings) ->
return _expr_contains_browser_receiver(func.args[0], bindings)
+def _browser_expression_kind(node: ast.AST, bindings: _BrowserBindings) -> _BrowserExpressionKind:
+ # Alias trust is intentionally limited to compiler-owned scout variables;
+ # submitted-code aliases never gain browser-read authority by assignment.
+ if isinstance(node, ast.Name) and node.id in bindings.locator_aliases and node.id in _INTERNAL_SCOUT_VARS:
+ return _BrowserExpressionKind.LOCATOR
+ if _direct_locator_receiver_signature(node) is not None:
+ return _BrowserExpressionKind.LOCATOR
+ if isinstance(node, ast.Attribute) and node.attr in {"first", "last"}:
+ if _browser_expression_kind(node.value, bindings) == _BrowserExpressionKind.LOCATOR:
+ return _BrowserExpressionKind.LOCATOR
+ if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute):
+ receiver_kind = _browser_expression_kind(node.func.value, bindings)
+ if (
+ node.func.attr in {"locator", "get_by_role", "frame_locator"}
+ and receiver_kind == _BrowserExpressionKind.LOCATOR
+ ):
+ return _BrowserExpressionKind.LOCATOR
+ if node.func.attr == "nth" and receiver_kind == _BrowserExpressionKind.LOCATOR and len(node.args) == 1:
+ index = node.args[0]
+ if _bounded_nth_constant(index):
+ return _BrowserExpressionKind.LOCATOR
+ if node.func.attr in _LOCATOR_READ_METHODS and receiver_kind == _BrowserExpressionKind.LOCATOR:
+ return _BrowserExpressionKind.SCALAR
+ if (
+ node.func.attr in _PAGE_READ_METHODS
+ and isinstance(node.func.value, ast.Name)
+ and node.func.value.id == "page"
+ ):
+ return _BrowserExpressionKind.SCALAR
+ if (
+ node.func.attr
+ in {
+ "casefold",
+ "endswith",
+ "join",
+ "lower",
+ "lstrip",
+ "replace",
+ "rsplit",
+ "rstrip",
+ "split",
+ "startswith",
+ "strip",
+ "upper",
+ }
+ and receiver_kind == _BrowserExpressionKind.SCALAR
+ ):
+ return _BrowserExpressionKind.SCALAR
+ if isinstance(node, ast.Await):
+ return _browser_expression_kind(node.value, bindings)
+ if (
+ isinstance(node, ast.Call)
+ and isinstance(node.func, ast.Name)
+ and node.func.id
+ in {
+ "all",
+ "any",
+ "bool",
+ "dict",
+ "enumerate",
+ "float",
+ "int",
+ "len",
+ "list",
+ "max",
+ "min",
+ "range",
+ "set",
+ "sorted",
+ "str",
+ "sum",
+ "tuple",
+ }
+ ):
+ values = [*node.args, *(keyword.value for keyword in node.keywords)]
+ kinds = [_browser_expression_kind(value, bindings) for value in values]
+ if (
+ any(kind == _BrowserExpressionKind.SCALAR for kind in kinds)
+ and all(kind != _BrowserExpressionKind.LOCATOR for kind in kinds)
+ and all(
+ kind == _BrowserExpressionKind.SCALAR or not _expr_contains_browser_receiver(value, bindings)
+ for value, kind in zip(values, kinds, strict=True)
+ )
+ ):
+ return _BrowserExpressionKind.SCALAR
+ return _BrowserExpressionKind.OTHER
+
+
def _is_allowed_browser_read_call(node: ast.Call, bindings: _BrowserBindings) -> bool:
func = node.func
if not isinstance(func, ast.Attribute):
return False
if func.attr in _PAGE_FACTORY_METHODS:
- return isinstance(func.value, ast.Name) and func.value.id == "page"
+ return _browser_expression_kind(node, bindings) == _BrowserExpressionKind.LOCATOR
if func.attr in _PAGE_READ_METHODS and isinstance(func.value, ast.Name) and func.value.id == "page":
return True
if func.attr in _LOCATOR_READ_METHODS:
- return _direct_locator_receiver_signature(func.value) is not None or (
- isinstance(func.value, ast.Name) and func.value.id in bindings.locator_aliases
- )
+ return _browser_expression_kind(func.value, bindings) == _BrowserExpressionKind.LOCATOR
return False
@@ -5853,7 +5961,11 @@ def _browser_surface_for_code(code: str) -> tuple[list[_BrowserMutationSignature
page_aliases = _collect_page_aliases(tree)
locator_aliases = _collect_locator_aliases(tree)
method_aliases = _collect_browser_method_aliases(tree, page_aliases, locator_aliases)
- bindings = _BrowserBindings(page_aliases, locator_aliases, method_aliases)
+ bindings = _BrowserBindings(
+ page_aliases,
+ locator_aliases,
+ method_aliases,
+ )
# ast.walk yields breadth-first; the coverage scan (uncovered_required_emitted_interactions)
# matches draft calls as an ordered subsequence, so enumerate Call nodes in source order or a
# nested-but-present rung is falsely reported uncovered.
@@ -5875,6 +5987,11 @@ def _browser_surface_for_code(code: str) -> tuple[list[_BrowserMutationSignature
if _is_dynamic_browser_dispatch(node, bindings):
ambiguous.append(_call_name(node))
continue
+ if _browser_expression_kind(node, bindings) in {
+ _BrowserExpressionKind.LOCATOR,
+ _BrowserExpressionKind.SCALAR,
+ }:
+ continue
if _is_allowed_browser_read_call(node, bindings):
continue
func = node.func
@@ -6351,7 +6468,6 @@ def _selector_refines(bare: str, candidate: str) -> bool:
and bool(candidate_name)
and not candidate_suffix
)
-
if not _BARE_TAG_RE.match(bare):
return False
if not candidate.startswith(bare) or _is_positional_selector(candidate):
@@ -7479,18 +7595,50 @@ def _maybe_impose_synthesized_code_block(workflow_yaml: str, ctx: AgentContext)
if not _submitted_code_block_changed(code_block, prior_yaml):
return _SynthesizedCodeImpositionResult(workflow_yaml=workflow_yaml)
- synthesized = synthesize_code_block(
- scout_trajectory,
- strict_selectors=True,
- reached_download_target=ctx.reached_download_target,
+ extraction_plan = requested_output_extraction_plan(ctx)
+ extraction_candidate_refresh_allowed = synthesized_persistence_reopened(
+ ctx
+ ) or requested_output_extraction_plan_changed(ctx, extraction_plan)
+ synthesized = (
+ synthesize_code_block_with_extraction(
+ scout_trajectory,
+ extraction_plan,
+ strict_selectors=True,
+ reached_download_target=ctx.reached_download_target,
+ )
+ if extraction_plan is not None
+ else synthesize_code_block(
+ scout_trajectory,
+ strict_selectors=True,
+ reached_download_target=ctx.reached_download_target,
+ )
)
if synthesized is None:
return _SynthesizedCodeImpositionResult(
workflow_yaml=workflow_yaml,
violations=["Unable to impose synthesized code block: scout trajectory produced no runnable code."],
)
+ if extraction_plan is not None:
+ candidate = freeze_requested_output_extraction_candidate(synthesized, extraction_plan, source="generated")
+ if candidate is None:
+ return _SynthesizedCodeImpositionResult(
+ workflow_yaml=workflow_yaml,
+ violations=["Unable to impose synthesized code block: extraction candidate is incomplete."],
+ )
+ existing_candidate = ctx.requested_output_extraction_candidate
+ if (
+ existing_candidate is not None
+ and existing_candidate != candidate
+ and not extraction_candidate_refresh_allowed
+ ):
+ return _SynthesizedCodeImpositionResult(
+ workflow_yaml=workflow_yaml,
+ violations=["Unable to impose synthesized code block: extraction candidate identity changed."],
+ )
+ ctx.requested_output_extraction_candidate = candidate
- if _separated_spine_already_imposed(code_blocks, code_block, synthesized.code):
+ synthesized_spine_code = synthesized.interaction_code or synthesized.code
+ if _separated_spine_already_imposed(code_blocks, code_block, synthesized_spine_code):
return _SynthesizedCodeImpositionResult(workflow_yaml=workflow_yaml)
diagnostics = synthesized.diagnostics
@@ -7530,13 +7678,27 @@ def _maybe_impose_synthesized_code_block(workflow_yaml: str, ctx: AgentContext)
code_blocks=code_blocks,
selected_code_block=code_block,
submitted_selected_code=submitted_code,
- synthesized_code=synthesized.code,
+ synthesized_code=synthesized_spine_code,
prior_yaml=prior_yaml,
synthesized_diagnostics=diagnostics,
)
violations.extend(surface_validation.violations)
ambiguous_reject_present = any(record.kind == "ambiguous" for record in surface_validation.provenance)
- extraction_suffix = _submitted_suffix_after_synthesized_code(submitted_code, synthesized.code)
+ submitted_extraction_suffix = _submitted_suffix_after_synthesized_code(submitted_code, synthesized_spine_code)
+ extraction_suffix = submitted_extraction_suffix or synthesized.extraction_code
+ extraction_candidate_source = "submitted" if submitted_extraction_suffix else "generated"
+ if (
+ submitted_extraction_suffix
+ and synthesized.extraction_code
+ and submitted_extraction_suffix.strip() != synthesized.extraction_code.strip()
+ ):
+ return _SynthesizedCodeImpositionResult(
+ workflow_yaml=workflow_yaml,
+ violations=[
+ "Unable to impose synthesized code block: submitted extraction candidate does not execute the "
+ "captured requested-output plan recipe."
+ ],
+ )
if extraction_suffix:
suffix_mutations, _, suffix_ambiguous = _browser_surface_for_code(extraction_suffix)
selected_label = _code_block_label(code_block)
@@ -7616,7 +7778,7 @@ def _maybe_impose_synthesized_code_block(workflow_yaml: str, ctx: AgentContext)
textwrap.dedent(submitted_code), parameter_reconciliation
)
reconciled_synthesized_code = _apply_parameter_reconciliation_to_code(
- textwrap.dedent(synthesized.code), parameter_reconciliation
+ textwrap.dedent(synthesized_spine_code), parameter_reconciliation
)
submitted_contains_full_spine = _browser_surface_contains_full_action_spine(
reconciled_submitted_code, reconciled_synthesized_code
@@ -7640,7 +7802,7 @@ def _maybe_impose_synthesized_code_block(workflow_yaml: str, ctx: AgentContext)
metadata_declares_goal_values
and submitted_contains_full_spine
and not extraction_suffix
- and not _is_submitted_code_synthesized_only(submitted_code, synthesized.code)
+ and not _is_submitted_code_synthesized_only(submitted_code, synthesized_spine_code)
)
append_selected_extraction = (
metadata_declares_goal_values
@@ -7667,6 +7829,14 @@ def _maybe_impose_synthesized_code_block(workflow_yaml: str, ctx: AgentContext)
"selector_provenance": diagnostics.locator_provenance,
"prior_source": prior_source,
}
+ if extraction_plan is not None:
+ substitutions.update(
+ {
+ "extraction_plan_identity": extraction_plan.identity,
+ "extraction_candidate_fingerprint": synthesized.extraction_fingerprint,
+ "extraction_candidate_source": extraction_candidate_source,
+ }
+ )
reconciled_extraction_suffix = (
_submitted_suffix_after_synthesized_code(reconciled_submitted_code, reconciled_synthesized_code)
if preserve_submitted_extraction
@@ -7734,9 +7904,9 @@ def _maybe_impose_synthesized_code_block(workflow_yaml: str, ctx: AgentContext)
return split_under_build
substitutions["separated_browser_stage_count"] = len(durable_stage_codes)
else:
- imposed_code = textwrap.dedent(submitted_code if preserve_submitted_extraction else synthesized.code).lstrip(
- "\n"
- )
+ imposed_code = textwrap.dedent(
+ submitted_code if preserve_submitted_extraction else synthesized_spine_code
+ ).lstrip("\n")
if extraction_suffix:
imposed_code = imposed_code.rstrip() + "\n" + extraction_suffix.rstrip() + "\n"
elif append_selected_extraction:
@@ -7769,6 +7939,15 @@ def _maybe_impose_synthesized_code_block(workflow_yaml: str, ctx: AgentContext)
selected_extraction_metadata_disposition = "self_authored_extraction_preserved"
elif extraction_suffix or append_selected_extraction:
selected_extraction_metadata_disposition = "sibling_or_suffix_extraction_preserved"
+ if extraction_plan is not None:
+ LOG.info(
+ "copilot_requested_output_extraction_candidate_imposed",
+ canonical_paths=list(extraction_plan.requested_output_paths),
+ extraction_plan_identity=extraction_plan.identity,
+ observation_step=extraction_plan.observation_step,
+ candidate_source=extraction_candidate_source,
+ candidate_fingerprint=synthesized.extraction_fingerprint,
+ )
return _SynthesizedCodeImpositionResult(
workflow_yaml=yaml.safe_dump(parsed, sort_keys=False),
substitutions=substitutions,
diff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.py
index dbcb96807..ed3962076 100644
--- a/tests/unit/test_copilot_agent_helpers.py
+++ b/tests/unit/test_copilot_agent_helpers.py
@@ -23,6 +23,7 @@ from skyvern.forge.sdk.copilot.agent import (
_build_built_unverified_exit_result,
_build_goal_satisfied_exit_result,
_resolve_wrapped_exception_exit_result,
+ _synthesized_block_offer_prompt,
_verified_workflow_or_none,
)
from skyvern.forge.sdk.copilot.blocker_signal import CopilotToolBlockerSignal
@@ -113,6 +114,61 @@ from tests.unit.copilot_test_helpers import make_verified_goal_contract as _veri
_HISTORY_SENTINEL_TS = datetime(2026, 1, 1, tzinfo=timezone.utc)
+def test_prompt_offer_compiles_plan_and_refreshes_when_observation_changes() -> None:
+ criterion = CompletionCriterion(
+ id="record_id",
+ outcome="Record Identifier",
+ output_path="output.record_id",
+ )
+ ctx = _ctx()
+ ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER
+ ctx.request_policy = RequestPolicy(completion_criteria=[criterion])
+ ctx.completion_criteria_turn_state = SimpleNamespace(decision=SimpleNamespace(criteria=(criterion,)))
+ ctx.copilot_config = CopilotConfig(requested_output_path_aliases={"record identifier": "output.record_id"})
+ ctx.scout_trajectory = [
+ {"tool_name": "click", "selector": "#show-details", "source_url": "https://example.com/provider"}
+ ]
+
+ def packet(step: int) -> dict[str, object]:
+ return {
+ "step": step,
+ "reached_via": "interaction",
+ "had_bounded_schema": True,
+ "evidence": {
+ "source_tool": "scout_interaction",
+ "interaction_tool": "click",
+ "interaction_selector": "#show-details",
+ "inspection_warnings": [],
+ "result_containers_truncated": False,
+ "key_value_relations_truncated": False,
+ "key_value_relations": [
+ {
+ "key_text": "Record Identifier",
+ "container_selector": ".record-kv",
+ "container_match_count": 1,
+ "container_position": 0,
+ "value_child_index": 1,
+ "direct_child_count": 2,
+ "visible": True,
+ "value_visible": True,
+ }
+ ],
+ "result_containers": [],
+ },
+ }
+
+ ctx.flow_evidence = [packet(2)]
+ first = _synthesized_block_offer_prompt(ctx)
+ ctx.flow_evidence.append(packet(3))
+ second = _synthesized_block_offer_prompt(ctx)
+
+ assert 'page.locator(".record-kv").nth(0)' in first
+ assert 'return {"output": {"record_id": _extraction_value_0}}' in first
+ assert second == ""
+ assert "Extraction plan identity" in first
+ assert ctx.requested_output_extraction_candidate is not None
+
+
def _history(*pairs: tuple[str, str]) -> list[WorkflowCopilotChatHistoryMessage]:
return [
WorkflowCopilotChatHistoryMessage(
diff --git a/tests/unit/test_copilot_code_block_persist_seam.py b/tests/unit/test_copilot_code_block_persist_seam.py
index 0a455c029..26e5a9094 100644
--- a/tests/unit/test_copilot_code_block_persist_seam.py
+++ b/tests/unit/test_copilot_code_block_persist_seam.py
@@ -47,7 +47,7 @@ from skyvern.forge.sdk.copilot.code_block_synthesis import (
_get_by_role_expr_strict,
)
from skyvern.forge.sdk.copilot.completion_verification import CompletionVerificationResult, CriterionVerdict
-from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy
+from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy, CopilotConfig
from skyvern.forge.sdk.copilot.context import CodeAuthoringRepairContext, CopilotContext, FillCarry
from skyvern.forge.sdk.copilot.enforcement import (
MAX_CODE_AUTHORING_GUARDRAIL_REJECTS,
@@ -427,6 +427,114 @@ def _single_code_block(parsed: dict[str, object]) -> dict[str, object]:
return blocks[0]
+ "code",
+ [
+ 'value = (await page.locator("#record").inner_text()).strip()',
+ 'parts = (await page.locator("#record").inner_text()).split("|")',
+ 'value = await page.locator("#rows").nth(2).locator("td").nth(1).inner_text()',
+ ],
+)
+def test_browser_surface_admits_awaited_read_postprocessing_and_bounded_nth(code: str) -> None:
+ mutations, unscouted, ambiguous = workflow_update_module._browser_surface_for_code(code)
+
+ assert mutations == []
+ assert unscouted == []
+ assert ambiguous == []
+
+
+ "code",
+ [
+ 'locator = page.locator("#record")\nvalue = await locator.inner_text()',
+ 'read = page.locator("#record").inner_text\nvalue = await read()',
+ 'await getattr(page.locator("#record"), "inner_text")()',
+ 'await page.locator("#record").nth(-1).inner_text()',
+ 'await page.locator("#record").nth(10001).inner_text()',
+ 'for index in range(await page.locator("#rows").count()):\n value = await page.locator("#rows").nth(index).inner_text()',
+ 'await page.locator("#record").fill("changed")',
+ 'values = sorted(await page.locator("#rows").all_text_contents(), key=page.set_default_timeout)',
+ ],
+)
+def test_browser_surface_rejects_alias_dynamic_unbounded_and_mutation_forms(code: str) -> None:
+ mutations, unscouted, ambiguous = workflow_update_module._browser_surface_for_code(code)
+
+ assert mutations or unscouted or ambiguous
+
+
+def test_plan_backed_imposition_executes_generated_and_submitted_live_read_recipe() -> None:
+ ctx = _code_only_ctx()
+ _enable_imposition(ctx)
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ CompletionCriterion(
+ id="record_id",
+ outcome="Record Identifier",
+ output_path="output.record_id",
+ )
+ ]
+ )
+ ctx.completion_criteria_turn_state = SimpleNamespace(
+ decision=SimpleNamespace(criteria=tuple(ctx.request_policy.completion_criteria))
+ )
+ ctx.copilot_config = CopilotConfig(requested_output_path_aliases={"record identifier": "output.record_id"})
+ ctx.flow_evidence = [
+ {
+ "step": 2,
+ "reached_via": "interaction",
+ "had_bounded_schema": True,
+ "evidence": {
+ "source_tool": "scout_interaction",
+ "interaction_tool": "click",
+ "interaction_selector": "#search-submit",
+ "inspection_warnings": [],
+ "result_containers_truncated": False,
+ "key_value_relations_truncated": False,
+ "key_value_relations": [
+ {
+ "key_text": "Record Identifier",
+ "container_selector": ".record-kv",
+ "container_match_count": 1,
+ "container_position": 0,
+ "value_child_index": 1,
+ "direct_child_count": 2,
+ "visible": True,
+ "value_visible": True,
+ }
+ ],
+ "result_containers": [],
+ },
+ }
+ ]
+ submitted = _yaml(
+ """
+ title: Record lookup
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: extract_record
+ code: |
+ await page.locator("#search-submit").click()
+ """
+ )
+
+ result = workflow_update_module._maybe_impose_synthesized_code_block(submitted, ctx)
+
+ assert result.violations == []
+ code = str(_single_code_block(parse_workflow_yaml(result.workflow_yaml))["code"])
+ assert 'page.locator(".record-kv").nth(0)' in code
+ assert code.count('return {"output": {"record_id": _extraction_value_0}}') == 1
+ assert result.substitutions["extraction_candidate_source"] == "generated"
+ assert result.substitutions["extraction_plan_identity"]
+
+ submitted_result = workflow_update_module._maybe_impose_synthesized_code_block(result.workflow_yaml, ctx)
+
+ assert submitted_result.violations == []
+ assert submitted_result.substitutions["extraction_candidate_source"] == "submitted"
+ submitted_code = str(_single_code_block(parse_workflow_yaml(submitted_result.workflow_yaml))["code"])
+ assert submitted_code == code
+
+
def _credential_code_yaml(*, code: str, credential_id: str = "cred_missing") -> str:
indented_code = textwrap.indent(textwrap.dedent(code).strip(), " " * 8)
return (
diff --git a/tests/unit/test_copilot_code_block_synthesis.py b/tests/unit/test_copilot_code_block_synthesis.py
index 520de8b65..7821ed636 100644
--- a/tests/unit/test_copilot_code_block_synthesis.py
+++ b/tests/unit/test_copilot_code_block_synthesis.py
@@ -33,6 +33,8 @@ from skyvern.forge.sdk.copilot.code_block_synthesis import (
is_optional_dismissal_only_trajectory,
render_synthesized_offer_text,
synthesize_code_block,
+ synthesize_code_block_with_extraction,
+ synthesize_extraction_suffix,
uncovered_required_emitted_interactions,
)
from skyvern.forge.sdk.copilot.context import (
@@ -40,6 +42,12 @@ from skyvern.forge.sdk.copilot.context import (
StructuredContext,
_fill_carry_from_scout_trajectory,
)
+from skyvern.forge.sdk.copilot.output_extraction_plan import (
+ LiveReadBinding,
+ LiveReadKind,
+ RequestedOutputExtractionPlan,
+ RevealAnchor,
+)
from skyvern.forge.sdk.copilot.reached_download_target import ReachedDownloadTarget
from skyvern.forge.sdk.copilot.tools import _normalize_code_artifact_metadata
from skyvern.forge.sdk.copilot.tools.scouting import _fill_carry_to_interaction
@@ -68,6 +76,186 @@ def _interaction(tool_name: str, **fields: Any) -> dict[str, Any]:
return {"tool_name": tool_name, **fields}
+def _extraction_plan() -> RequestedOutputExtractionPlan:
+ return RequestedOutputExtractionPlan(
+ requested_output_paths=(
+ "output.records[].detail",
+ "output.records[].state",
+ "output.record_id",
+ "output.overall_state",
+ ),
+ observation_step=9,
+ observation_identity="observation-identity",
+ reveal=RevealAnchor(selector="#show-details"),
+ live_reads=(
+ LiveReadBinding(
+ output_path="output.record_id",
+ kind=LiveReadKind.KEY_VALUE,
+ selector=".kv",
+ selector_count=2,
+ selector_index=0,
+ child_index=1,
+ child_count=2,
+ relation_label="Record Identifier",
+ ),
+ LiveReadBinding(
+ output_path="output.records[].detail",
+ kind=LiveReadKind.TABLE_COLUMN,
+ selector="#records",
+ selector_count=1,
+ selector_index=0,
+ row_selector="#records > tbody > tr",
+ row_count=3,
+ column_index=1,
+ headers=("Record", "Detail", "State"),
+ row_cell_counts=(3, 3, 3),
+ row_identities=("One Detail State", "Two Detail State", "Three Detail State"),
+ ),
+ LiveReadBinding(
+ output_path="output.records[].state",
+ kind=LiveReadKind.TABLE_COLUMN,
+ selector="#records",
+ selector_count=1,
+ selector_index=0,
+ row_selector="#records > tbody > tr",
+ row_count=3,
+ column_index=2,
+ headers=("Record", "Detail", "State"),
+ row_cell_counts=(3, 3, 3),
+ row_identities=("One Detail State", "Two Detail State", "Three Detail State"),
+ ),
+ LiveReadBinding(
+ output_path="output.overall_state",
+ kind=LiveReadKind.KEY_VALUE,
+ selector=".kv",
+ selector_count=2,
+ selector_index=1,
+ child_index=1,
+ child_count=2,
+ relation_label="Overall State",
+ ),
+ ),
+ identity="plan-identity",
+ )
+
+
+def test_extraction_suffix_compiles_direct_guarded_live_reads() -> None:
+ suffix = synthesize_extraction_suffix(_extraction_plan())
+
+ assert suffix is not None
+ assert 'page.locator(".kv").nth(0).locator(":scope > *").nth(1).inner_text()' in suffix.code
+ assert 'page.locator("#records > tbody > tr").count() != 3' in suffix.code
+ assert "for _extraction_index" not in suffix.code
+ assert 'page.locator("#records > tbody > tr").nth(2)' in suffix.code
+ assert '"overall_state": _extraction_value_1' in suffix.code
+ assert '"detail"' in suffix.code
+ assert '"state"' in suffix.code
+
+
+class _RecipeLocator:
+ def __init__(self, page: _RecipePage, selector: str, indices: tuple[int, ...] = ()) -> None:
+ self.page = page
+ self.selector = selector
+ self.indices = indices
+
+ def nth(self, index: int) -> _RecipeLocator:
+ return _RecipeLocator(self.page, self.selector, (*self.indices, index))
+
+ def locator(self, selector: str) -> _RecipeLocator:
+ return _RecipeLocator(self.page, f"{self.selector}|{selector}", self.indices)
+
+ async def count(self) -> int:
+ return self.page.counts[(self.selector, self.indices)]
+
+ async def is_visible(self) -> bool:
+ return self.page.visibility.get((self.selector, self.indices), True)
+
+ async def inner_text(self) -> str:
+ return self.page.text[(self.selector, self.indices)]
+
+
+class _RecipePage:
+ def __init__(self) -> None:
+ self.visibility: dict[tuple[str, tuple[int, ...]], bool] = {}
+ self.counts: dict[tuple[str, tuple[int, ...]], int] = {
+ (".kv", ()): 2,
+ (".kv|:scope > *", (0,)): 2,
+ (".kv|:scope > *", (1,)): 2,
+ ("#records", ()): 1,
+ ("#records|:scope > thead > tr > th", (0,)): 3,
+ ("#records|[colspan], [rowspan]", (0,)): 0,
+ ("#records > tbody > tr", ()): 3,
+ ("#records|:scope table", (0,)): 0,
+ }
+ self.text: dict[tuple[str, tuple[int, ...]], str] = {
+ (".kv|:scope > *", (0, 0)): "Record Identifier",
+ (".kv|:scope > *", (0, 1)): "record-123",
+ (".kv|:scope > *", (1, 0)): "Overall State",
+ (".kv|:scope > *", (1, 1)): "Ready",
+ }
+ for index, header in enumerate(("Record", "Detail", "State")):
+ self.text[("#records|:scope > thead > tr > th", (0, index))] = header
+ for row, identity in enumerate(("One Detail State", "Two Detail State", "Three Detail State")):
+ self.counts[("#records > tbody > tr|:scope > th, :scope > td", (row,))] = 3
+ self.counts[("#records > tbody > tr|:scope > th", (row,))] = 0
+ self.text[("#records > tbody > tr", (row,))] = identity
+ self.text[("#records > tbody > tr|:scope > th, :scope > td", (row, 1))] = f"Detail {row}"
+ self.text[("#records > tbody > tr|:scope > th, :scope > td", (row, 2))] = "Ready"
+
+ def locator(self, selector: str) -> _RecipeLocator:
+ return _RecipeLocator(self, selector)
+
+
+async def _execute_recipe(page: _RecipePage) -> dict[str, object]:
+ suffix = synthesize_extraction_suffix(_extraction_plan())
+ assert suffix is not None
+ namespace: dict[str, object] = {}
+ exec("async def recipe(page):\n" + textwrap.indent(suffix.code, " "), namespace)
+ recipe = namespace["recipe"]
+ assert callable(recipe)
+ return await recipe(page)
+
+
+async def test_generated_recipe_executes_and_fails_closed_on_runtime_drift() -> None:
+ page = _RecipePage()
+ result = await _execute_recipe(page)
+ assert result["output"]["record_id"] == "record-123"
+ assert len(result["output"]["records"]) == 3
+
+ page.counts[(".kv", ())] = 3
+ with pytest.raises(ValueError, match="scalar selector cardinality"):
+ await _execute_recipe(page)
+
+ page.counts[(".kv", ())] = 2
+ page.visibility[("#records > tbody > tr|:scope > th, :scope > td", (1, 2))] = False
+ with pytest.raises(ValueError, match="cell is no longer visible"):
+ await _execute_recipe(page)
+
+
+def test_plan_compiler_requires_exact_reveal_and_is_idempotent() -> None:
+ trajectory = [
+ _interaction(
+ "click",
+ selector="#show-details",
+ role="button",
+ accessible_name="Show details",
+ source_url="https://example.com/records",
+ )
+ ]
+
+ first = synthesize_code_block_with_extraction(trajectory, _extraction_plan())
+ second = synthesize_code_block_with_extraction(trajectory, _extraction_plan())
+
+ assert first is not None
+ assert second is not None
+ assert first.code == second.code
+ assert first.code.count(".click()") == 1
+ assert first.extraction_plan_identity == "plan-identity"
+ assert first.extraction_fingerprint == second.extraction_fingerprint
+ assert synthesize_code_block_with_extraction([], _extraction_plan()) is None
+
+
class TestLocatorSynthesis:
def test_role_selector_emits_get_by_role(self) -> None:
# A `role=...` selector is an ARIA anchor (ref_to_selector form), not a native CSS path.
diff --git a/tests/unit/test_copilot_composition_evidence.py b/tests/unit/test_copilot_composition_evidence.py
index 358497963..9cb47b0f6 100644
--- a/tests/unit/test_copilot_composition_evidence.py
+++ b/tests/unit/test_copilot_composition_evidence.py
@@ -2217,6 +2217,128 @@ def test_structured_preserves_populated_result_container_content() -> None:
assert records["text_excerpt"] == "Record A ready"
+def test_structured_preserves_live_key_and_table_binding_shape() -> None:
+ payload = {
+ "page_title": "Records",
+ "forms": [],
+ "navigation_targets": [],
+ "result_containers": [
+ {
+ "tag": "table",
+ "selector": "#records",
+ "selector_match_count": 1,
+ "visible": True,
+ "span_free": True,
+ "nested_table_free": True,
+ "row_selector": "#records > tbody > tr",
+ "headers": [
+ {"text": "Address", "column_index": 0},
+ {"text": "Status", "column_index": 1},
+ ],
+ "row_count": 2,
+ "rows_truncated": False,
+ "rows": [
+ {
+ "row_index": row_index,
+ "visible": True,
+ "has_row_header": False,
+ "cells": [
+ {"column_index": 0, "visible": True},
+ {"column_index": 1, "visible": True},
+ ],
+ }
+ for row_index in range(2)
+ ],
+ "sample_rows": ["Record One Ready", "Record Two Pending"],
+ }
+ ],
+ "result_containers_truncated": False,
+ "key_value_relations": [
+ {
+ "key_text": "Record Identifier",
+ "container_selector": ".kv",
+ "container_match_count": 1,
+ "container_position": 0,
+ "value_child_index": 1,
+ "direct_child_count": 2,
+ "visible": True,
+ "value_visible": True,
+ }
+ ],
+ "key_value_relations_truncated": False,
+ "challenge_controls": [],
+ "modal_overlays": [],
+ "visual_obstruction_candidates": [],
+ "visible_text_excerpt": "Record details",
+ "anti_bot_indicators": [],
+ }
+
+ parsed = parse_composition_structured(
+ payload, inspected_url="https://example.com/records", current_url="https://example.com/records"
+ )
+
+ assert parsed is not None
+ assert parsed["key_value_relations"] == payload["key_value_relations"]
+ assert parsed["key_value_relations_truncated"] is False
+ assert parsed["result_containers_truncated"] is False
+ assert parsed["result_containers"][0]["headers"] == payload["result_containers"][0]["headers"]
+ assert parsed["result_containers"][0]["selector_match_count"] == 1
+ assert parsed["result_containers"][0]["rows_truncated"] is False
+ assert parsed["result_containers"][0]["nested_table_free"] is True
+ assert parsed["result_containers"][0]["row_selector"] == "#records > tbody > tr"
+ assert parsed["result_containers"][0]["rows"][1]["has_row_header"] is False
+ assert parsed["result_containers"][0]["rows"][1]["cells"][1] == {
+ "column_index": 1,
+ "visible": True,
+ }
+
+
+def test_html_packet_excludes_hidden_bindings_and_preserves_revealed_structure() -> None:
+ details = """
+ <div id="details" STYLE>
+ <div class="kv"><div>Record Identifier</div><div>record-123</div></div>
+ <table id="records">
+ <thead><tr><th>Address</th><th>Status</th></tr></thead>
+ <tbody><tr><td>Record One</td><td>Ready</td></tr></tbody>
+ </table>
+ </div>
+ """
+ hidden = parse_composition_html(
+ details.replace("STYLE", 'style="display:none"'),
+ inspected_url="https://example.com/records",
+ current_url="https://example.com/records",
+ )
+ revealed = parse_composition_html(
+ details.replace("STYLE", ""),
+ inspected_url="https://example.com/records",
+ current_url="https://example.com/records",
+ )
+
+ assert hidden["key_value_relations"] == []
+ assert hidden["result_containers"] == []
+ assert revealed["key_value_relations"][0]["key_text"] == "Record Identifier"
+ table = revealed["result_containers"][0]
+ assert table["selector"] == "#records"
+ assert table["selector_match_count"] == 1
+ assert table["headers"] == [
+ {"text": "Address", "column_index": 0},
+ {"text": "Status", "column_index": 1},
+ ]
+ assert table["row_count"] == 1
+ assert table["rows_truncated"] is False
+ assert table["rows"] == [
+ {
+ "row_index": 0,
+ "visible": True,
+ "has_row_header": False,
+ "cells": [
+ {"column_index": 0, "visible": True},
+ {"column_index": 1, "visible": True},
+ ],
+ }
+ ]
+
+
def test_structured_detects_modal_overlay_with_dismiss_controls() -> None:
payload = {
"page_title": "",
diff --git a/tests/unit/test_copilot_enforcement_pruning.py b/tests/unit/test_copilot_enforcement_pruning.py
index 1ebfd75b0..931ae2e41 100644
--- a/tests/unit/test_copilot_enforcement_pruning.py
+++ b/tests/unit/test_copilot_enforcement_pruning.py
@@ -29,7 +29,11 @@ from skyvern.forge.sdk.copilot.build_test_outcome import (
)
from skyvern.forge.sdk.copilot.code_block_synthesis import SynthesizedCodeBlock
from skyvern.forge.sdk.copilot.completion_verification import CompletionVerificationResult, CriterionVerdict
-from skyvern.forge.sdk.copilot.config import SYNTHESIZED_OFFER_REFRESH_STEP_THRESHOLD, BlockAuthoringPolicy
+from skyvern.forge.sdk.copilot.config import (
+ SYNTHESIZED_OFFER_REFRESH_STEP_THRESHOLD,
+ BlockAuthoringPolicy,
+ CopilotConfig,
+)
from skyvern.forge.sdk.copilot.context import CodeAuthoringRepairContext
from skyvern.forge.sdk.copilot.enforcement import (
KEEP_RECENT_TOOL_OUTPUTS,
@@ -123,6 +127,8 @@ class _Ctx:
self.synthesized_block_reopened_after_failed_run = False
self.synthesized_block_reopened_for_output_coverage = False
self.scouted_output_covered_paths: set[str] = set()
+ self.flow_evidence: list[dict[str, object]] = []
+ self.copilot_config: CopilotConfig | None = None
self.uncovered_output_rescout_context_key = None
self.uncovered_output_rescout_steer_key = None
self.latest_recorded_build_test_outcome = None
@@ -2254,6 +2260,49 @@ class TestScoutOutputCoverageGate:
ctx.synthesized_block_offered_goal_complete = synthesized_trajectory_is_goal_complete(ctx)
return ctx
+ @staticmethod
+ def _attach_document_plan(ctx: _Ctx, *, step: int) -> None:
+ ctx.copilot_config = CopilotConfig(requested_output_path_aliases={"document name": "output.document_name"})
+ ctx.flow_evidence = [
+ {
+ "step": step,
+ "reached_via": "interaction",
+ "had_bounded_schema": True,
+ "evidence": {
+ "source_tool": "scout_interaction",
+ "interaction_tool": "click",
+ "interaction_selector": "button[data-action='search']",
+ "inspection_warnings": [],
+ "result_containers_truncated": False,
+ "key_value_relations_truncated": False,
+ "key_value_relations": [
+ {
+ "key_text": "Document Name",
+ "container_selector": ".document-kv",
+ "container_match_count": 1,
+ "container_position": 0,
+ "value_child_index": 1,
+ "direct_child_count": 2,
+ "visible": True,
+ "value_visible": True,
+ }
+ ],
+ "result_containers": [],
+ },
+ }
+ ]
+
+ def test_post_turn_offer_compiles_plan_recipe(self) -> None:
+ ctx = self._authoring_ctx(_criterion("output.document_name", "Document Name"))
+ self._attach_document_plan(ctx, step=6)
+ ctx.synthesized_block_offered = False
+
+ message = _maybe_synthesized_block_offer_msg(ctx)
+
+ assert message is not None
+ assert 'page.locator(".document-kv").nth(0)' in str(message["content"])
+ assert 'return {"output": {"document_name": _extraction_value_0}}' in str(message["content"])
+
def test_empty_output_set_falls_through_to_shape_heuristic(self) -> None:
ctx = self._authoring_ctx()
assert uncovered_requested_output_paths(ctx) == set()
@@ -2291,7 +2340,7 @@ class TestScoutOutputCoverageGate:
assert _should_block_mutating_tool_after_synthesized_offer(ctx, "click") is False
assert synthesized_block_persistence_signal(ctx, "click") is None
- def test_value_bearing_container_covers_path_and_force_fires(self) -> None:
+ def test_value_bearing_container_coverage_without_plan_does_not_force(self) -> None:
ctx = self._authoring_ctx(_criterion("output.document_name", "the order status document name is captured"))
page_evidence = {
"result_containers": [
@@ -2302,8 +2351,8 @@ class TestScoutOutputCoverageGate:
assert ctx.scouted_output_covered_paths == {"output.document_name"}
assert uncovered_requested_output_paths(ctx) == set()
ctx.synthesized_block_offered_goal_complete = synthesized_trajectory_is_goal_complete(ctx)
- assert synthesized_trajectory_is_goal_complete(ctx) is True
- assert _should_force_synthesized_block_persistence(ctx) is True
+ assert synthesized_trajectory_is_goal_complete(ctx) is False
+ assert _should_force_synthesized_block_persistence(ctx) is False
def test_empty_shell_selector_tokens_do_not_credit(self) -> None:
ctx = self._authoring_ctx(_criterion("output.document_name", "the order status document name is captured"))
@@ -2345,12 +2394,12 @@ class TestScoutOutputCoverageGate:
ctx.completion_criteria_turn_state = SimpleNamespace(decision=None)
assert uncovered_requested_output_paths(ctx) == set()
- def test_all_generic_token_path_is_exempt_and_falls_through_to_shape(self) -> None:
+ def test_all_generic_token_path_still_requires_producer_plan(self) -> None:
ctx = self._authoring_ctx(_criterion("output.data", "the data is captured"))
assert uncovered_requested_output_paths(ctx) == set()
ctx.synthesized_block_offered_goal_complete = synthesized_trajectory_is_goal_complete(ctx)
- assert synthesized_trajectory_is_goal_complete(ctx) is True
- assert _should_force_synthesized_block_persistence(ctx) is True
+ assert synthesized_trajectory_is_goal_complete(ctx) is False
+ assert _should_force_synthesized_block_persistence(ctx) is False
def test_generic_path_exemption_keeps_specific_path_gating(self) -> None:
ctx = self._authoring_ctx(
@@ -2542,7 +2591,7 @@ class TestScoutOutputCoverageGate:
assert _uncovered_output_reject_admits_evaluate(ctx, "evaluate") is False
assert consume_uncovered_output_reopen_event(ctx) is False
- def test_coverage_reopen_refreshes_synthesized_offer_after_authoring(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ def test_coverage_reopen_without_plan_does_not_refresh_offer(self, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(
"skyvern.forge.sdk.copilot.enforcement.synthesize_code_block",
lambda *args, **kwargs: SynthesizedCodeBlock(code="await page.click('button')"),
@@ -2551,7 +2600,7 @@ class TestScoutOutputCoverageGate:
ctx.update_workflow_called = True
assert _maybe_synthesized_block_offer_msg(ctx) is None
ctx.synthesized_block_reopened_for_output_coverage = True
- assert _maybe_synthesized_block_offer_msg(ctx) is not None
+ assert _maybe_synthesized_block_offer_msg(ctx) is None
def test_post_hook_failure_rolls_back_coverage_credit(self) -> None:
assert "scouted_output_covered_paths" in _POST_HOOK_CONTEXT_ROLLBACK_FIELDS
diff --git a/tests/unit/test_copilot_output_extraction_plan.py b/tests/unit/test_copilot_output_extraction_plan.py
new file mode 100644
index 000000000..536846ca3
--- /dev/null
+++ b/tests/unit/test_copilot_output_extraction_plan.py
@@ -0,0 +1,223 @@
+from __future__ import annotations
+
+from types import SimpleNamespace
+
+from skyvern.forge.sdk.copilot.composition_evidence import parse_composition_structured
+from skyvern.forge.sdk.copilot.mcp_adapter import _restore_post_hook_context, _snapshot_post_hook_context
+from skyvern.forge.sdk.copilot.output_extraction_plan import (
+ LiveReadKind,
+ derive_requested_output_extraction_plan,
+)
+
+LABELS_BY_PATH = {
+ "output.record_id": ("Record Identifier",),
+ "output.records[].detail": ("Detail",),
+ "output.records[].state": ("State",),
+ "output.overall_state": ("Overall State",),
+}
+
+
+def _flow_packet(*, visible: bool = True, ambiguous_id: bool = False, truncated: bool = False) -> dict[str, object]:
+ id_relation = {
+ "key_text": "Record Identifier",
+ "container_selector": ".kv",
+ "container_match_count": 7,
+ "container_position": 0,
+ "value_child_index": 1,
+ "direct_child_count": 2,
+ "visible": visible,
+ "value_visible": visible,
+ }
+ overall_relation = dict(id_relation, key_text="Overall State", container_position=2)
+ relations = [id_relation, overall_relation]
+ if ambiguous_id:
+ relations.append(dict(id_relation, container_position=1))
+ return {
+ "step": 4,
+ "reached_via": "interaction",
+ "had_bounded_schema": True,
+ "evidence": {
+ "source_tool": "scout_interaction",
+ "interaction_tool": "click",
+ "interaction_selector": "#show-details",
+ "inspection_warnings": [],
+ "result_containers_truncated": truncated,
+ "key_value_relations_truncated": False,
+ "key_value_relations": relations,
+ "result_containers": [
+ {
+ "tag": "table",
+ "selector": "#records",
+ "selector_match_count": 1,
+ "visible": visible,
+ "span_free": True,
+ "nested_table_free": True,
+ "headers": [
+ {"text": "Record", "column_index": 0},
+ {"text": "Detail", "column_index": 1},
+ {"text": "State", "column_index": 2},
+ ],
+ "row_selector": "#records tbody tr",
+ "row_count": 3,
+ "rows_truncated": False,
+ "sample_rows": [f"Record {row_index} Detail State" for row_index in range(3)],
+ "rows": [
+ {
+ "row_index": row_index,
+ "visible": True,
+ "has_row_header": False,
+ "cells": [
+ {"column_index": 0, "visible": True},
+ {"column_index": 1, "visible": True},
+ {"column_index": 2, "visible": True},
+ ],
+ }
+ for row_index in range(3)
+ ],
+ }
+ ],
+ },
+ }
+
+
+def _derive(*, packet: dict[str, object] | None = None):
+ return derive_requested_output_extraction_plan(
+ flow_evidence=[packet or _flow_packet()],
+ labels_by_path=LABELS_BY_PATH,
+ )
+
+
+def test_derives_complete_plan_from_one_visible_interaction_packet() -> None:
+ plan = _derive()
+
+ assert plan is not None
+ assert plan.observation_step == 4
+ assert plan.reveal.selector == "#show-details"
+ assert {binding.output_path for binding in plan.live_reads} == {
+ "output.record_id",
+ "output.records[].detail",
+ "output.records[].state",
+ "output.overall_state",
+ }
+ assert {binding.kind for binding in plan.live_reads} == {LiveReadKind.KEY_VALUE, LiveReadKind.TABLE_COLUMN}
+ assert plan.identity
+
+
+def test_identical_reobservation_keeps_structural_candidate_identity() -> None:
+ first = _flow_packet()
+ second = _flow_packet()
+ second["step"] = 5
+
+ first_plan = _derive(packet=first)
+ second_plan = _derive(packet=second)
+
+ assert first_plan is not None
+ assert second_plan is not None
+ assert first_plan.observation_step != second_plan.observation_step
+ assert first_plan.observation_identity == second_plan.observation_identity
+ assert first_plan.identity == second_plan.identity
+
+
+def test_derives_table_plan_through_production_structured_normalizer() -> None:
+ raw_evidence = _flow_packet()["evidence"]
+ parsed = parse_composition_structured(
+ raw_evidence,
+ inspected_url="https://example.com/records",
+ current_url="https://example.com/records",
+ )
+ assert parsed is not None
+ parsed.update(
+ source_tool="scout_interaction",
+ interaction_tool="click",
+ interaction_selector="#show-details",
+ )
+
+ plan = derive_requested_output_extraction_plan(
+ flow_evidence=[
+ {
+ "step": 4,
+ "reached_via": "interaction",
+ "had_bounded_schema": True,
+ "evidence": parsed,
+ }
+ ],
+ labels_by_path=LABELS_BY_PATH,
+ )
+
+ assert plan is not None
+ assert {binding.output_path for binding in plan.live_reads if binding.kind == LiveReadKind.TABLE_COLUMN} == {
+ "output.records[].detail",
+ "output.records[].state",
+ }
+
+
+def test_hidden_ambiguous_truncated_and_mixed_packets_fail_closed() -> None:
+ assert _derive(packet=_flow_packet(visible=False)) is None
+ assert _derive(packet=_flow_packet(ambiguous_id=True)) is None
+ assert _derive(packet=_flow_packet(truncated=True)) is None
+
+ hidden_row = _flow_packet()
+ hidden_row["evidence"]["result_containers"][0]["rows"][1]["visible"] = False
+ assert _derive(packet=hidden_row) is None
+
+ partial_id = _flow_packet()
+ partial_id["evidence"] = dict(partial_id["evidence"], result_containers=[])
+ partial_table = _flow_packet()
+ partial_table["step"] = 5
+ partial_table["evidence"] = dict(partial_table["evidence"], key_value_relations=[])
+ assert (
+ derive_requested_output_extraction_plan(
+ flow_evidence=[partial_id, partial_table],
+ labels_by_path={key: value for key, value in LABELS_BY_PATH.items() if key != "output.overall_state"},
+ )
+ is None
+ )
+ assert (
+ derive_requested_output_extraction_plan(
+ flow_evidence=[_flow_packet(), partial_table],
+ labels_by_path={key: value for key, value in LABELS_BY_PATH.items() if key != "output.overall_state"},
+ )
+ is None
+ )
+
+
+def test_only_exact_configured_aliases_bind_observed_labels() -> None:
+ fuzzy = _flow_packet()
+ fuzzy["evidence"]["key_value_relations"][0]["key_text"] = "Approximate record value"
+
+ assert _derive(packet=fuzzy) is None
+
+
+def test_table_identity_coordinate_and_shape_ambiguity_fail_closed() -> None:
+ nested = _flow_packet()
+ nested["evidence"]["result_containers"][0]["nested_table_free"] = False
+ assert _derive(packet=nested) is None
+
+ row_header = _flow_packet()
+ row_header["evidence"]["result_containers"][0]["rows"][0]["has_row_header"] = True
+ assert _derive(packet=row_header) is None
+
+ shifted_cell = _flow_packet()
+ shifted_cell["evidence"]["result_containers"][0]["rows"][1]["cells"][1]["column_index"] = 2
+ assert _derive(packet=shifted_cell) is None
+
+ span = _flow_packet()
+ span["evidence"]["result_containers"][0]["span_free"] = False
+ assert _derive(packet=span) is None
+
+
+def test_jit_plan_cannot_see_flow_evidence_rolled_back_after_failed_hook() -> None:
+ ctx = SimpleNamespace(flow_evidence=[])
+ snapshot = _snapshot_post_hook_context(ctx)
+ ctx.flow_evidence.append(_flow_packet())
+ assert _derive(packet=ctx.flow_evidence[0]) is not None
+
+ _restore_post_hook_context(ctx, snapshot)
+
+ assert (
+ derive_requested_output_extraction_plan(
+ flow_evidence=ctx.flow_evidence,
+ labels_by_path={"output.record_id": ("Record Identifier",)},
+ )
+ is None
+ )